update filter logic and av. check

Author: codingPF

src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java

@@ -10,7 +10,6 @@
 
 import de.mediathekview.mserver.daten.Film;
 import de.mediathekview.mserver.daten.Filmlist;
-import de.mediathekview.mserver.daten.Resolution;
 import de.mediathekview.mserver.daten.Sender;
 import de.mediathekview.mserver.base.utils.FileSizeDeterminer.ResponseInfo;
 

src/main/java/de/mediathekview/mserver/base/utils/FilmDBService.java

@@ -142,20 +142,18 @@ public <T> List<T> filterNewVideos(List<T> videos, Function<T, String> idExtract
     }
     try {
       List<Future<List<T>>> futures = new ArrayList<>();
-      
+      // sort to avoid deadlocks
       List<T> allVideos = videos.stream()
           .sorted(Comparator.comparing(idExtractor))
           .toList();
-
       for (int i = 0; i < allVideos.size(); i += batchSize) {
         int from = i;
         int to = Math.min(i + batchSize, allVideos.size());
         List<T> batch = allVideos.subList(from, to);
-
         futures.add(executorService.submit(() -> {
           List<T> newVideos = new ArrayList<>();
-
-          String sql = "UPDATE filme SET last_seen = now() WHERE id = ?";
+          // update every 7 days
+          String sql = "UPDATE filme SET last_seen = now() WHERE id = ? AND last_seen - last_update <= interval '7' DAY";
 
           try (Connection con = dataSource.getConnection(); PreparedStatement ps = con.prepareStatement(sql)) {
 
@@ -266,12 +264,11 @@ private int saveBatch(List<Film> films) throws SQLException, IOException {
     int successCounter = 0;
 
     String sql = """
-            INSERT INTO filme (id, data, created_at, last_update)
-            VALUES (?, ?::jsonb, now(), now())
+            INSERT INTO filme (id, data)
+            VALUES (?, ?::jsonb)
             ON CONFLICT (id) DO UPDATE
             SET data = EXCLUDED.data,
-                last_update = now(),
-                created_at = filme.created_at
+                last_update = now()
         """;
 
     try (Connection con = dataSource.getConnection(); PreparedStatement ps = con.prepareStatement(sql)) {

src/main/java/de/mediathekview/mserver/ui/config/MServerExecutionFlow.java

@@ -105,7 +105,7 @@ void importFilmlistIntoDB() {
   }
   void checkAvailability() {
     FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
-    String condition = "where last_url_check IS NULL OR last_url_check < NOW() - INTERVAL '1 DAY'";
+    String condition = "WHERE last_url_check < NOW() - INTERVAL '3' DAY LIMIT 400000";
     Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB(condition);
     dbFilmlist.ifPresent(filmlist -> manager.getFilmlist().addAllFilms(filmlist.getFilms().values()));
     CheckUrlAvailability checkUrlAvailability = new CheckUrlAvailability(