Merge branch 'feature/ArdGroupStructure' into develop3

codingPF · codingPF · commit 75f36a5dfbf4 · 2026-01-25T11:12:26.000+01:00
diff --git a/src/main/docker/runDocker b/src/main/docker/runDocker
@@ -4,16 +4,16 @@
 docker compose up -d postgresMV
 
 # cron 01 01 long run
-docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R1.yaml" mserver-r1
+docker compose run -d --rm -e MSERVER_OPTS="--config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R1.yaml" mserver-r1
 
 # cron 06-22 1,31 short run
-docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R2.yaml" mserver-r2
+docker compose run -d --rm -e MSERVER_OPTS="--config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R2.yaml" mserver-r2
 
 # cron 22 55 url check
-docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R2.yaml --flow checkAvailability" mserver-checkUrls
+docker compose run -d --rm -e MSERVER_OPTS="--config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R2.yaml --flow checkAvailability" mserver-checkUrls
 
 # on demand - do not run this unless you know what you are doing!
-docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R2.yaml --flow importFilmlistIntoDB" mserver-r3
+docker compose run -d --rm -e MSERVER_OPTS="--config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R2.yaml --flow importFilmlistIntoDB" mserver-r3
 
 ## docker save -o mserver.tar mediathekview/mserver:4.0.1-SNAPSHOT
 ## docker load -i mserver.tar 
diff --git a/src/main/java/de/mediathekview/mserver/base/utils/FilmDBService.java b/src/main/java/de/mediathekview/mserver/base/utils/FilmDBService.java
@@ -88,6 +88,7 @@ public void updateLastUrlCheck(List<Film> checked) {
               ps.addBatch();
             }
             int [] rs = ps.executeBatch();
+            con.commit();
             for (int rsCode : rs) {
               updateCounter.addAndGet(rsCode);
             }
@@ -124,6 +125,7 @@ public void deleteFilms(Collection<Film> abandonedFilmlist) {
               ps.addBatch();
             }
             ps.executeBatch();
+            con.commit();
           } catch (SQLException e) {
             LOG.error(e);
           }
@@ -203,6 +205,7 @@ public <T> List<T> filterNewVideos(Sender sender, List<T> videos, Function<T, St
               }
             }
             int[] rs = ps.executeBatch();
+            con.commit();
             for (int rsIndex = 0; rsIndex < rs.length; rsIndex++) {
               if (rs[rsIndex] == 0) {
                 newVideos.add(batch.get(rsIndex));
@@ -220,6 +223,10 @@ public <T> List<T> filterNewVideos(Sender sender, List<T> videos, Function<T, St
         result.addAll(f.get());
       }
       LOG.debug("Filtered {} in {} (in {} vs out {})",(videos.size()-result.size()), sender.getName(), videos.size(), result.size());
+      // CARP films pro Nacht. Die filme werden dann am nächsten tag gefunden. TODO: für die nächste runde.
+      /* result = new ArrayList<>(result.subList(0, 200000));
+       * 
+       */
       return result;
     } catch (Exception e) {
       LOG.error("{}", e);
@@ -319,8 +326,8 @@ ON CONFLICT (id) DO UPDATE
           LOG.error("saveBatch - Missing ID for film {}", film);
         }
       }
-
       ps.executeBatch();
+      con.commit();
     }
     return successCounter;
   }
diff --git a/src/main/java/de/mediathekview/mserver/base/utils/PostgreSQLDataSourceProvider.java b/src/main/java/de/mediathekview/mserver/base/utils/PostgreSQLDataSourceProvider.java
@@ -39,19 +39,19 @@ private void init() {
     cfg.setUsername(aMServerConfigManager.getConfig().getDatabaseConfig().getUsername());
     cfg.setPassword(aMServerConfigManager.getConfig().getDatabaseConfig().getPassword());
 
-    // === Pool Sizing (wichtig!) ===
-    cfg.setMaximumPoolSize(16); // Sweet Spot für 10k+/min
+    // === Pool Sizing ===
+    cfg.setMaximumPoolSize(50);
     cfg.setMinimumIdle(4);
 
     // === Performance ===
-    cfg.setAutoCommit(true);
+    cfg.setAutoCommit(false);
     cfg.setConnectionTimeout(3000);
     cfg.setIdleTimeout(600_000);
     cfg.setMaxLifetime(1_800_000);
 
     // === PostgreSQL Optimierungen ===
-    cfg.addDataSourceProperty("reWriteBatchedInserts", "true");
     cfg.addDataSourceProperty("stringtype", "unspecified");
+    cfg.addDataSourceProperty("defaultRowFetchSize", "10000");
 
     // === Debug (optional) ===
     cfg.setPoolName("CrawlerPool");
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java b/src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java
@@ -123,7 +123,7 @@ private ForkJoinTask<Set<CrawlerUrlDTO>> getTopicEntriesBySender(final String se
      Set<CrawlerUrlDTO> senderSingleLetterUrls = forkJoinPool.submit(
         new ArdTopicsTask(this, sender, CreateLetterUrlQuery(sender))).get();
 
-     LOG.debug("topics task result {}", senderSingleLetterUrls.size());
+     //LOG.debug("topics task result {}", senderSingleLetterUrls.size());
      return forkJoinPool.submit(new ArdTopicsLetterTask(this, sender, new ConcurrentLinkedQueue<>(senderSingleLetterUrls)));
   }
 
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTeasersDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTeasersDeserializer.java
@@ -60,19 +60,7 @@ private ArdFilmInfoDto createFilmInfo(final String id, final int numberOfClips)
     if(id.contains(":")) {
       refId = id.replace(":", "%3A");
     }
-    
     final String url = String.format(ArdConstants.ITEM_URL, refId);
-    
-    
-    
-    if (id.contains("a04c5a47-0801-40e5-b530-b7f9a4312be9:6898178275329995836") 
-        || id.contains("Y3JpZDovL25kci5kZS9wcm9wbGFuXzE5NjM4MTA5N19nYW56ZVNlbmR1bmc")
-        || id.contains("1TDLUvc8cVEtcSb9GGsOnt:6898178275329995836")
-        || id.contains("6b64fc2c-4bd7-47ae-af6c-680e65b53b89")
-        ) {
-      System.out.println("stop");
-    }
-      
     return new ArdFilmInfoDto(id, url, numberOfClips);
   }
   
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/tasks/ArdTopicPageTask.java b/src/main/java/de/mediathekview/mserver/crawler/ard/tasks/ArdTopicPageTask.java
@@ -39,7 +39,7 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
         && topicInfo.getFilmInfos() != null
         && !topicInfo.getFilmInfos().isEmpty()) {
       taskResults.addAll(topicInfo.getFilmInfos());
-      LOG.debug("Found {} shows for a topic of ARD.", topicInfo.getFilmInfos().size());
+      //LOG.debug("Found {} shows for a topic of ARD.", topicInfo.getFilmInfos().size());
 
       final Queue<CrawlerUrlDTO> subpages = createSubPageUrls(aTarget, topicInfo);
       if (!subpages.isEmpty()) {
@@ -65,7 +65,7 @@ private Queue<CrawlerUrlDTO> createSubPageUrls(
         break;
       }
     }
-    LOG.debug("Found {} subpage", subpages.size());
+    //LOG.debug("Found {} subpage", subpages.size());
     return subpages;
   }
 
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/tasks/ArdTopicsLetterTask.java b/src/main/java/de/mediathekview/mserver/crawler/ard/tasks/ArdTopicsLetterTask.java
@@ -42,7 +42,7 @@ protected AbstractRecursiveConverterTask<CrawlerUrlDTO, CrawlerUrlDTO> createNew
   @Override
   protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarget) {
     final PaginationUrlDto results = deserialize(aTarget, PAGINATION_URL_DTO_TYPE_TOKEN, aDTO);
-    LOG.debug("Found {} shows for {}.", results.getUrls().size(), sender);
+    //LOG.debug("Found {} shows for {}.", results.getUrls().size(), sender);
     taskResults.addAll(results.getUrls());
 
     if (results.getActualPage() == 0 && results.getMaxPages() > 1) {
diff --git a/src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsLetterDeserializerTest.java b/src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsLetterDeserializerTest.java
@@ -19,15 +19,15 @@ public void testDeserialize() {
     final CrawlerUrlDTO[] expected =
         new CrawlerUrlDTO[] {
           new CrawlerUrlDTO(
-              "https://api.ardmediathek.de/page-gateway/widgets/ard/asset/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX2J1dGVudW5iaW5uZW4?pageSize=50"),
+              "https://api.ardmediathek.de/page-gateway/pages/ard/grouping/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL2J1dGVudW5iaW5uZW5nZWJhZXJkZW5zcHJhY2hl?embedded=true"),
           new CrawlerUrlDTO(
-              "https://api.ardmediathek.de/page-gateway/widgets/ard/asset/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX3Nwb3J0YmxpdHo?pageSize=50"),
+              "https://api.ardmediathek.de/page-gateway/pages/ard/grouping/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX3Nwb3J0YmxpdHo?embedded=true"),
           new CrawlerUrlDTO(
-              "https://api.ardmediathek.de/page-gateway/widgets/ard/asset/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX2J1dGVudW5iaW5uZW53ZXR0ZXI?pageSize=50"),
+              "https://api.ardmediathek.de/page-gateway/pages/ard/grouping/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX2J1dGVudW5iaW5uZW53ZXR0ZXI?embedded=true"),
           new CrawlerUrlDTO(
-              "https://api.ardmediathek.de/page-gateway/widgets/ard/asset/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX2J1dGVudW5iaW5uZW51bTY?pageSize=50"),
+              "https://api.ardmediathek.de/page-gateway/pages/ard/grouping/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX2J1dGVudW5iaW5uZW4?embedded=true"),
           new CrawlerUrlDTO(
-              "https://api.ardmediathek.de/page-gateway/widgets/ard/asset/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL2J1dGVudW5iaW5uZW5nZWJhZXJkZW5zcHJhY2hl?pageSize=50"),
+              "https://api.ardmediathek.de/page-gateway/pages/ard/grouping/Y3JpZDovL3JhZGlvYnJlbWVuLmRlL3NlbmRlcmVpaGVuL2lkX2J1dGVudW5iaW5uZW51bTY?embedded=true"),
         };
 
     final ArdTopicsLetterDeserializer instance = new ArdTopicsLetterDeserializer();

Original file line number	Diff line number	Diff line change
`@@ -88,6 +88,7 @@ public void updateLastUrlCheck(List<Film> checked) {`
`88`	`88`	`ps.addBatch();`
`89`	`89`	`}`
`90`	`90`	`int [] rs = ps.executeBatch();`
	`91`	`+ con.commit();`
`91`	`92`	`for (int rsCode : rs) {`
`92`	`93`	`updateCounter.addAndGet(rsCode);`
`93`	`94`	`}`
`@@ -124,6 +125,7 @@ public void deleteFilms(Collection<Film> abandonedFilmlist) {`
`124`	`125`	`ps.addBatch();`
`125`	`126`	`}`
`126`	`127`	`ps.executeBatch();`
	`128`	`+ con.commit();`
`127`	`129`	`} catch (SQLException e) {`
`128`	`130`	`LOG.error(e);`
`129`	`131`	`}`
`@@ -203,6 +205,7 @@ public <T> List<T> filterNewVideos(Sender sender, List<T> videos, Function<T, St`
`203`	`205`	`}`
`204`	`206`	`}`
`205`	`207`	`int[] rs = ps.executeBatch();`
	`208`	`+ con.commit();`
`206`	`209`	`for (int rsIndex = 0; rsIndex < rs.length; rsIndex++) {`
`207`	`210`	`if (rs[rsIndex] == 0) {`
`208`	`211`	`newVideos.add(batch.get(rsIndex));`
`@@ -220,6 +223,10 @@ public <T> List<T> filterNewVideos(Sender sender, List<T> videos, Function<T, St`
`220`	`223`	`result.addAll(f.get());`
`221`	`224`	`}`
`222`	`225`	`LOG.debug("Filtered {} in {} (in {} vs out {})",(videos.size()-result.size()), sender.getName(), videos.size(), result.size());`
	`226`	`+ // CARP films pro Nacht. Die filme werden dann am nächsten tag gefunden. TODO: für die nächste runde.`
	`227`	`+ /* result = new ArrayList<>(result.subList(0, 200000));`
	`228`	`+ *`
	`229`	`+ */`
`223`	`230`	`return result;`
`224`	`231`	`} catch (Exception e) {`
`225`	`232`	`LOG.error("{}", e);`
`@@ -319,8 +326,8 @@ ON CONFLICT (id) DO UPDATE`
`319`	`326`	`LOG.error("saveBatch - Missing ID for film {}", film);`
`320`	`327`	`}`
`321`	`328`	`}`
`322`		`-`
`323`	`329`	`ps.executeBatch();`
	`330`	`+ con.commit();`
`324`	`331`	`}`
`325`	`332`	`return successCounter;`
`326`	`333`	`}`
Original file line number	Diff line number	Diff line change
`@@ -123,7 +123,7 @@ private ForkJoinTask<Set<CrawlerUrlDTO>> getTopicEntriesBySender(final String se`
`123`	`123`	`Set<CrawlerUrlDTO> senderSingleLetterUrls = forkJoinPool.submit(`
`124`	`124`	`new ArdTopicsTask(this, sender, CreateLetterUrlQuery(sender))).get();`
`125`	`125`
`126`		`- LOG.debug("topics task result {}", senderSingleLetterUrls.size());`
	`126`	`+ //LOG.debug("topics task result {}", senderSingleLetterUrls.size());`
`127`	`127`	`return forkJoinPool.submit(new ArdTopicsLetterTask(this, sender, new ConcurrentLinkedQueue<>(senderSingleLetterUrls)));`
`128`	`128`	`}`
`129`	`129`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg`
`39`	`39`	`&& topicInfo.getFilmInfos() != null`
`40`	`40`	`&& !topicInfo.getFilmInfos().isEmpty()) {`
`41`	`41`	`taskResults.addAll(topicInfo.getFilmInfos());`
`42`		`- LOG.debug("Found {} shows for a topic of ARD.", topicInfo.getFilmInfos().size());`
	`42`	`+ //LOG.debug("Found {} shows for a topic of ARD.", topicInfo.getFilmInfos().size());`
`43`	`43`
`44`	`44`	`final Queue<CrawlerUrlDTO> subpages = createSubPageUrls(aTarget, topicInfo);`
`45`	`45`	`if (!subpages.isEmpty()) {`
`@@ -65,7 +65,7 @@ private Queue<CrawlerUrlDTO> createSubPageUrls(`
`65`	`65`	`break;`
`66`	`66`	`}`
`67`	`67`	`}`
`68`		`- LOG.debug("Found {} subpage", subpages.size());`
	`68`	`+ //LOG.debug("Found {} subpage", subpages.size());`
`69`	`69`	`return subpages;`
`70`	`70`	`}`
`71`	`71`