ZDF: Crawler läuft nicht durch

Author: alex1702

build.gradle

@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.261'
+version = '3.1.262'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/sender/MediathekCrawler.java

@@ -9,6 +9,7 @@
 import java.util.concurrent.RecursiveTask;
 import java.util.concurrent.TimeUnit;
 import mServer.crawler.FilmeSuchen;
+import mServer.crawler.RunSender;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -19,7 +20,7 @@ public abstract class MediathekCrawler extends MediathekReader {
 
   protected final ForkJoinPool forkJoinPool;
 
-  public MediathekCrawler(FilmeSuchen aMSearchFilmeSuchen, String aSendername, int aSenderMaxThread, int aSenderWartenSeiteLaden, int aStartPrio) {
+  protected MediathekCrawler(FilmeSuchen aMSearchFilmeSuchen, String aSendername, int aSenderMaxThread, int aSenderWartenSeiteLaden, int aStartPrio) {
     super(aMSearchFilmeSuchen, aSendername, aSenderMaxThread, aSenderWartenSeiteLaden, aStartPrio);
 
     forkJoinPool = createForkJoinPool(aSendername);
@@ -61,13 +62,20 @@ protected void runCrawler() {
 
     Log.sysLog(getSendername() + ": Filme einsortieren..." + films.size());
     if (films.isEmpty()) {
-      LOG.fatal(getSendername() + ": no films found!");
+      LOG.fatal("{}: no films found!", getSendername());
     }
 
     films.forEach(film -> {
       if (!Config.getStop()) {
-        prepareFilm(film);
-        addFilm(film);
+        try {
+          prepareFilm(film);
+          addFilm(film);
+        } catch (Exception e) {
+          final String index = film.getIndexAddOld();
+          Log.errorLog(974513456, e, index);
+          LOG.error("{}: Error while processing film: {}: {}", getSendername(), index, e);
+          FilmeSuchen.listeSenderLaufen.inc(film.arr[DatenFilm.FILM_SENDER], RunSender.Count.FEHLER);
+        }
       }
     });
   }

src/main/java/mServer/crawler/sender/srf/tasks/SrfTopicOverviewTask.java

@@ -59,11 +59,13 @@ private AbstractRecursivConverterTask<CrawlerUrlDTO, TopicUrlDTO> createNewOwnIn
   protected void processRestTarget(TopicUrlDTO aDTO, WebTarget aTarget) {
     PagedElementListDTO<CrawlerUrlDTO> results
             = deserialize(aTarget, PAGED_ELEMENT_LIST_URL_TYPE_TOKEN);
-    taskResults.addAll(results.getElements());
+    if (results != null) {
+      taskResults.addAll(results.getElements());
 
-    Optional<String> nextPageId = results.getNextPage();
-    if (nextPageId.isPresent() && pageNumber < maxSubPages) {
-      processNextPage(aDTO.getTopic(), nextPageId.get());
+      Optional<String> nextPageId = results.getNextPage();
+      if (nextPageId.isPresent() && pageNumber < maxSubPages) {
+        processNextPage(aDTO.getTopic(), nextPageId.get());
+      }
     }
   }