update maxSubpages and search by broadcastdate #1109

Author: alex1702

build.gradle

@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
 targetCompatibility = JavaVersion.VERSION_17
 group = 'de.mediathekview'
 archivesBaseName = "MServer"
-version = '3.1.274'
+version = '3.1.275'
 
 def jarName = 'MServer.jar'
 def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/sender/arte/ArteConstants.java

@@ -1,10 +1,11 @@
 package mServer.crawler.sender.arte;
 
 public class ArteConstants {
-  public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-creationDate&language=%s";
-  public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=creationDate&language=%s";
+  public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-broadcastBegin&language=%s";
+  public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=broadcastBegin&language=%s";
   public static final String VIDEO_URL ="https://www.arte.tv/hbbtvv2/services/web/index.php/OPA/v3/streams/%s/%s/%s"; //PROGRAMID/KIND/LANG
   public static final String API_TOKEN = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA";
+  public static final int MAX_POSSIBLE_SUBPAGES = 100;
   private ArteConstants() {}
 
 }

src/main/java/mServer/crawler/sender/arte/ArteCrawler.java

@@ -50,7 +50,7 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
 
       final ArteVideoInfoTask aArteRestVideoInfoTask;
       // DO NOT overload - maximumUrlsPerTask used to reduce threads to 4
-      aArteRestVideoInfoTask = new ArteVideoInfoTask(this, videoUrls);
+      aArteRestVideoInfoTask = new ArteVideoInfoTask(this, videoUrls, getMaxPagesForOverview(getLanguage().toString().toLowerCase()));
       final ConcurrentLinkedQueue<ArteVideoInfoDto> videos = new ConcurrentLinkedQueue<>();
       videos.addAll(aArteRestVideoInfoTask.fork().join());
       //
@@ -71,29 +71,25 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
   }
 
   private ConcurrentLinkedQueue<TopicUrlDTO> createVideosQueue(String language) {
-    int maxPages = getMaxPagesForOverview(language);
     final ConcurrentLinkedQueue<TopicUrlDTO> root = new ConcurrentLinkedQueue<>();
     String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, language);
-    root.add(new TopicUrlDTO("all videos1", rootUrl));
-    if (maxPages >= 100) {
-      String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language);
-      root.add(new TopicUrlDTO("all videos2", rootUrl2));
-    }
+    root.add(new TopicUrlDTO("all videos sorted up", rootUrl));
+    String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language);
+    root.add(new TopicUrlDTO("all videos sorted down", rootUrl2));
     return root;
   }
 
   private int getMaxPagesForOverview(String lang) {
     final int maxAvailablePages = getNumberOfAvailablePages(lang);
     final int configuredMaxPages = getMaximumSubpages();
     if (configuredMaxPages > maxAvailablePages) {
-      return Math.min(configuredMaxPages, maxAvailablePages / 2);
+      return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, maxAvailablePages / 2);
     } else {
-      return Math.min(configuredMaxPages, configuredMaxPages / 2);
+      return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, configuredMaxPages / 2);
     }
   }
 
   private int getNumberOfAvailablePages(String lang) {
-    final int naturalLimit = Math.min(100, getMaximumSubpages());
     try {
       String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, lang);
       String[] path= {"meta", "videos", "pages"};
@@ -110,14 +106,14 @@ private int getNumberOfAvailablePages(String lang) {
     } catch (IOException e) {
       LOG.error("getMaxPagesForOverview", e);
     }
-    return naturalLimit;
+    return ArteConstants.MAX_POSSIBLE_SUBPAGES;
   }
 
   private int getMaximumSubpages() {
     if (CrawlerTool.loadLongMax()) {
-      return 10;
+      return 30;
     } else {
-      return 1;
+      return 10;
     }
   }
 }

src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java

@@ -29,10 +29,11 @@ public class ArteVideoInfoTask
   private static final long serialVersionUID = 1L;
   protected final transient Logger log = LogManager.getLogger(this.getClass());
   protected transient Optional<AbstractRecursivConverterTask<ArteVideoInfoDto, TopicUrlDTO>> nextPageTask = Optional.empty();
-
+  protected int maxSubpages;
 
-  public ArteVideoInfoTask(MediathekReader crawler, ConcurrentLinkedQueue<TopicUrlDTO> urlToCrawlDTOs) {
+  public ArteVideoInfoTask(MediathekReader crawler, ConcurrentLinkedQueue<TopicUrlDTO> urlToCrawlDTOs, int maxSubpages) {
     super(crawler, urlToCrawlDTOs, Optional.of(ArteConstants.API_TOKEN));
+    this.maxSubpages = maxSubpages;
   }
 
   @Override
@@ -49,9 +50,8 @@ protected void postProcessingNextPage(PagedElementListDTO<ArteVideoInfoDto> aRes
     if (aResponseObj.getNextPage().isEmpty()) {
       return;
     }
-    int maxPages = Math.min(100, getMaximumSubpages());
-    if (aResponseObj.getNextPage().get().contains("age="+maxPages)) {
-      log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxPages);
+    if (aResponseObj.getNextPage().get().contains("age="+maxSubpages)) {
+      log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxSubpages);
       return;
     }
 
@@ -61,14 +61,6 @@ protected void postProcessingNextPage(PagedElementListDTO<ArteVideoInfoDto> aRes
     nextPageTask.get().fork();
   }
 
-  private int getMaximumSubpages() {
-    if (CrawlerTool.loadLongMax()) {
-      return 10;
-    } else {
-      return 3;
-    }
-  }
-
   protected void postProcessingElements(Set<ArteVideoInfoDto> elements) {
     for (ArteVideoInfoDto element : elements)  {
       taskResults.add(element);
@@ -86,7 +78,7 @@ protected void postProcessing(PagedElementListDTO<ArteVideoInfoDto> aResponseObj
   @Override
   protected AbstractRecursivConverterTask<ArteVideoInfoDto, TopicUrlDTO> createNewOwnInstance(
       ConcurrentLinkedQueue<TopicUrlDTO> aElementsToProcess) {
-    return new ArteVideoInfoTask(crawler, aElementsToProcess);
+    return new ArteVideoInfoTask(crawler, aElementsToProcess, maxSubpages);
   }
 
   @Override