DB config via yaml, move atomic filmlist, backwards compatibility

codingPF · codingPF · commit 45c8d353d014 · 2026-01-09T17:11:56.000+01:00
diff --git a/src/main/java/de/mediathekview/mserver/base/config/MServerDBConfig.java b/src/main/java/de/mediathekview/mserver/base/config/MServerDBConfig.java
@@ -7,23 +7,57 @@ public class MServerDBConfig {
   private String url;
   private String username;
   private String password;
+  private Integer refreshIntervalInDays;
+  private Integer checkUrlIntervalInDays;
+  private Integer batchSize;
   
   public MServerDBConfig() {
-    active = true;
+    active = false;
     url = "jdbc:postgresql://postgresMV:55432/crawler";
     username = "crawler";
     password = "secret";
+    refreshIntervalInDays = 7;
+    checkUrlIntervalInDays = 3;
+    batchSize = 2000;
   }
   
-  public MServerDBConfig(Boolean active, String url, String username, String password) {
+  public MServerDBConfig(Boolean active, String url, String username, String password, int refreshIntervalInDays, int checkUrlIntervalInDays, int batchSize ) {
     this.active = active;
     this.url = url;
     this.username = username;
     this.password = password;
+    this.refreshIntervalInDays = refreshIntervalInDays;
+    this.checkUrlIntervalInDays = checkUrlIntervalInDays;
+    this.batchSize = batchSize;
   }
 
 
 
+  public Integer getBatchSize() {
+    return batchSize;
+  }
+
+  public void setBatchSize(Integer batchSize) {
+    this.batchSize = batchSize;
+  }
+
+  public Integer getRefreshIntervalInDays() {
+    return refreshIntervalInDays;
+  }
+
+  public void setRefreshIntervalInDays(Integer refreshIntervalInDays) {
+    this.refreshIntervalInDays = refreshIntervalInDays;
+  }
+
+  public Integer getCheckUrlIntervalInDays() {
+    return checkUrlIntervalInDays;
+  }
+
+  public void setCheckUrlIntervalInDays(Integer checkUrlIntervalInDays) {
+    this.checkUrlIntervalInDays = checkUrlIntervalInDays;
+  }
+
+
   public Boolean getActive() {
     return active;
   }
diff --git a/src/main/java/de/mediathekview/mserver/base/uploader/copy/FileCopyTask.java b/src/main/java/de/mediathekview/mserver/base/uploader/copy/FileCopyTask.java
@@ -32,7 +32,9 @@ protected void upload() {
         Path backup = backupExistingFile(target);
         LOG.debug("CopyTask found existing file - rename existing file to {} before overwrite", backup.getFileName());
       }
-      Files.copy(sourcePath, uploadTarget.getTargetPath(), StandardCopyOption.REPLACE_EXISTING);
+      Path tmpTarget = Files.createTempFile( uploadTarget.getTargetPath().getParent(), uploadTarget.getTargetPath().getFileName().toString(), ".tmp");
+      Files.copy(sourcePath, tmpTarget, StandardCopyOption.REPLACE_EXISTING);
+      Files.move(tmpTarget, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
     } catch (final IOException ioException) {
       LOG.error("Something went wrong on copying the film list.", ioException);
       printMessage(ServerMessages.FILE_COPY_ERROR);
diff --git a/src/main/java/de/mediathekview/mserver/base/utils/FilmDBService.java b/src/main/java/de/mediathekview/mserver/base/utils/FilmDBService.java
@@ -45,11 +45,13 @@ public class FilmDBService {
   private final Gson gson;
   private final ExecutorService executorService;
   private final int batchSize;
+  private final Integer refreshIntervalInDays;
 
-  public FilmDBService(ExecutorService executorService, int batchSize) {
+  public FilmDBService(ExecutorService executorService, int batchSize, int refreshIntervalInDays) {
     this.dataSource = PostgreSQLDataSourceProvider.get();
     this.executorService = executorService;
     this.batchSize = batchSize;
+    this.refreshIntervalInDays = refreshIntervalInDays;
 
     this.gson = new GsonBuilder().registerTypeAdapter(LocalDateTime.class, new GsonLocalDateTimeAdapter())
         .registerTypeAdapter(Duration.class, new GsonDurationAdapter()).create();
@@ -153,10 +155,14 @@ public <T> List<T> filterNewVideos(Sender sender, List<T> videos, Function<T, St
         List<T> batch = allVideos.subList(from, to);
         futures.add(executorService.submit(() -> {
           List<T> newVideos = new ArrayList<>();
-          // update every 7 days
-          String sql = "UPDATE filme SET last_seen = now() WHERE id = ? AND last_seen - last_update <= interval '7' DAY";
-
-          try (Connection con = dataSource.getConnection(); PreparedStatement ps = con.prepareStatement(sql)) {
+          StringBuffer sql = new StringBuffer();
+          sql.append("UPDATE filme SET last_seen = now() ")
+          .append("WHERE id = ? AND (")
+          .append("( cast(created_at as date) = cast(last_update as date) and cast(created_at as date) <> cast(now() as date) )")
+          .append(" OR ")
+          .append("(last_seen - last_update <= interval '").append(refreshIntervalInDays).append("' DAY)")
+          .append(")");
+          try (Connection con = dataSource.getConnection(); PreparedStatement ps = con.prepareStatement(sql.toString())) {
 
             for (T video : batch) {
               String id = idExtractor.apply(video);
diff --git a/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java b/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java
@@ -95,7 +95,7 @@ public MServerConfigManager getConfigManager() {
   }
   
   public void storeFilmsToDB() {
-    FilmDBService filmDBService = new FilmDBService(executorService, 200);
+    FilmDBService filmDBService = new FilmDBService(executorService, getConfigManager().getConfig().getDatabaseConfig().getBatchSize(), getConfigManager().getConfig().getDatabaseConfig().getRefreshIntervalInDays());
     try { 
       filmDBService.saveAll(filmlist);
     } catch (Exception e) {
@@ -459,7 +459,7 @@ private Set<AbstractCrawler> getCrawlerToRun() {
   }
 
   private Optional<Filmlist> importFilmlistFromDB() throws IOException {
-    FilmDBService filmDBService = new FilmDBService(getExecutorService(), 2000);
+    FilmDBService filmDBService = new FilmDBService(getExecutorService(), getConfigManager().getConfig().getDatabaseConfig().getBatchSize(), getConfigManager().getConfig().getDatabaseConfig().getRefreshIntervalInDays());
     Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB();
     return dbFilmlist;
   }
diff --git a/src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java b/src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java
@@ -13,7 +13,6 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
-import java.time.format.DateTimeFormatter;
 import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
@@ -24,8 +23,6 @@
 public class ArdCrawler extends AbstractCrawler {
 
   private static final Logger LOG = LogManager.getLogger(ArdCrawler.class);
-  private static final DateTimeFormatter DAY_PAGE_DATE_FORMATTER =
-      DateTimeFormatter.ofPattern("yyyy-MM-dd");
 
   public ArdCrawler(
       final ForkJoinPool aForkJoinPool,
diff --git a/src/main/java/de/mediathekview/mserver/crawler/basic/AbstractCrawler.java b/src/main/java/de/mediathekview/mserver/crawler/basic/AbstractCrawler.java
@@ -71,7 +71,7 @@ protected AbstractCrawler(
         rootConfig.getSenderConfig(getSender()).getSocketTimeoutInSeconds(), 
         runtimeConfig.getMaximumCpuThreads());
     rateLimiter = RateLimiter.create(rootConfig.getSenderConfig(getSender()).getMaximumRequestsPerSecond());
-    filmDBService = new FilmDBService(forkJoinPool, 200);
+    filmDBService = new FilmDBService(forkJoinPool, getRuntimeConfig().getDatabaseConfig().getBatchSize(), getRuntimeConfig().getDatabaseConfig().getRefreshIntervalInDays());
     films = ConcurrentHashMap.newKeySet();
   }
 
diff --git a/src/main/java/de/mediathekview/mserver/crawler/dw/tasks/DwFilmDetailTask.java b/src/main/java/de/mediathekview/mserver/crawler/dw/tasks/DwFilmDetailTask.java
@@ -4,7 +4,6 @@
 import de.mediathekview.mserver.daten.Film;
 import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
 import de.mediathekview.mserver.crawler.basic.AbstractRecursiveConverterTask;
-import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
 import de.mediathekview.mserver.crawler.basic.TopicUrlDTO;
 import de.mediathekview.mserver.crawler.dw.DWTaskBase;
 import de.mediathekview.mserver.crawler.dw.parser.DwFilmDetailDeserializer;
@@ -46,7 +45,8 @@ protected void processRestTarget(final TopicUrlDTO aDTO, final WebTarget aTarget
     } catch (Exception e) {
       LOG.error("error processing {} ", aDTO.getUrl(), e);
     }
-    if (filmDetailDtoOptional.isEmpty()) {
+    // Optional can be null if response code is 200 and response body is empty
+    if (filmDetailDtoOptional == null || filmDetailDtoOptional.isEmpty()) {
       crawler.incrementAndGetErrorCount();
       crawler.updateProgress();
       return;
diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfFilmTask.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfFilmTask.java
@@ -108,7 +108,7 @@ private static void updateTitle(final String aLanguage, final Film aFilm) {
   @Override
   protected void processRestTarget(ZdfFilmDto aDTO, WebTarget aTarget) {
     final Optional<DownloadDto> downloadDto =
-        deserialize(aTarget, OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN);
+        deserializeOptional(aTarget, OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN);
     if (downloadDto.isPresent()) {
       try {
         addFilm(downloadDto.get(), createFilm(aDTO, downloadDto.get()), aDTO.getVideoType());
diff --git a/src/main/java/de/mediathekview/mserver/filmlisten/writer/AbstractFilmlistWriter.java b/src/main/java/de/mediathekview/mserver/filmlisten/writer/AbstractFilmlistWriter.java
@@ -8,7 +8,10 @@
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.OutputStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -26,9 +29,17 @@ protected AbstractFilmlistWriter(final MessageListener... aListeners) {
   public abstract boolean write(Filmlist filmlist, OutputStream outputStream) throws IOException;
 
   public boolean write(Filmlist filmlist, Path savePath) {
-    try (final OutputStream os = new FileOutputStream(savePath.toFile());
-         final BufferedOutputStream fos = new BufferedOutputStream(os, 512000)) {
-      return write(filmlist, fos);
+    try {
+      Path tmpTarget = Files.createTempFile( savePath.getParent(), savePath.getFileName().toString(), ".tmp");
+          try (final OutputStream os = new FileOutputStream(tmpTarget.toFile());
+              final BufferedOutputStream fos = new BufferedOutputStream(os, 512000)) {
+           boolean succuess =  write(filmlist, fos);
+           fos.close();
+           if (succuess) {
+             Files.move(tmpTarget, savePath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
+           }
+           return succuess;
+         }
     } catch (final IOException ioException) {
       LOG.debug("Something went wrong on writing the film list.", ioException);
       publishMessage(FilmListMessages.FILMLIST_WRITE_ERROR, savePath.toAbsolutePath().toString());
diff --git a/src/main/java/de/mediathekview/mserver/filmlisten/writer/FilmlistOldFormatWriter.java b/src/main/java/de/mediathekview/mserver/filmlisten/writer/FilmlistOldFormatWriter.java
@@ -16,6 +16,7 @@
 import java.io.OutputStreamWriter;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.time.LocalDateTime;
 import java.time.LocalTime;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
@@ -41,6 +42,7 @@ public class FilmlistOldFormatWriter extends AbstractFilmlistWriter {
   private static final char GEO_SPLITTERATOR = '-';
   private static final DateTimeFormatter DATE_TIME_FORMAT =
       DateTimeFormatter.ofLocalizedDateTime(MEDIUM, SHORT).withLocale(Locale.GERMANY);
+  private static final LocalDateTime EMPTY_DATE_TIME= LocalDateTime.of(1970, 1, 1, 00, 00, 00);
 
   private static final String META_HEADER_VERSION = "4";
   private static final String META_HEADER_VERSION_LONG = "MSearch [Vers.: 4.0.1]";
@@ -196,10 +198,16 @@ private String writeRecord03Titel(AbstractMediaResource<?> in) {
   }
 
   private String writeRecord04Datum(AbstractMediaResource<?> in) {
+    if(in.getTime().isEqual(EMPTY_DATE_TIME)) {
+      return "";
+    }
     return in.getTime().format(DATE_FORMATTER);
   }
 
   private String writeRecord05Zeit(AbstractMediaResource<?> in) {
+    if(in.getTime().isEqual(EMPTY_DATE_TIME)) {
+      return "";
+    }
     return in.getTime().format(TIME_FORMATTER);
   }
       
@@ -211,7 +219,7 @@ private String writeRecord06Dauer(AbstractMediaResource<?> in) {
   }
 
   private String writeRecord07Groesse(AbstractMediaResource<?> in) {
-    if ((in instanceof Podcast pIn) && pIn.getUrl(Resolution.NORMAL) != null)
+    if ((in instanceof Podcast pIn) && pIn.getUrl(Resolution.NORMAL) != null && pIn.getUrl(Resolution.NORMAL).getFileSize() != 0)
       return (pIn.getUrl(Resolution.NORMAL).getFileSize()/1024) + "";
     return "";
   }
diff --git a/src/main/java/de/mediathekview/mserver/ui/config/MServerExecutionFlow.java b/src/main/java/de/mediathekview/mserver/ui/config/MServerExecutionFlow.java
@@ -11,6 +11,7 @@
 import de.mediathekview.mserver.base.utils.FilmDBService;
 import de.mediathekview.mserver.crawler.CrawlerManager;
 import de.mediathekview.mserver.daten.Filmlist;
+import de.mediathekview.mserver.daten.Resolution;
 import de.mediathekview.mserver.ui.config.MServerCommandLine.CMDARG;
 
 public class MServerExecutionFlow {
@@ -63,7 +64,7 @@ void startCrawlerFlow() {
   
   void exportFilmListFromDB() {
     try {
-      FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
+      FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000, -1);
       Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB();
       dbFilmlist.ifPresent(filmlist -> manager.getFilmlist().addAllFilms(filmlist.getFilms().values()));
       //
@@ -81,12 +82,15 @@ void exportFilmListFromDB() {
   }
   void importFilmlistIntoDB() {
     manager.importFilmlist();
-    FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
+    FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000, -1);
     HashSet<String> allVideoUrls = filmDBService.getAllVideoUrls();
     LOG.debug("allVideoUrls loaded {} entries", allVideoUrls.size());
     manager.getFilmlist().getFilms().entrySet().parallelStream()
       .forEach(entry -> {
-          if (allVideoUrls.contains(entry.getValue().getSender().name()+entry.getValue().getDefaultUrl().get().getUrl().toString())) {
+          if (allVideoUrls.contains(entry.getValue().getSender().name()+entry.getValue().getDefaultUrl().get().getUrl().toString()) || 
+              (entry.getValue().getUrl(Resolution.SMALL) != null && allVideoUrls.contains(entry.getValue().getSender().name()+entry.getValue().getUrl(Resolution.SMALL).getUrl().toString())) ||
+              (entry.getValue().getUrl(Resolution.HD) != null && allVideoUrls.contains(entry.getValue().getSender().name()+entry.getValue().getUrl(Resolution.HD).getUrl().toString()))
+          ) {
             manager.getFilmlist().getFilms().remove(entry.getKey());
           }
       });
@@ -104,7 +108,10 @@ void importFilmlistIntoDB() {
     manager.stop();
   }
   void checkAvailability() {
-    FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
+    FilmDBService filmDBService = new FilmDBService(
+        manager.getExecutorService(), 
+        manager.getConfigManager().getConfig().getDatabaseConfig().getBatchSize(),
+        manager.getConfigManager().getConfig().getDatabaseConfig().getRefreshIntervalInDays());
     String condition = "WHERE last_url_check < NOW() - INTERVAL '3' DAY";
     String limit = " LIMIT 400000";
     Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB(condition, limit);
diff --git a/src/main/resources/MServer-Config.yaml b/src/main/resources/MServer-Config.yaml
@@ -193,6 +193,9 @@ databaseConfig:
   url: "jdbc:postgresql://postgresMV:5432/crawler"
   username: "crawler"
   password: "secret"
+  refreshIntervalInDays: 7
+  checkUrlIntervalInDays: 3
+  batchSize: 2000
   
 #### Logging ####
 logSettings:

Original file line number	Diff line number	Diff line change
`@@ -95,7 +95,7 @@ public MServerConfigManager getConfigManager() {`
`95`	`95`	`}`
`96`	`96`
`97`	`97`	`public void storeFilmsToDB() {`
`98`		`- FilmDBService filmDBService = new FilmDBService(executorService, 200);`
	`98`	`+ FilmDBService filmDBService = new FilmDBService(executorService, getConfigManager().getConfig().getDatabaseConfig().getBatchSize(), getConfigManager().getConfig().getDatabaseConfig().getRefreshIntervalInDays());`
`99`	`99`	`try {`
`100`	`100`	`filmDBService.saveAll(filmlist);`
`101`	`101`	`} catch (Exception e) {`
`@@ -459,7 +459,7 @@ private Set<AbstractCrawler> getCrawlerToRun() {`
`459`	`459`	`}`
`460`	`460`
`461`	`461`	`private Optional<Filmlist> importFilmlistFromDB() throws IOException {`
`462`		`- FilmDBService filmDBService = new FilmDBService(getExecutorService(), 2000);`
	`462`	`+ FilmDBService filmDBService = new FilmDBService(getExecutorService(), getConfigManager().getConfig().getDatabaseConfig().getBatchSize(), getConfigManager().getConfig().getDatabaseConfig().getRefreshIntervalInDays());`
`463`	`463`	`Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB();`
`464`	`464`	`return dbFilmlist;`
`465`	`465`	`}`
Original file line number	Diff line number	Diff line change
`@@ -71,7 +71,7 @@ protected AbstractCrawler(`
`71`	`71`	`rootConfig.getSenderConfig(getSender()).getSocketTimeoutInSeconds(),`
`72`	`72`	`runtimeConfig.getMaximumCpuThreads());`
`73`	`73`	`rateLimiter = RateLimiter.create(rootConfig.getSenderConfig(getSender()).getMaximumRequestsPerSecond());`
`74`		`- filmDBService = new FilmDBService(forkJoinPool, 200);`
	`74`	`+ filmDBService = new FilmDBService(forkJoinPool, getRuntimeConfig().getDatabaseConfig().getBatchSize(), getRuntimeConfig().getDatabaseConfig().getRefreshIntervalInDays());`
`75`	`75`	`films = ConcurrentHashMap.newKeySet();`
`76`	`76`	`}`
`77`	`77`