update

Author: codingPF

MServer-Config.yaml

@@ -1,5 +1,12 @@
 #### Server configurations ####
 
+# Film DB
+databaseConfig:
+  active: false
+  url: "jdbc:postgresql://localhost:55432/crawler"
+  username: "crawler"
+  password: "secret"
+
 # The maximum amount of cpu threads to be used.
 maximumCpuThreads: 10
 
@@ -18,20 +25,19 @@ maximumRequestsPerSecond: 999.0
 # If set only these Sender will be crawled all other will be ignored.
 senderIncluded:
   #- ARD
-  - ARTE_DE
-  - ARTE_FR
-  - ARTE_PL
-  - ARTE_IT
-  - ARTE_ES
-  - ARTE_EN
+  #- ARTE_DE
+  #- ARTE_FR
+  #- ARTE_PL
+  #- ARTE_IT
+  #- ARTE_ES
+  #- ARTE_EN
   #- DREISAT
-  #- FUNK
   #- KIKA
-  # - DW
+  #- DW
   #- ORF
   #- PHOENIX
   #- SRF
-  #- SR
+  - SR
   #- ZDF
 
 #SRF,SR,PHONIX,ORF,KIKA,DW,3SAT<
@@ -113,7 +119,17 @@ importFilmlistConfigurations :
       path: "https://verteiler1.mediathekview.de/filme-org.xz"
       format: OLD_JSON_COMPRESSED_XZ
       createDiff: true
-      checkImportListUrl: true    
+      checkImportListUrl: true
+    - active: false
+      path: "https://verteiler1.mediathekview.de/Filmliste-akt.xz"
+      format: OLD_JSON_COMPRESSED_XZ
+      createDiff: false
+      checkImportListUrl: false
+    - active: true
+      path: jdbc
+      format: OLD_JSON
+      createDiff: false
+      checkImportListUrl: false
 
 # film url is consider invalid if the size is below the minSize
 checkImportListUrlMinSize: 5012
@@ -171,7 +187,7 @@ senderConfigurations:
   ARTE_ES:
     maximumSubpages: 6
   KIKA:
-    maximumSubpages: 2
+    maximumSubpages: 4
     maximumRequestsPerSecond: 8.0
   ZDF:
     maximumDaysForSendungVerpasstSection: 21
@@ -206,6 +222,8 @@ copySettings:
     # JSON_COMPRESSED: /var/www/mediathekview/filmlisten/filmliste_diff.json.xz
     OLD_JSON_COMPRESSED_XZ: copyTarget/filmliste_old_diff.json.xz
 
+
+
 #### Logging ####
 logSettings:
   # The log level for the console.

pom.xml

@@ -93,7 +93,7 @@
     <wiremock.version>2.35.2</wiremock.version>
     <xz.version>1.10</xz.version>
     <yacl4j-core.version>0.9.2</yacl4j-core.version>
-    <docker-maven-plugin.version>0.40.2</docker-maven-plugin.version>
+    <docker-maven-plugin.version>0.48.0</docker-maven-plugin.version>
     <sortpom-maven-plugin.version>3.2.0</sortpom-maven-plugin.version>
   </properties>
 
@@ -500,24 +500,22 @@
       <build>
         <plugins>
           <!-- Build docker -->
-
           <plugin>
             <groupId>io.fabric8</groupId>
             <artifactId>docker-maven-plugin</artifactId>
             <version>${docker-maven-plugin.version}</version>
             <configuration>
-              <images>
+            <images>
                 <image>
                   <name>mediathekview/mserver:${project.version}</name>
-
                   <build>
                     <from>eclipse-temurin:${maven.compiler.target}</from>
                     <maintainer>Nicklas Wiegandt &lt;nicklas@wiegandt.eu&gt;</maintainer>
                     <assembly>
                       <descriptor>docker-assembly.xml</descriptor>
                     </assembly>
                     <env>
-                      <JAVA_OPTS>-Xmx4G</JAVA_OPTS>
+                      <JAVA_OPTS>-Xmx8G</JAVA_OPTS>
                       <MSERVER_OPTS>config.yaml</MSERVER_OPTS>
                     </env>
                     <entryPoint>

src/main/docker/docker-compose.yml

@@ -0,0 +1,61 @@
+
+configs:
+  init_sql:
+    content: |
+      CREATE TABLE IF NOT EXISTS filme (
+          id TEXT PRIMARY KEY,          -- eindeutige Film-ID
+          data JSONB NOT NULL,          -- JSON-Daten des Films
+          created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+          last_update TIMESTAMPTZ NOT NULL DEFAULT now(),
+          last_url_check TIMESTAMPTZ NOT NULL DEFAULT now()
+      );
+      
+services:
+
+  postgresMV:
+    image: postgres:18
+    container_name: crawler-postgres
+    environment:
+      POSTGRES_DB: crawler
+      POSTGRES_USER: crawler
+      POSTGRES_PASSWORD: secret
+      TZ: Europe/Berlin
+    ports:
+      - "55432:5432"
+    volumes:
+      # Persistente Daten
+      - c:/tmp/pgdata:/var/lib/postgresql
+    configs:
+      - source: init_sql
+        target: /docker-entrypoint-initdb.d/init.sql
+    restart: unless-stopped
+
+  mserver-r1:
+    image: mediathekview/mserver:4.0.1-SNAPSHOT
+    environment:
+      MSERVER_OPTS: --config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R1.yaml
+    volumes:
+      - c:/tmp/filmlists:/filmlists
+      - c:/tmp/logs:/logs
+      - c:/tmp/hist:/hist
+      - c:/tmp/config:/config
+      
+  mserver-r2:
+    image: mediathekview/mserver:4.0.1-SNAPSHOT
+    environment:
+      MSERVER_OPTS: --config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R2.yaml
+    volumes:
+      - c:/tmp/filmlists:/filmlists
+      - c:/tmp/logs:/logs
+      - c:/tmp/hist:/hist
+      - c:/tmp/config:/config
+
+  mserver-checkUrls:
+    image: mediathekview/mserver:4.0.1-SNAPSHOT
+    environment:
+      MSERVER_OPTS: --config https://mediathekview.github.io/MVCrawlerConfig/config/MServer-Config-R2.yaml --flow checkAvailability
+    volumes:
+      - /volume1/docker/Projekt-mv2/filmlists:/filmlists
+      - /volume1/docker/Projekt-mv2/logs:/logs
+      - /volume1/docker/Projekt-mv2/hist:/hist
+      - /volume1/docker/Projekt-mv2/config:/config

src/main/docker/runDocker

@@ -0,0 +1,18 @@
+# all docker command
+
+# run PG
+docker compose up -d postgresMV
+
+# cron 01 01 long run
+docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R1.yaml" mserver-r1
+
+# cron 06-22 1,31 short run
+docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R2.yaml" mserver-r2
+
+# cron 22 55 url check
+docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R2.yaml --flow checkAvailability" mserver-checkUrls
+
+# on demand - do not run this unless you know what you are doing!
+docker compose run -d --rm -e MSERVER_OPTS="--config /config/MServer-Config-R2.yaml --flow importFilmlistIntoDB" mserver-r3
+
+

src/main/java/de/mediathekview/mserver/base/config/ConfigManager.java

@@ -11,7 +11,7 @@
 /** A manager to load configurations. */
 public abstract class ConfigManager<T extends ConfigDTO> {
   private T config;
-  private static final Logger LOG = LogManager.getLogger(ConfigManager.class);
+  //private static final Logger LOG = LogManager.getLogger(ConfigManager.class);
 
   protected abstract String getConfigFileName();
 
@@ -58,7 +58,8 @@ public String getResourcePath(String resourceName) {
         }
       }
     } catch(Exception e) {
-      LOG.debug(e);
+      //LOG.debug(e);
+      e.printStackTrace();
     }
     return null;
   }

src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java

@@ -8,12 +8,13 @@
 
 /** A POJO with the configs for MServer. */
 public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO {
+  private MServerDBConfig databaseConfig;
   private final MServerCopySettings copySettings;
   private final Boolean writeFilmlistHashFileEnabled;
   private final String filmlistHashFilePath;
   private final Boolean writeFilmlistIdFileEnabled;
   private final String filmlistIdFilePath;
-  private final MServerDBConfig mServerDBConfig;
+  
   /** ignore certain film by title **/
   private String ignoreFilmlistPath;
   /** add livestreams from external list **/
@@ -49,7 +50,7 @@ public MServerConfigDTO() {
     filmlistSavePaths = new EnumMap<>(FilmlistFormats.class);
     filmlistDiffSavePaths = new EnumMap<>(FilmlistFormats.class);
     copySettings = new MServerCopySettings();
-    mServerDBConfig = new MServerDBConfig();
+    databaseConfig = new MServerDBConfig();
     logSettings = new MServerLogSettingsDTO();
     crawlerURLs = new EnumMap<>(CrawlerUrlType.class);
 
@@ -245,8 +246,12 @@ public List<ImportFilmlistConfiguration> getImportFilmlistConfigurations() {
     return importFilmlistConfigurations;
   }
 
-  public MServerDBConfig getMServerDBConfig() {
-    return mServerDBConfig;
+  public MServerDBConfig getDatabaseConfig() {
+    return databaseConfig;
+  }
+
+  public void setDatabaseConfig(MServerDBConfig databaseConfig) {
+    this.databaseConfig = databaseConfig;
   }
 
   /**
@@ -293,7 +298,7 @@ public boolean equals(final Object o) {
         && Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath())
         && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration())
         && Objects.equals(getImportFilmlistConfigurations(), that.getImportFilmlistConfigurations())
-        && Objects.equals(getMServerDBConfig(), that.getMServerDBConfig());
+        && Objects.equals(getDatabaseConfig(), that.getDatabaseConfig());
   }
 
   @Override
@@ -321,7 +326,7 @@ public int hashCode() {
         getIgnoreFilmslistPath(),
         getImportLivestreamConfiguration(),
         getImportFilmlistConfigurations(),
-        getMServerDBConfig());
+        getDatabaseConfig());
   }
 
   public void initializeSenderConfigurations() {

src/main/java/de/mediathekview/mserver/base/config/MServerDBConfig.java

@@ -3,21 +3,19 @@
 import java.util.Objects;
 
 public class MServerDBConfig {
-  private final Boolean active;
-  private final String url;
-  private final String username;
-  private final String password;
+  private boolean active;
+  private String url;
+  private String username;
+  private String password;
 
   public MServerDBConfig() {
-    super();
     active = true;
-    url = "jdbc:postgresql://OscarDS:55432/crawler";
+    url = "jdbc:postgresql://postgresMV:55432/crawler";
     username = "crawler";
     password = "secret";
   }
 
   public MServerDBConfig(Boolean active, String url, String username, String password) {
-    super();
     this.active = active;
     this.url = url;
     this.username = username;
@@ -38,6 +36,23 @@ public String getUsername() {
   public String getPassword() {
     return password;
   }
+  
+
+  public void setActive(Boolean active) {
+    this.active = active;
+  }
+
+  public void setUrl(String url) {
+    this.url = url;
+  }
+
+  public void setUsername(String username) {
+    this.username = username;
+  }
+
+  public void setPassword(String password) {
+    this.password = password;
+  }
 
   @Override
   public boolean equals(Object o) {

src/main/java/de/mediathekview/mserver/base/uploader/copy/FileCopyTask.java

@@ -10,6 +10,8 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
 
 public class FileCopyTask extends UploadTask<FileCopyTarget> {
   private static final Logger LOG = LogManager.getLogger(FileCopyTask.class);
@@ -25,15 +27,36 @@ protected void printMessage(final Message aMessage, final Object... args) {
   @Override
   protected void upload() {
     try {
-      if (Files.exists(uploadTarget.getTargetPath())) {
-        printMessage(
-            ServerMessages.FILE_COPY_TARGET_EXISTS,
-            uploadTarget.getTargetPath().toAbsolutePath().toString());
+      Path target = uploadTarget.getTargetPath();
+      if (Files.exists(target)) {
+        Path backup = backupExistingFile(target);
+        LOG.debug("CopyTask found existing file - rename existing file to {} before overwrite", backup.getFileName());
       }
       Files.copy(sourcePath, uploadTarget.getTargetPath(), StandardCopyOption.REPLACE_EXISTING);
     } catch (final IOException ioException) {
       LOG.error("Something went wrong on copying the film list.", ioException);
       printMessage(ServerMessages.FILE_COPY_ERROR);
     }
   }
+  
+  private Path backupExistingFile(Path target) throws IOException {
+    String fileName = target.getFileName().toString();
+    Path dir = target.getParent();
+
+    String date = LocalDate.now()
+        .format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
+
+    Path backup = dir.resolve(fileName + "." + date);
+
+    int counter = 1;
+    while (Files.exists(backup)) {
+      backup = dir.resolve(fileName + "." + date + "." + counter);
+      counter++;
+    }
+
+    Files.move(target, backup);
+    
+    return backup;
+  }
+
 }

src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java

@@ -59,9 +59,6 @@ private boolean isAvailable(Film pFilm) {
       timeout.set(true);
       return true;
     }
-    if(pFilm.getDefaultUrl().isEmpty()) {
-      System.out.println("asdf");
-    }
     String normalUrl = pFilm.getDefaultUrl().get().getUrl().toString();
     ResponseInfo ri = fsd.getRequestInfo(normalUrl);