Skip to content

Commit c939829

Browse files
codingPFcodingPF
committed
update datasource and config
1 parent 12627e2 commit c939829

10 files changed

Lines changed: 154 additions & 66 deletions

File tree

‎src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
1313
private final String filmlistHashFilePath;
1414
private final Boolean writeFilmlistIdFileEnabled;
1515
private final String filmlistIdFilePath;
16+
private final MServerDBConfig mServerDBConfig;
1617
/** ignore certain film by title **/
1718
private String ignoreFilmlistPath;
1819
/** add livestreams from external list **/
@@ -48,6 +49,7 @@ public MServerConfigDTO() {
4849
filmlistSavePaths = new EnumMap<>(FilmlistFormats.class);
4950
filmlistDiffSavePaths = new EnumMap<>(FilmlistFormats.class);
5051
copySettings = new MServerCopySettings();
52+
mServerDBConfig = new MServerDBConfig();
5153
logSettings = new MServerLogSettingsDTO();
5254
crawlerURLs = new EnumMap<>(CrawlerUrlType.class);
5355

@@ -242,7 +244,11 @@ public ImportLivestreamConfiguration getImportLivestreamConfiguration() {
242244
public List<ImportFilmlistConfiguration> getImportFilmlistConfigurations() {
243245
return importFilmlistConfigurations;
244246
}
245-
247+
248+
public MServerDBConfig getMServerDBConfig() {
249+
return mServerDBConfig;
250+
}
251+
246252
/**
247253
* Loads the {@link Sender} specific configuration and if it not exist creates one.
248254
*
@@ -286,7 +292,8 @@ public boolean equals(final Object o) {
286292
&& Objects.equals(getFilmlistIdFilePath(), that.getFilmlistIdFilePath())
287293
&& Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath())
288294
&& Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration())
289-
&& Objects.equals(getImportFilmlistConfigurations(), that.getImportFilmlistConfigurations());
295+
&& Objects.equals(getImportFilmlistConfigurations(), that.getImportFilmlistConfigurations())
296+
&& Objects.equals(getMServerDBConfig(), that.getMServerDBConfig());
290297
}
291298

292299
@Override
@@ -313,7 +320,8 @@ public int hashCode() {
313320
getFilmlistIdFilePath(),
314321
getIgnoreFilmslistPath(),
315322
getImportLivestreamConfiguration(),
316-
getImportFilmlistConfigurations());
323+
getImportFilmlistConfigurations(),
324+
getMServerDBConfig());
317325
}
318326

319327
public void initializeSenderConfigurations() {

‎src/main/java/de/mediathekview/mserver/base/config/MServerDBConfig.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package de.mediathekview.mserver.base.config;
2+
3+
import java.util.Objects;
4+
5+
public class MServerDBConfig {
6+
private final Boolean active;
7+
private final String url;
8+
private final String username;
9+
private final String password;
10+
11+
public MServerDBConfig() {
12+
super();
13+
active = true;
14+
url = "jdbc:postgresql://OscarDS:55432/crawler";
15+
username = "crawler";
16+
password = "secret";
17+
}
18+
19+
public MServerDBConfig(Boolean active, String url, String username, String password) {
20+
super();
21+
this.active = active;
22+
this.url = url;
23+
this.username = username;
24+
this.password = password;
25+
}
26+
27+
28+
29+
public Boolean getActive() {
30+
return active;
31+
}
32+
public String getUrl() {
33+
return url;
34+
}
35+
public String getUsername() {
36+
return username;
37+
}
38+
public String getPassword() {
39+
return password;
40+
}
41+
42+
@Override
43+
public boolean equals(Object o) {
44+
if (this == o) return true;
45+
if (o == null || getClass() != o.getClass()) return false;
46+
47+
MServerDBConfig that = (MServerDBConfig) o;
48+
49+
return Objects.equals(active, that.active)
50+
&& Objects.equals(url, that.url)
51+
&& Objects.equals(username, that.username)
52+
&& Objects.equals(password, that.password);
53+
}
54+
55+
@Override
56+
public int hashCode() {
57+
return Objects.hash(active, url, username, password);
58+
}
59+
}

‎src/main/java/de/mediathekview/mserver/base/utils/FilmDBService.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,8 @@ public class FilmDBService {
4545
private final ExecutorService executorService;
4646
private final int batchSize;
4747

48-
public FilmDBService(DataSource dataSource, ExecutorService executorService, int batchSize) {
49-
this.dataSource = dataSource;
48+
public FilmDBService(ExecutorService executorService, int batchSize) {
49+
this.dataSource = PostgreSQLDataSourceProvider.get();
5050
this.executorService = executorService;
5151
this.batchSize = batchSize;
5252

@@ -136,6 +136,9 @@ public Optional<Filmlist> readFilmlistFromDB(String where) {
136136
/////////////////////////////////////////////////////////////////////////////////////////
137137

138138
public <T> List<T> filterNewVideos(List<T> videos, Function<T, String> idExtractor) {
139+
if(!PostgreSQLDataSourceProvider.isEnabled()) {
140+
return videos;
141+
}
139142
try {
140143
List<Future<List<T>>> futures = new ArrayList<>();
141144

@@ -184,6 +187,7 @@ public <T> List<T> filterNewVideos(List<T> videos, Function<T, String> idExtract
184187
LOG.debug("Filtered {} (in {} out {})",(videos.size()-result.size()), videos.size(), result.size());
185188
return result;
186189
} catch (Exception e) {
190+
LOG.error("{}", e);
187191
return videos;
188192
}
189193
}
@@ -223,6 +227,9 @@ public HashSet<String> getAllVideoUrls() {
223227
* Speichert alle Filme einer Filmlist parallel in der DB.
224228
*/
225229
public void saveAll(Filmlist filmlist) throws Exception {
230+
if(!PostgreSQLDataSourceProvider.isEnabled()) {
231+
return;
232+
}
226233
// Map in List konvertieren
227234
List<Film> films = new ArrayList<>(filmlist.getFilms().values());
228235
films = makeUniqueIds(films);

‎src/main/java/de/mediathekview/mserver/base/utils/GPDataSourceProvider.java‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

‎src/main/java/de/mediathekview/mserver/base/utils/PostgreSQLDataSourceProvider.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
package de.mediathekview.mserver.base.utils;
2+
3+
import com.zaxxer.hikari.HikariConfig;
4+
import com.zaxxer.hikari.HikariDataSource;
5+
6+
import de.mediathekview.mserver.base.config.MServerConfigManager;
7+
8+
import javax.sql.DataSource;
9+
10+
public final class PostgreSQLDataSourceProvider {
11+
private static HikariDataSource DATA_SOURCE;
12+
private static Boolean enabled = false;
13+
private MServerConfigManager aMServerConfigManager;
14+
15+
public PostgreSQLDataSourceProvider(MServerConfigManager aMServerConfigManager) {
16+
this.aMServerConfigManager = aMServerConfigManager;
17+
init();
18+
}
19+
20+
public static boolean isEnabled() {
21+
return enabled;
22+
}
23+
24+
public static DataSource get() {
25+
return DATA_SOURCE;
26+
}
27+
28+
public static void shutdown() {
29+
DATA_SOURCE.close();
30+
}
31+
32+
private void init() {
33+
HikariConfig cfg = new HikariConfig();
34+
enabled = aMServerConfigManager.getConfig().getMServerDBConfig().getActive();
35+
if(!enabled) {
36+
return;
37+
}
38+
cfg.setJdbcUrl(aMServerConfigManager.getConfig().getMServerDBConfig().getUrl());
39+
cfg.setUsername(aMServerConfigManager.getConfig().getMServerDBConfig().getUsername());
40+
cfg.setPassword(aMServerConfigManager.getConfig().getMServerDBConfig().getPassword());
41+
42+
// === Pool Sizing (wichtig!) ===
43+
cfg.setMaximumPoolSize(16); // Sweet Spot für 10k+/min
44+
cfg.setMinimumIdle(4);
45+
46+
// === Performance ===
47+
cfg.setAutoCommit(true);
48+
cfg.setConnectionTimeout(3000);
49+
cfg.setIdleTimeout(600_000);
50+
cfg.setMaxLifetime(1_800_000);
51+
52+
// === PostgreSQL Optimierungen ===
53+
cfg.addDataSourceProperty("reWriteBatchedInserts", "true");
54+
cfg.addDataSourceProperty("stringtype", "unspecified");
55+
56+
// === Debug (optional) ===
57+
cfg.setPoolName("CrawlerPool");
58+
59+
DATA_SOURCE = new HikariDataSource(cfg);
60+
}
61+
}

‎src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
import de.mediathekview.mserver.base.uploader.copy.FileCopyTask;
1717
import de.mediathekview.mserver.base.utils.CheckUrlAvailability;
1818
import de.mediathekview.mserver.base.utils.FilmDBService;
19-
import de.mediathekview.mserver.base.utils.GPDataSourceProvider;
2019
import de.mediathekview.mserver.crawler.ard.ArdCrawler;
2120
import de.mediathekview.mserver.crawler.arte.ArteCrawler;
2221
import de.mediathekview.mserver.crawler.arte.ArteCrawler_EN;
@@ -46,7 +45,6 @@
4645
import java.util.*;
4746
import java.util.Map.Entry;
4847
import java.util.concurrent.*;
49-
import javax.sql.DataSource;
5048
import org.apache.logging.log4j.LogManager;
5149
import org.apache.logging.log4j.Logger;
5250
import org.jetbrains.annotations.NotNull;
@@ -97,8 +95,7 @@ public MServerConfigManager getConfigManager() {
9795
}
9896

9997
public void storeFilmsToDB() {
100-
DataSource ds = GPDataSourceProvider.get();
101-
FilmDBService filmDBService = new FilmDBService(ds, executorService, 200);
98+
FilmDBService filmDBService = new FilmDBService(executorService, 200);
10299
try {
103100
filmDBService.saveAll(filmlist);
104101
} catch (Exception e) {

‎src/main/java/de/mediathekview/mserver/crawler/basic/AbstractCrawler.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
import de.mediathekview.mserver.base.messages.listener.MessageListener;
77
import de.mediathekview.mserver.base.progress.Progress;
88
import de.mediathekview.mserver.base.utils.FilmDBService;
9-
import de.mediathekview.mserver.base.utils.GPDataSourceProvider;
109
import de.mediathekview.mserver.base.config.MServerBasicConfigDTO;
1110
import de.mediathekview.mserver.base.config.MServerConfigDTO;
1211
import de.mediathekview.mserver.base.config.MServerConfigManager;
@@ -72,7 +71,7 @@ protected AbstractCrawler(
7271
rootConfig.getSenderConfig(getSender()).getSocketTimeoutInSeconds(),
7372
runtimeConfig.getMaximumCpuThreads());
7473
rateLimiter = RateLimiter.create(rootConfig.getSenderConfig(getSender()).getMaximumRequestsPerSecond());
75-
filmDBService = new FilmDBService(GPDataSourceProvider.get(), forkJoinPool, 200);
74+
filmDBService = new FilmDBService(forkJoinPool, 200);
7675
films = ConcurrentHashMap.newKeySet();
7776
}
7877

‎src/main/java/de/mediathekview/mserver/ui/config/MServerConfigUI.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import de.mediathekview.mserver.base.messages.listener.LogMessageListener;
44
import de.mediathekview.mserver.base.messages.listener.MessageListener;
5+
import de.mediathekview.mserver.base.utils.PostgreSQLDataSourceProvider;
56
import de.mediathekview.mserver.base.config.Log4JConfigurationFactory;
67
import de.mediathekview.mserver.base.config.MServerConfigManager;
78
import de.mediathekview.mserver.base.config.MServerLogSettingsDTO;
@@ -114,6 +115,8 @@ void start(final String[] aProgramAgruments) {
114115
LOG = LogManager.getLogger(MServerConfigUI.class);
115116
logMessageListener = new LogMessageListener();
116117
//
118+
new PostgreSQLDataSourceProvider(aMServerConfigManager); // init singleton
119+
//
117120
manager = new CrawlerManager(aMServerConfigManager);
118121
final MServerLogSettingsDTO logSettings = aMServerConfigManager.getConfig().getLogSettings();
119122
logSettings.setLogActivateConsole(true);

‎src/main/java/de/mediathekview/mserver/ui/config/MServerExecutionFlow.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99

1010
import de.mediathekview.mserver.base.utils.CheckUrlAvailability;
1111
import de.mediathekview.mserver.base.utils.FilmDBService;
12-
import de.mediathekview.mserver.base.utils.GPDataSourceProvider;
1312
import de.mediathekview.mserver.crawler.CrawlerManager;
1413
import de.mediathekview.mserver.daten.Filmlist;
1514
import de.mediathekview.mserver.ui.config.MServerCommandLine.CMDARG;
@@ -65,7 +64,7 @@ void startCrawlerFlow() {
6564

6665
void exportFilmListFromDB() {
6766
try {
68-
FilmDBService filmDBService = new FilmDBService(GPDataSourceProvider.get(), manager.getExecutorService(), 2000);
67+
FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
6968
Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB();
7069
dbFilmlist.ifPresent(filmlist -> manager.getFilmlist().addAllFilms(filmlist.getFilms().values()));
7170
//
@@ -83,7 +82,7 @@ void exportFilmListFromDB() {
8382
}
8483
void importFilmlistIntoDB() {
8584
manager.importFilmlist();
86-
FilmDBService filmDBService = new FilmDBService(GPDataSourceProvider.get(), manager.getExecutorService(), 2000);
85+
FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
8786
HashSet<String> allVideoUrls = filmDBService.getAllVideoUrls();
8887
LOG.debug("allVideoUrls loaded {} entries", allVideoUrls.size());
8988
manager.getFilmlist().getFilms().entrySet().parallelStream()
@@ -106,7 +105,7 @@ void importFilmlistIntoDB() {
106105
manager.stop();
107106
}
108107
void checkAvailability() {
109-
FilmDBService filmDBService = new FilmDBService(GPDataSourceProvider.get(), manager.getExecutorService(), 2000);
108+
FilmDBService filmDBService = new FilmDBService(manager.getExecutorService(), 2000);
110109
String condition = "where last_url_check IS NULL OR last_url_check < NOW() - INTERVAL '1 DAY'";
111110
Optional<Filmlist> dbFilmlist = filmDBService.readFilmlistFromDB(condition);
112111
dbFilmlist.ifPresent(filmlist -> manager.getFilmlist().addAllFilms(filmlist.getFilms().values()));

‎src/main/resources/MServer-Config.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,12 @@ copySettings:
188188
# JSON_COMPRESSED: /var/www/mediathekview/filmlisten/filmliste_diff.json.xz
189189
OLD_JSON_COMPRESSED_XZ: copyTarget/filmliste_old_diff.json.xz
190190

191+
mServerDBConfig:
192+
active: false
193+
url: "url"
194+
username: "username"
195+
password: "password"
196+
191197
#### Logging ####
192198
logSettings:
193199
# The log level for the console.

0 commit comments

Comments
 (0)