Skip to content

Commit 433605b

Browse files
codingPFcodingPF
committed
move logic for daysToCrawl into util method
1 parent f84cc5a commit 433605b

7 files changed

Lines changed: 67 additions & 75 deletions

File tree

‎src/main/java/de/mediathekview/mserver/base/utils/DateUtils.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
package de.mediathekview.mserver.base.utils;
22

3+
import java.time.LocalDateTime;
4+
import java.time.format.DateTimeFormatter;
5+
import java.util.ArrayList;
6+
import java.util.List;
7+
8+
import de.mediathekview.mserver.base.config.MServerBasicConfigDTO;
9+
310
/** A set of util methods to work with dates. */
411
public class DateUtils {
512
private static final String SPLITTED_NUMBERS_REGEX_PATTERN = "$1:$2";
@@ -14,4 +21,23 @@ private DateUtils() {
1421
public static String changeDateTimeForMissingISO8601Support(final String aDateTimeString) {
1522
return aDateTimeString.replaceAll(SPLIT_NUMBERS_REGEX_PATTERN, SPLITTED_NUMBERS_REGEX_PATTERN);
1623
}
24+
25+
public static List<String> generateDaysToCrawl(MServerBasicConfigDTO config) {
26+
return generateDaysToCrawl(
27+
config.getMaximumDaysForSendungVerpasstSection(),
28+
config.getMaximumDaysForSendungVerpasstSectionFuture(),
29+
DateTimeFormatter.ofPattern("yyyy-MM-dd"));
30+
}
31+
32+
public static List<String> generateDaysToCrawl(int numberOfDaysInThePast, int numberOfDaysInTheFuture, DateTimeFormatter formatter) {
33+
List<String> days = new ArrayList<>();
34+
final LocalDateTime now = LocalDateTime.now();
35+
for (int i = 0; i <= numberOfDaysInThePast; i++) {
36+
days.add(now.minusDays(i).format(formatter));
37+
}
38+
for (int i = 1; i < numberOfDaysInTheFuture; i++) {
39+
days.add(now.plusDays(i).format(formatter));
40+
}
41+
return days;
42+
}
1743
}

‎src/main/java/de/mediathekview/mserver/crawler/ard/ArdCrawler.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import de.mediathekview.mserver.daten.Film;
44
import de.mediathekview.mserver.daten.Sender;
55
import de.mediathekview.mserver.base.messages.listener.MessageListener;
6+
import de.mediathekview.mserver.base.utils.DateUtils;
67
import de.mediathekview.mserver.base.config.MServerConfigManager;
78
import de.mediathekview.mserver.base.messages.ServerMessages;
89
import de.mediathekview.mserver.crawler.ard.tasks.*;
@@ -12,10 +13,10 @@
1213
import org.apache.logging.log4j.LogManager;
1314
import org.apache.logging.log4j.Logger;
1415

15-
import java.time.LocalDateTime;
1616
import java.time.format.DateTimeFormatter;
1717
import java.util.Collection;
1818
import java.util.HashSet;
19+
import java.util.List;
1920
import java.util.Queue;
2021
import java.util.Set;
2122
import java.util.concurrent.*;
@@ -41,17 +42,13 @@ public Sender getSender() {
4142

4243
private Queue<CrawlerUrlDTO> createDayUrlsToCrawl() {
4344
final Queue<CrawlerUrlDTO> dayUrlsToCrawl = new ConcurrentLinkedQueue<>();
44-
45-
final LocalDateTime now = LocalDateTime.now();
46-
for (int i = 0; i <= crawlerConfig.getMaximumDaysForSendungVerpasstSection(); i++) {
47-
final String day = now.minusDays(i).format(DAY_PAGE_DATE_FORMATTER);
48-
45+
final List<String> days = DateUtils.generateDaysToCrawl(crawlerConfig);
46+
days.forEach( dateString -> {
4947
for (final String client : ArdConstants.CLIENTS) {
50-
final String url =
51-
String.format(ArdConstants.DAY_PAGE_URL, day, client);
48+
final String url = String.format(ArdConstants.DAY_PAGE_URL, dateString, client);
5249
dayUrlsToCrawl.offer(new CrawlerUrlDTO(url));
5350
}
54-
}
51+
});
5552
return dayUrlsToCrawl;
5653
}
5754

‎src/main/java/de/mediathekview/mserver/crawler/dreisat/DreiSatCrawler.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,17 @@
22

33
import de.mediathekview.mserver.daten.Sender;
44
import de.mediathekview.mserver.base.messages.listener.MessageListener;
5+
import de.mediathekview.mserver.base.utils.DateUtils;
56
import de.mediathekview.mserver.base.config.MServerConfigManager;
67
import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
78
import de.mediathekview.mserver.crawler.zdf.AbstractZdfCrawler;
89
import de.mediathekview.mserver.crawler.zdf.ZdfConfiguration;
910
import de.mediathekview.mserver.progress.listeners.SenderProgressListener;
1011
import org.jetbrains.annotations.NotNull;
1112

12-
import java.time.LocalDateTime;
1313
import java.time.format.DateTimeFormatter;
14-
import java.time.temporal.ChronoUnit;
1514
import java.util.Collection;
15+
import java.util.List;
1616
import java.util.Queue;
1717
import java.util.concurrent.ConcurrentLinkedQueue;
1818
import java.util.concurrent.ExecutionException;
@@ -75,14 +75,14 @@ protected Collection<CrawlerUrlDTO> getExtraDaysEntries()
7575

7676
private Queue<CrawlerUrlDTO> getExtraDayUrls() {
7777
final Queue<CrawlerUrlDTO> urls = new ConcurrentLinkedQueue<>();
78-
for (int i = 0; i <= getMaximumDaysPast(); i++) {
79-
80-
final LocalDateTime local = LocalDateTime.now().minus(i, ChronoUnit.DAYS);
81-
final String date = local.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
82-
final String url = String.format(DreisatConstants.URL_HTML_DAY, date);
78+
final List<String> days = DateUtils.generateDaysToCrawl(
79+
getMaximumDaysPast(),
80+
crawlerConfig.getMaximumDaysForSendungVerpasstSectionFuture(),
81+
DateTimeFormatter.ofPattern("yyyy-MM-dd"));
82+
days.forEach( dateString -> {
83+
final String url = String.format(DreisatConstants.URL_HTML_DAY, dateString);
8384
urls.add(new CrawlerUrlDTO(url));
84-
}
85-
85+
});
8686
return urls;
8787
}
8888

‎src/main/java/de/mediathekview/mserver/crawler/orfon/OrfOnCrawler.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import de.mediathekview.mserver.daten.Film;
44
import de.mediathekview.mserver.daten.Sender;
55
import de.mediathekview.mserver.base.messages.listener.MessageListener;
6+
import de.mediathekview.mserver.base.utils.DateUtils;
67
import de.mediathekview.mserver.base.config.MServerConfigManager;
78
import de.mediathekview.mserver.base.messages.ServerMessages;
89
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
@@ -17,10 +18,9 @@
1718
import org.apache.logging.log4j.LogManager;
1819
import org.apache.logging.log4j.Logger;
1920

20-
import java.time.LocalDateTime;
21-
import java.time.format.DateTimeFormatter;
2221
import java.util.Collection;
2322
import java.util.HashSet;
23+
import java.util.List;
2424
import java.util.Queue;
2525
import java.util.Set;
2626
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -31,7 +31,6 @@
3131

3232
public class OrfOnCrawler extends AbstractCrawler {
3333
private static final Logger LOG = LogManager.getLogger(OrfOnCrawler.class);
34-
private static final DateTimeFormatter DAY_PAGE_DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
3534

3635
public OrfOnCrawler(
3736
final ForkJoinPool aForkJoinPool,
@@ -92,14 +91,15 @@ private Set<OrfOnBreadCrumsUrlDTO> processDayUrlsToCrawl() throws InterruptedExc
9291
return dayTaskFilms;
9392
}
9493

94+
95+
9596
private Queue<OrfOnBreadCrumsUrlDTO> createDayUrlsToCrawl() {
9697
final Queue<OrfOnBreadCrumsUrlDTO> dayUrlsToCrawl = new ConcurrentLinkedQueue<>();
97-
final LocalDateTime now = LocalDateTime.now();
98-
for (int i = 0; i <= crawlerConfig.getMaximumDaysForSendungVerpasstSection(); i++) {
99-
final String day = now.minusDays(i).format(DAY_PAGE_DATE_FORMATTER);
100-
final String url = OrfOnConstants.SCHEDULE + "/" + day;
101-
dayUrlsToCrawl.offer(new OrfOnBreadCrumsUrlDTO(day,url));
102-
}
98+
final List<String> days = DateUtils.generateDaysToCrawl(crawlerConfig);
99+
days.forEach( dateString -> {
100+
final String url = OrfOnConstants.SCHEDULE + "/" + dateString;
101+
dayUrlsToCrawl.offer(new OrfOnBreadCrumsUrlDTO(dateString, url));
102+
});
103103
return dayUrlsToCrawl;
104104
}
105105

‎src/main/java/de/mediathekview/mserver/crawler/srf/SrfCrawler.java‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import de.mediathekview.mserver.daten.Film;
44
import de.mediathekview.mserver.daten.Sender;
55
import de.mediathekview.mserver.base.messages.listener.MessageListener;
6+
import de.mediathekview.mserver.base.utils.DateUtils;
67
import de.mediathekview.mserver.base.config.MServerConfigManager;
78
import de.mediathekview.mserver.base.messages.ServerMessages;
89
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
@@ -15,11 +16,9 @@
1516
import de.mediathekview.mserver.progress.listeners.SenderProgressListener;
1617
import org.apache.logging.log4j.LogManager;
1718
import org.apache.logging.log4j.Logger;
18-
19-
import java.time.LocalDateTime;
20-
import java.time.format.DateTimeFormatter;
2119
import java.util.Collection;
2220
import java.util.HashSet;
21+
import java.util.List;
2322
import java.util.Queue;
2423
import java.util.Set;
2524
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -31,9 +30,6 @@ public class SrfCrawler extends AbstractCrawler {
3130

3231
private static final Logger LOG = LogManager.getLogger(SrfCrawler.class);
3332

34-
private static final DateTimeFormatter ISO_DATE_FORMAT =
35-
DateTimeFormatter.ofPattern("yyyy-MM-dd");
36-
3733
public SrfCrawler(
3834
final ForkJoinPool aForkJoinPool,
3935
final Collection<MessageListener> aMessageListeners,
@@ -93,15 +89,11 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
9389

9490
private Queue<CrawlerUrlDTO> createScheduleUrls() {
9591
final Queue<CrawlerUrlDTO> scheduleUrls = new ConcurrentLinkedQueue<>();
96-
final LocalDateTime now = LocalDateTime.now();
97-
for (int i = 0; i <= crawlerConfig.getMaximumDaysForSendungVerpasstSection(); i++) {
98-
final String day = now.minusDays(i).format(ISO_DATE_FORMAT);
99-
final String url = String.format(SrfConstants.SCHEDULE_PER_DAY, day);
92+
final List<String> days = DateUtils.generateDaysToCrawl(crawlerConfig);
93+
days.forEach( dateString -> {
94+
final String url = String.format(SrfConstants.SCHEDULE_PER_DAY, dateString);
10095
scheduleUrls.offer(new CrawlerUrlDTO(url));
101-
}
102-
LOG.debug("SRF crawler for schedule {} to {}",
103-
now.minusDays(crawlerConfig.getMaximumDaysForSendungVerpasstSection()).format(ISO_DATE_FORMAT),
104-
now.minusDays(0).format(ISO_DATE_FORMAT));
96+
});
10597
return scheduleUrls;
10698
}
10799

‎src/main/java/de/mediathekview/mserver/crawler/zdf/AbstractZdfCrawler.java‎

Lines changed: 6 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import de.mediathekview.mserver.daten.Film;
44
import de.mediathekview.mserver.daten.Sender;
55
import de.mediathekview.mserver.base.messages.listener.MessageListener;
6+
import de.mediathekview.mserver.base.utils.DateUtils;
67
import de.mediathekview.mserver.base.config.MServerConfigManager;
78
import de.mediathekview.mserver.base.messages.ServerMessages;
89
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
@@ -15,12 +16,10 @@
1516
import org.apache.logging.log4j.Logger;
1617
import org.jetbrains.annotations.NotNull;
1718

18-
import java.time.LocalDateTime;
19-
import java.time.format.DateTimeFormatter;
20-
import java.time.temporal.ChronoUnit;
2119
import java.util.ArrayDeque;
2220
import java.util.Collection;
2321
import java.util.HashSet;
22+
import java.util.List;
2423
import java.util.Map;
2524
import java.util.Queue;
2625
import java.util.Set;
@@ -113,21 +112,11 @@ protected Collection<CrawlerUrlDTO> getExtraDaysEntries()
113112

114113
private Queue<CrawlerUrlDTO> getDayUrls() {
115114
final Queue<CrawlerUrlDTO> urls = new ConcurrentLinkedQueue<>();
116-
for (int i = 0;
117-
i
118-
<= crawlerConfig.getMaximumDaysForSendungVerpasstSection()
119-
+ crawlerConfig.getMaximumDaysForSendungVerpasstSectionFuture();
120-
i++) {
121-
122-
final LocalDateTime local =
123-
LocalDateTime.now()
124-
.plus(crawlerConfig.getMaximumDaysForSendungVerpasstSectionFuture(), ChronoUnit.DAYS)
125-
.minus(i, ChronoUnit.DAYS);
126-
final String date = local.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
127-
final String url = String.format(getUrlDay(), date, date);
115+
final List<String> days = DateUtils.generateDaysToCrawl(crawlerConfig);
116+
days.forEach( dateString -> {
117+
final String url = String.format(getUrlDay(), dateString, dateString);
128118
urls.add(new CrawlerUrlDTO(url));
129-
}
130-
119+
});
131120
return urls;
132121
}
133122

‎src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfCrawler.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import de.mediathekview.mserver.daten.Film;
44
import de.mediathekview.mserver.daten.Sender;
55
import de.mediathekview.mserver.base.messages.listener.MessageListener;
6+
import de.mediathekview.mserver.base.utils.DateUtils;
67
import de.mediathekview.mserver.base.config.MServerConfigManager;
78
import de.mediathekview.mserver.base.messages.ServerMessages;
89
import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
@@ -11,9 +12,6 @@
1112
import de.mediathekview.mserver.crawler.zdf.tasks.*;
1213
import de.mediathekview.mserver.progress.listeners.SenderProgressListener;
1314

14-
import java.time.LocalDateTime;
15-
import java.time.format.DateTimeFormatter;
16-
import java.time.temporal.ChronoUnit;
1715
import java.util.*;
1816
import java.util.concurrent.ConcurrentLinkedQueue;
1917
import java.util.concurrent.ExecutionException;
@@ -151,21 +149,11 @@ private Set<CrawlerUrlDTO> getDaysEntries(final ZdfConfiguration configuration)
151149

152150
private Queue<CrawlerUrlDTO> getDayUrls() {
153151
final Queue<CrawlerUrlDTO> urls = new ConcurrentLinkedQueue<>();
154-
for (int i = 0;
155-
i
156-
<= crawlerConfig.getMaximumDaysForSendungVerpasstSection()
157-
+ crawlerConfig.getMaximumDaysForSendungVerpasstSectionFuture();
158-
i++) {
159-
160-
final LocalDateTime local =
161-
LocalDateTime.now()
162-
.plus(crawlerConfig.getMaximumDaysForSendungVerpasstSectionFuture(), ChronoUnit.DAYS)
163-
.minus(i, ChronoUnit.DAYS);
164-
final String date = local.format(DateTimeFormatter.ofPattern("yyyy-MM-dd"));
165-
final String url = String.format(getUrlDay(), date, date);
152+
final List<String> days = DateUtils.generateDaysToCrawl(crawlerConfig);
153+
days.forEach( dateString -> {
154+
final String url = String.format(getUrlDay(), dateString, dateString);
166155
urls.add(new CrawlerUrlDTO(url));
167-
}
168-
156+
});
169157
return urls;
170158
}
171159

0 commit comments

Comments
 (0)