Skip to content

Commit be49f30

Browse files
alex1702alex1702
committed
ZDF fix normal urls not optimized
fix 3Sat and Phoenix
2 parents 6e4370f + 98ec3d5 commit be49f30

10 files changed

Lines changed: 48 additions & 62 deletions

File tree

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
2727
targetCompatibility = JavaVersion.VERSION_17
2828
group = 'de.mediathekview'
2929
archivesBaseName = "MServer"
30-
version = '3.1.257'
30+
version = '3.1.258'
3131

3232
def jarName = 'MServer.jar'
3333
def mainClass = 'mServer.Main'

src/main/java/mServer/crawler/sender/zdf/AbstractZdfCrawler.java renamed to src/main/java/mServer/crawler/sender/dreisat/AbstractZdfCrawler.java

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package mServer.crawler.sender.zdf;
1+
package mServer.crawler.sender.dreisat;
22

33
import de.mediathekview.mlib.daten.DatenFilm;
44
import de.mediathekview.mlib.tool.Log;
@@ -7,6 +7,7 @@
77
import mServer.crawler.sender.MediathekCrawler;
88
import mServer.crawler.sender.base.CrawlerUrlDTO;
99
import mServer.crawler.sender.base.JsoupConnection;
10+
import mServer.crawler.sender.zdf.ZdfConfiguration;
1011
import mServer.crawler.sender.zdf.tasks.ZdfDayPageTask;
1112
import mServer.crawler.sender.zdf.tasks.ZdfFilmDetailTask;
1213
import mServer.crawler.sender.zdf.tasks.ZdfIndexPageTask;
@@ -53,7 +54,7 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
5354
meldungAddMax(shows.size());
5455
}
5556

56-
return new ZdfFilmDetailTask(this, getApiUrlBase(), new ConcurrentLinkedQueue<>(shows), configuration.getVideoAuthKey());
57+
return new ZdfFilmDetailTask(this, getApiUrlBase(), new ConcurrentLinkedQueue<>(shows), configuration.getVideoAuthKey(), DreisatConstants.PARTNER_TO_SENDER);
5758
} catch (final InterruptedException ex) {
5859
LOG.debug("{} crawler interrupted.", getSendername(), ex);
5960
Thread.currentThread().interrupt();

‎src/main/java/mServer/crawler/sender/dreisat/DreiSatCrawler.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import mServer.crawler.FilmeSuchen;
55
import mServer.crawler.sender.base.CrawlerUrlDTO;
66
import mServer.crawler.sender.base.JsoupConnection;
7-
import mServer.crawler.sender.zdf.AbstractZdfCrawler;
87
import mServer.crawler.sender.zdf.ZdfConfiguration;
98
import org.jetbrains.annotations.NotNull;
109

‎src/main/java/mServer/crawler/sender/dreisat/DreisatConstants.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
package mServer.crawler.sender.dreisat;
22

3+
import de.mediathekview.mlib.Const;
4+
5+
import java.util.HashMap;
6+
import java.util.Map;
7+
38
public final class DreisatConstants {
49

510
/**
@@ -19,6 +24,11 @@ public final class DreisatConstants {
1924

2025
public static final String URL_HTML_DAY = URL_BASE + "/programm?airtimeDate=%s";
2126

27+
public static final Map<String, String> PARTNER_TO_SENDER = new HashMap<>();
28+
static {
29+
PARTNER_TO_SENDER.put("3sat", Const.DREISAT);
30+
}
31+
2232
private DreisatConstants() {
2333
}
2434
}

‎src/main/java/mServer/crawler/sender/phoenix/PhoenixConstants.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
package mServer.crawler.sender.phoenix;
22

3+
import de.mediathekview.mlib.Const;
4+
5+
import java.util.HashMap;
6+
import java.util.Map;
7+
38
public final class PhoenixConstants {
49
private PhoenixConstants() {}
510

@@ -9,4 +14,9 @@ private PhoenixConstants() {}
914

1015
public static final String URL_FILM_DETAIL_JSON = "/response/id/";
1116
public static final String URL_VIDEO_DETAILS = "%s/php/mediaplayer/data/beitrags_details.php?id=%s";
17+
18+
public static final Map<String, String> PARTNER_TO_SENDER = new HashMap<>();
19+
static {
20+
PARTNER_TO_SENDER.put("Phoenix", Const.PHOENIX);
21+
}
1222
}

‎src/main/java/mServer/crawler/sender/phoenix/tasks/PhoenixFilmDetailTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ protected void processRestTarget(CrawlerUrlDTO aDTO, WebTarget aTarget) {
5656
this.filmDetailHost,
5757
filmDetailDto.getBaseName())));
5858
final ZdfFilmDetailTask zdfFilmDetailTask =
59-
new ZdfFilmDetailTask(this.crawler, "", shows, Optional.empty());
59+
new ZdfFilmDetailTask(this.crawler, "", shows, Optional.empty(), PhoenixConstants.PARTNER_TO_SENDER);
6060
final Set<DatenFilm> films = zdfFilmDetailTask.invoke();
6161
films.forEach(
6262
film -> {

‎src/main/java/mServer/crawler/sender/zdf/ZdfCrawler.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
6969
Set<CrawlerUrlDTO> shows = new HashSet<>(getDaysEntries(configuration));
7070
Log.sysLog(getSendername() + " Anzahl: " + shows.size());
7171
meldungAddMax(shows.size());
72-
return new ZdfFilmDetailTask(this, getApiUrlBase(), new ConcurrentLinkedQueue<>(shows), configuration.getVideoAuthKey());
72+
return new ZdfFilmDetailTask(this, getApiUrlBase(), new ConcurrentLinkedQueue<>(shows), configuration.getVideoAuthKey(), ZdfConstants.PARTNER_TO_SENDER);
7373
}
7474
}
7575
} catch (final InterruptedException ex) {

‎src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import de.mediathekview.mlib.Const;
55
import mServer.crawler.sender.base.JsonUtils;
66
import mServer.crawler.sender.base.UrlUtils;
7-
import mServer.crawler.sender.zdf.ZdfConstants;
87
import mServer.crawler.sender.zdf.ZdfFilmDtoOld;
98
import org.apache.logging.log4j.LogManager;
109
import org.apache.logging.log4j.Logger;
@@ -59,9 +58,11 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
5958
= DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ssXXX"); // 2016-10-29T16:15:00+02:00
6059

6160
private final String apiUrlBase;
61+
private final Map<String, String> partnerToSender;
6262

63-
public ZdfFilmDetailDeserializer(final String apiUrlBase) {
63+
public ZdfFilmDetailDeserializer(final String apiUrlBase, Map<String, String> partnerToSender) {
6464
this.apiUrlBase = apiUrlBase;
65+
this.partnerToSender = partnerToSender;
6566
}
6667

6768
@Override
@@ -72,7 +73,7 @@ public Optional<ZdfFilmDtoOld> deserialize(
7273
JsonObject mainVideoTarget = null;
7374

7475
final Optional<String> tvService = JsonUtils.getAttributeAsString(rootNode, "tvService");
75-
if (tvService.isPresent() && !ZdfConstants.PARTNER_TO_SENDER.containsKey(tvService.get())) {
76+
if (tvService.isPresent() && !partnerToSender.containsKey(tvService.get())) {
7677
LOG.debug("ZdfFilmDetailDeserializer: ignore film of sender {}", tvService.orElse("EMPTY"));
7778
return Optional.empty();
7879
}
@@ -106,7 +107,7 @@ public Optional<ZdfFilmDtoOld> deserialize(
106107
final Map<String, String> downloadUrl = parseDownloadUrls(mainVideoTarget);
107108

108109
if (title.isPresent() && downloadUrl.containsKey(DOWNLOAD_URL_DEFAULT)) {
109-
return Optional.of(new ZdfFilmDtoOld(tvService.orElse(Const.ZDF), downloadUrl.get(DOWNLOAD_URL_DEFAULT), topic, title.get(), description, website, time, duration, downloadUrl.get(DOWNLOAD_URL_DGS)));
110+
return Optional.of(new ZdfFilmDtoOld(partnerToSender.get(tvService.orElse(Const.ZDF)), downloadUrl.get(DOWNLOAD_URL_DEFAULT), topic, title.get(), description, website, time, duration, downloadUrl.get(DOWNLOAD_URL_DGS)));
110111
} else {
111112
LOG.error("ZdfFilmDetailDeserializer: no title or url found");
112113
}

‎src/main/java/mServer/crawler/sender/zdf/tasks/ZdfFilmDetailTask.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,16 +44,19 @@ public class ZdfFilmDetailTask extends ZdfTaskBase<DatenFilm, CrawlerUrlDTO> {
4444

4545
private final transient ZdfVideoUrlOptimizer optimizer = new ZdfVideoUrlOptimizer();
4646
private final String apiUrlBase;
47+
private final Map<String, String> partnerToSender;
4748

4849
public ZdfFilmDetailTask(
4950
final MediathekReader aCrawler,
5051
final String aApiUrlBase,
5152
final ConcurrentLinkedQueue<CrawlerUrlDTO> aUrlToCrawlDtos,
52-
final Optional<String> aAuthKey) {
53+
final Optional<String> aAuthKey,
54+
final Map<String, String> partnerToSender) {
5355
super(aCrawler, aUrlToCrawlDtos, aAuthKey);
5456
apiUrlBase = aApiUrlBase;
57+
this.partnerToSender = partnerToSender;
5558

56-
registerJsonDeserializer(OPTIONAL_FILM_TYPE_TOKEN, new ZdfFilmDetailDeserializer(apiUrlBase));
59+
registerJsonDeserializer(OPTIONAL_FILM_TYPE_TOKEN, new ZdfFilmDetailDeserializer(apiUrlBase, partnerToSender));
5760
registerJsonDeserializer(OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN, new ZdfDownloadDtoDeserializer());
5861
}
5962

@@ -104,7 +107,7 @@ private void appendSignLanguage(DownloadDto downloadDto, Optional<String> urlSig
104107
@Override
105108
protected AbstractRecursivConverterTask<DatenFilm, CrawlerUrlDTO> createNewOwnInstance(
106109
final ConcurrentLinkedQueue<CrawlerUrlDTO> aElementsToProcess) {
107-
return new ZdfFilmDetailTask(crawler, apiUrlBase, aElementsToProcess, authKey);
110+
return new ZdfFilmDetailTask(crawler, apiUrlBase, aElementsToProcess, authKey, partnerToSender);
108111
}
109112

110113
private void addFilm(final DownloadDto downloadDto, final ZdfFilmDtoOld result) {

‎src/main/java/mServer/crawler/sender/zdf/tasks/ZdfFilmTask.java‎

Lines changed: 11 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -81,32 +81,16 @@ protected AbstractRestTask<DatenFilm, ZdfFilmDto> createNewOwnInstance(
8181

8282
private void addFilm(final ZdfFilmDto zdfFilmDto, final DownloadDto downloadDto) {
8383

84-
String previousLanguage = null;
85-
DatenFilm previousMainFilm = null;
8684
for (final String language : downloadDto.getLanguages().stream().sorted().toList()) {
8785
// todo deu-ad bei audiodescription passt nicht im title
88-
// todo kann der auskommentierte code hier weg?
89-
/* if (previousLanguage != null && language.startsWith(previousLanguage)) {
90-
final DatenFilm currentFilm = previousMainFilm;
91-
92-
if (language.endsWith(ZdfConstants.LANGUAGE_SUFFIX_AD)) {
93-
final Map<Qualities, String> urls =
94-
getOptimizedUrls(downloadDto.getDownloadUrls(language));
95-
urls.forEach(currentFilm::addAudioDescription);
96-
} else if (language.endsWith(ZdfConstants.LANGUAGE_SUFFIX_DGS)) {
97-
final Map<Qualities, String> urls =
98-
getOptimizedUrls(downloadDto.getDownloadUrls(language));
99-
urls.forEach(currentFilm::addSignLanguage);
100-
} else {
101-
LOG.debug("unknown language suffix: {}", language);
102-
}
103-
} else {*/
104-
final DatenFilm filmWithLanguage = createFilm(zdfFilmDto, downloadDto, language);
86+
DownloadDtoFilmConverter.getOptimizedUrls(
87+
downloadDto.getDownloadUrls(language), Optional.of(optimizer));
88+
89+
final DatenFilm filmWithLanguage = createFilm(zdfFilmDto, downloadDto, language);
10590
setSubtitle(downloadDto, filmWithLanguage, language);
10691
setGeoLocation(downloadDto, filmWithLanguage);
10792

108-
final Map<Qualities, String> urls =
109-
getOptimizedUrls(downloadDto.getDownloadUrls(language));
93+
final Map<Qualities, String> urls = downloadDto.getDownloadUrls(language);
11094
if (urls.containsKey(Qualities.SMALL)) {
11195
CrawlerTool.addUrlKlein(filmWithLanguage, urls.get(Qualities.SMALL));
11296
}
@@ -117,35 +101,7 @@ private void addFilm(final ZdfFilmDto zdfFilmDto, final DownloadDto downloadDto)
117101
if (!taskResults.add(filmWithLanguage)) {
118102
LOG.error("Rejected duplicate {}", filmWithLanguage);
119103
}
120-
previousMainFilm = filmWithLanguage;
121-
previousLanguage = language;
122-
//}
123-
}
124-
}
125-
126-
private Map<Qualities, String> getOptimizedUrls(Map<Qualities, String> urls) {
127-
Map<Qualities, String> result = new EnumMap<>(Qualities.class);
128-
129-
for (final Map.Entry<Qualities, String> qualitiesEntry : urls.entrySet()) {
130-
String url = qualitiesEntry.getValue();
131-
132-
if (qualitiesEntry.getKey() == Qualities.NORMAL) {
133-
url = optimizer.getOptimizedUrlNormal(url);
134-
} else if (qualitiesEntry.getKey() == Qualities.HD) {
135-
url = optimizer.getOptimizedUrlHd(url);
136-
}
137-
138-
result.put(qualitiesEntry.getKey(), url);
139-
}
140-
141-
if (!result.containsKey(Qualities.HD)) {
142-
final Optional<String> hdUrl = optimizer.determineUrlHd(result.get(Qualities.NORMAL));
143-
if (hdUrl.isPresent()) {
144-
result.put(
145-
Qualities.HD, hdUrl.get());
146-
}
147104
}
148-
return result;
149105
}
150106

151107
private static void setSubtitle(DownloadDto downloadDto, DatenFilm filmWithLanguage, String language) {
@@ -165,6 +121,12 @@ private static String updateTitle(final String aLanguage, final String aTitle) {
165121
switch (aLanguage) {
166122
case ZdfConstants.LANGUAGE_GERMAN:
167123
break;
124+
case ZdfConstants.LANGUAGE_GERMAN_AD:
125+
title += " (Audiodeskription)";
126+
break;
127+
case ZdfConstants.LANGUAGE_GERMAN_DGS:
128+
title += " (Gebärdensprache)";
129+
break;
168130
case ZdfConstants.LANGUAGE_ENGLISH:
169131
title += " (Englisch)";
170132
break;

0 commit comments

Comments
 (0)