Skip to content

Commit 96ab15b

Browse files
alex1702alex1702
committed
ARD OV videos missing
title: rename (deu-ad) to (audiodescription)
2 parents be49f30 + dd2626c commit 96ab15b

4 files changed

Lines changed: 107 additions & 71 deletions

File tree

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
2727
targetCompatibility = JavaVersion.VERSION_17
2828
group = 'de.mediathekview'
2929
archivesBaseName = "MServer"
30-
version = '3.1.258'
30+
version = '3.1.259'
3131

3232
def jarName = 'MServer.jar'
3333
def mainClass = 'mServer.Main'

‎src/main/java/mServer/crawler/AddToFilmlist.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ private void performInitialCleanup() {
136136
removeTimeFromOrf(listeEinsortieren);
137137
updateAudioDescriptionOrf(listeEinsortieren);
138138
updateAudioDescriptionSrf(listeEinsortieren);
139+
updateAudioDescriptionZdf(listeEinsortieren);
139140
updateThema(listeEinsortieren);
140141
updateTitle(listeEinsortieren);
141142
updateArdWebsite(listeEinsortieren);
@@ -247,6 +248,19 @@ private void updateAudioDescriptionSrf(ListeFilme listeEinsortieren) {
247248
}
248249
}
249250

251+
private void updateAudioDescriptionZdf(ListeFilme listeEinsortieren) {
252+
final List<DatenFilm> list = listeEinsortieren.parallelStream()
253+
.filter(
254+
film -> film.arr[DatenFilm.FILM_TITEL].contains("(deu-ad)"))
255+
.toList();
256+
Log.sysLog("ZDF: erstetze (deu-ad) mit " + list.size() + " (Audiodeskription).");
257+
if (!list.isEmpty()) {
258+
list.forEach(film -> {
259+
film.arr[DatenFilm.FILM_TITEL] = film.arr[DatenFilm.FILM_TITEL].replace("(deu-ad)", " (Audiodeskription)").trim();
260+
});
261+
}
262+
}
263+
250264
private void removeTimeFromOrf(ListeFilme listeEinsortieren) {
251265
final List<DatenFilm> list = listeEinsortieren.parallelStream()
252266
.filter(

‎src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java‎

Lines changed: 74 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ private static Optional<String> parseTopic(final JsonObject playerPageObject) {
146146
private Optional<String> parseTitle(final JsonObject playerPageObject) {
147147
Optional<String> title = JsonUtils.getAttributeAsString(playerPageObject, ATTRIBUTE_TITLE);
148148
if (title.isPresent()) {
149-
String[] replaceWords = {" - Hörfassung", " (mit Gebärdensprache)", " mit Gebärdensprache"," (mit Audiodeskription)", "Audiodeskription"};
149+
String[] replaceWords = {" - Hörfassung", " (mit Gebärdensprache)", " mit Gebärdensprache"," (mit Audiodeskription)", "Audiodeskription", " - (Originalversion)", " (OV)"};
150150
String cleanTitle = title.get().trim();
151151
for (String replaceWord : replaceWords) {
152152
cleanTitle = cleanTitle.replace(replaceWord, "");
@@ -224,80 +224,47 @@ public List<ArdFilmDto> deserialize(
224224

225225
final Optional<String> topic = parseTopic(itemObject);
226226
final Optional<String> title = parseTitle(itemObject);
227-
final Optional<String> description
228-
= JsonUtils.getAttributeAsString(itemObject, ATTRIBUTE_SYNOPSIS);
227+
final Optional<String> titleoriginal = JsonUtils.getAttributeAsString(itemObject, ATTRIBUTE_TITLE);
228+
final Optional<String> description = JsonUtils.getAttributeAsString(itemObject, ATTRIBUTE_SYNOPSIS);
229229
final Optional<LocalDateTime> date = parseDate(itemObject);
230230
final Optional<Duration> duration = parseDuration(itemObject);
231231
final Optional<String> partner = parsePartner(itemObject);
232-
final Optional<Map<Qualities, String>> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
233-
final Optional<Map<Qualities, String>> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
234-
final Optional<Map<Qualities, String>> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
235-
final Optional<Map<Qualities, String>> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
236-
final Optional<Map<Qualities, String>> videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
237-
final Optional<String> subtitles = prepareSubtitleUrl(itemObject);
232+
Optional<Map<Qualities, String>> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
233+
Optional<Map<Qualities, String>> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
234+
Optional<Map<Qualities, String>> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
235+
Optional<Map<Qualities, String>> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
236+
Optional<Map<Qualities, String>> videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
237+
Optional<String> subtitles = prepareSubtitleUrl(itemObject);
238238

239239
if (topic.isEmpty() || title.isEmpty() || partner.isEmpty() || ADDITIONAL_SENDER.get(partner.get()) == null) {
240240
if (partner.isPresent() && ADDITIONAL_SENDER.get(partner.get()) == null) {
241241
LOG.warn("Missing Partner " + partner.get());
242242
}
243243
return films;
244244
}
245-
246-
247-
248-
if ((title.get().toUpperCase().contains("(OV)") || title.get().toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) {
249-
final ArdFilmDto filmDto
250-
= new ArdFilmDto(
251-
createFilm(
252-
ADDITIONAL_SENDER.get(partner.get()),
253-
topic.get(),
254-
title.get(),
255-
description,
256-
date,
257-
duration,
258-
videoInfoOV.get(),
259-
subtitles));
260-
films.add(filmDto);
261-
return films;
245+
boolean subVideoPage = titleoriginal.get().contains(" - (Originalversion)") || titleoriginal.get().contains(" (OV)")
246+
|| titleoriginal.get().contains(" (mit Gebärdensprache)") || titleoriginal.get().contains(" mit Gebärdensprache")
247+
|| titleoriginal.get().contains("- Hörfassung") || titleoriginal.get().contains("(mit Audiodeskription)");
248+
// mainly funk
249+
if (videoInfoStandard.isEmpty() && videoInfoAD.isEmpty() && videoInfoDGS.isEmpty() && videoInfoOV.isEmpty() && videoInfoAdaptive.isPresent()) {
250+
videoInfoStandard = resolveFallbackFromPlaylist(videoInfoAdaptive);
262251
}
263-
264-
if(videoInfoStandard.isEmpty() && videoInfoAD.isEmpty() && videoInfoDGS.isEmpty() && videoInfoAdaptive.isPresent()) {
265-
// UUAAAARRGGGG - SAD
266-
Map<Qualities, URL> qualitiesUrls = videoInfoAdaptive.get().entrySet().stream()
267-
.collect(Collectors.toMap(Map.Entry::getKey, entry -> {
268-
try {
269-
return new URL(entry.getValue());
270-
} catch (MalformedURLException e) {
271-
LOG.error("failed converting string {} to url", entry.getValue(), e);
272-
return null;
273-
}
274-
}));
275-
if (!qualitiesUrls.containsKey(Qualities.NORMAL)) {
276-
qualitiesUrls.put(Qualities.NORMAL, qualitiesUrls.entrySet().stream().findFirst().get().getValue());
277-
}
278-
//
279-
ArdVideoInfoJsonDeserializer.loadM3U8(qualitiesUrls);
280-
//
281-
Map<Qualities, String> fallback = qualitiesUrls.entrySet().stream()
282-
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toString()));
283-
284-
if (fallback.size() > 0) {
285-
final ArdFilmDto filmDto
286-
= new ArdFilmDto(
287-
createFilm(
288-
ADDITIONAL_SENDER.get(partner.get()),
289-
topic.get(),
290-
title.get(),
291-
description,
292-
date,
293-
duration,
294-
fallback,
295-
subtitles));
296-
films.add(filmDto);
297-
}
252+
// incorrect langueage code for OV
253+
if ((titleoriginal.get().contains(" - (Originalversion)") || titleoriginal.get().contains(" (OV)")) && videoInfoOV.isEmpty()) {
254+
videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, "*");
298255
}
299-
300-
if (videoInfoStandard.isPresent() && videoInfoStandard.get().size() > 0) {
256+
// mainly tagesschau
257+
if ((titleoriginal.get().contains(" (mit Gebärdensprache)") || titleoriginal.get().contains(" mit Gebärdensprache")) && videoInfoStandard.isPresent() && videoInfoDGS.isEmpty()) {
258+
videoInfoDGS = videoInfoStandard;
259+
videoInfoStandard = Optional.empty();
260+
}
261+
// in aller freundschaft
262+
if ((titleoriginal.get().contains("- Hörfassung") || titleoriginal.get().contains("(mit Audiodeskription)")) && videoInfoStandard.isPresent() && videoInfoAD.isEmpty()) {
263+
videoInfoAD = videoInfoStandard;
264+
videoInfoStandard = Optional.empty();
265+
}
266+
//
267+
if (!subVideoPage && videoInfoStandard.isPresent() && videoInfoStandard.get().size() > 0) {
301268
// add film standard
302269
final ArdFilmDto filmDto
303270
= new ArdFilmDto(
@@ -310,9 +277,25 @@ public List<ArdFilmDto> deserialize(
310277
duration,
311278
videoInfoStandard.get(),
312279
subtitles));
280+
films.add(filmDto);
313281
if (widgets.size() > 1) {
314282
parseRelatedFilms(filmDto, widgets.get(1).getAsJsonObject());
315283
}
284+
}
285+
//
286+
if (videoInfoOV.isPresent() && videoInfoOV.get().size() > 0) {
287+
Map<Qualities, String> vid = videoInfoOV.get();
288+
final ArdFilmDto filmDto
289+
= new ArdFilmDto(
290+
createFilm(
291+
ADDITIONAL_SENDER.get(partner.get()),
292+
topic.get(),
293+
title.get() + " (Originalversion)",
294+
description,
295+
date,
296+
duration,
297+
vid,
298+
subtitles));
316299
films.add(filmDto);
317300
}
318301
//
@@ -329,9 +312,6 @@ public List<ArdFilmDto> deserialize(
329312
duration,
330313
videoInfoAD.get(),
331314
subtitles));
332-
if (widgets.size() > 1) {
333-
parseRelatedFilms(filmDto, widgets.get(1).getAsJsonObject());
334-
}
335315
films.add(filmDto);
336316
}
337317
//
@@ -348,9 +328,6 @@ public List<ArdFilmDto> deserialize(
348328
duration,
349329
videoInfoDGS.get(),
350330
subtitles));
351-
if (widgets.size() > 1) {
352-
parseRelatedFilms(filmDto, widgets.get(1).getAsJsonObject());
353-
}
354331
films.add(filmDto);
355332
}
356333

@@ -473,7 +450,8 @@ private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerP
473450
Optional<String> resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H);
474451
Optional<String> url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL);
475452
Optional<String> languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG);
476-
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) {
453+
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) &&
454+
(languageCode.orElse("").equalsIgnoreCase(language) || (language.equalsIgnoreCase("*") && !languageCode.orElse("").equalsIgnoreCase("deu")))) {
477455
videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get()));
478456
}
479457
}
@@ -486,4 +464,30 @@ private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerP
486464
}
487465
return Optional.of(videoInfo);
488466
}
467+
468+
private Optional<Map<Qualities, String>> resolveFallbackFromPlaylist(Optional<Map<Qualities, String>> videoInfoAdaptive) {
469+
Map<Qualities, URL> qualitiesUrls = videoInfoAdaptive.get().entrySet().stream()
470+
.collect(Collectors.toMap(Map.Entry::getKey, entry -> {
471+
try {
472+
return new URL(entry.getValue());
473+
} catch (MalformedURLException e) {
474+
LOG.error("failed converting string {} to url", entry.getValue(), e);
475+
return null;
476+
}
477+
}));
478+
if (!qualitiesUrls.containsKey(Qualities.NORMAL)) {
479+
qualitiesUrls.put(Qualities.NORMAL, qualitiesUrls.entrySet().stream().findFirst().get().getValue());
480+
}
481+
//
482+
ArdVideoInfoJsonDeserializer.loadM3U8(qualitiesUrls);
483+
//
484+
Map<Qualities, String> fallback = qualitiesUrls.entrySet().stream()
485+
.collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().toString()));
486+
//
487+
if (!fallback.containsKey(Qualities.NORMAL) && !fallback.isEmpty()) {
488+
fallback.put(Qualities.NORMAL, fallback.entrySet().stream().findFirst().get().getValue());
489+
}
490+
491+
return Optional.of(fallback);
492+
}
489493
}

‎src/test/developTest/java/mServer/crawler/AddToFilmlistTest.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,6 +541,24 @@ public void testArdTagesschau24UpdateSender() {
541541
assertEquals(Const.TAGESSCHAU24, testFilmArd24.arr[DatenFilm.FILM_SENDER]);
542542
}
543543

544+
@Test
545+
public void testZdfReplaceAdDeutWithAudiodescription() {
546+
final DatenFilm testFilm1 = createTestFilm(Const.ZDF, "Filme ZDF", "Film1(deu-ad)", FILM_NAME_ONLINE);
547+
final DatenFilm testFilm2 = createTestFilm(Const.ZDF_NEO, "Filme Neo", "Film2(deu-ad)", FILM_NAME_ONLINE);
548+
final DatenFilm testFilm3 = createTestFilm(Const.ARD, "Filme ARD", "Film3(deu-ad)", FILM_NAME_ONLINE);
549+
listToAdd.add(testFilm1);
550+
listToAdd.add(testFilm2);
551+
listToAdd.add(testFilm3);
552+
553+
AddToFilmlist target =new AddToFilmlist(list, listToAdd);
554+
target.addOldList();
555+
556+
assertEquals(list.size(),5);
557+
assertEquals("Film1 (Audiodeskription)", testFilm1.arr[DatenFilm.FILM_TITEL]);
558+
assertEquals("Film2 (Audiodeskription)", testFilm2.arr[DatenFilm.FILM_TITEL]);
559+
assertEquals("Film3 (Audiodeskription)", testFilm3.arr[DatenFilm.FILM_TITEL]);
560+
}
561+
544562
private static DatenFilm createTestFilm(String sender, String topic, String title,
545563
String filmUrl) {
546564
DatenFilm film = new DatenFilm(sender, topic, "url", title, baseUrl + filmUrl, "", "", "", 12,

0 commit comments

Comments
 (0)