Skip to content

Commit 129e281

Browse files
codingPFcodingPF
committed
add ratelimiter, cleanup comments, fix ard test, DW backwards compat fix
1 parent a94230a commit 129e281

File tree

5 files changed

+2
-74
lines changed

5 files changed

+2
-74
lines changed

‎src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -210,9 +210,6 @@ public List<ArdFilmDto> deserialize(
210210
date.orElse(null),
211211
duration.orElse(null),
212212
videoInfo.get()));
213-
//if (widgets.size() > 1) {
214-
//parseRelatedFilms(filmDto, widgets.get(1).getAsJsonObject());
215-
//}
216213
films.add(filmDto);
217214
}
218215
// OV - long term this should go into Film as "OV"
@@ -297,23 +294,6 @@ private Optional<String> parsePartner(final JsonObject playerPageObject) {
297294
return Optional.empty();
298295
}
299296

300-
private void parseRelatedFilms(final ArdFilmDto filmDto, final JsonObject playerPageObject) {
301-
if (playerPageObject.has(ELEMENT_TEASERS)) {
302-
final JsonElement teasersElement = playerPageObject.get(ELEMENT_TEASERS);
303-
if (teasersElement.isJsonArray()) {
304-
for (final JsonElement teasersItemElement : teasersElement.getAsJsonArray()) {
305-
final JsonObject teasersItemObject = teasersItemElement.getAsJsonObject();
306-
final Optional<String> id =
307-
JsonUtils.getAttributeAsString(teasersItemObject, ATTRIBUTE_ID);
308-
if (id.isPresent()) {
309-
final String url = String.format(ArdConstants.ITEM_URL, id.get());
310-
filmDto.addRelatedFilm(new ArdFilmInfoDto(id.get(), url, 0));
311-
}
312-
}
313-
}
314-
}
315-
}
316-
317297
private Film createFilm(
318298
final String id,
319299
final Sender sender,
@@ -446,12 +426,6 @@ private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject,
446426

447427
}
448428
}
449-
/*
450-
Optional<Map<Integer, String>> tt = parseVideoUrlMap(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
451-
String a = videoInfoAdaptive.get().entrySet().stream().findFirst().get().getValue();
452-
if(tt.isPresent() && !a.startsWith("https://funk") && !a.contains("arte") )
453-
//UrlOptimizer.debug(a, tt.get());
454-
urlOptimizer.debug2(a, videoInfoStandard.get());*/
455429
}
456430

457431
return Optional.of(allVideoUrls);

‎src/main/java/de/mediathekview/mserver/crawler/ard/tasks/ArdTopicPageTask.java‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
3939
&& topicInfo.getFilmInfos() != null
4040
&& !topicInfo.getFilmInfos().isEmpty()) {
4141
taskResults.addAll(topicInfo.getFilmInfos());
42-
//LOG.debug("Found {} shows for a topic of ARD.", topicInfo.getFilmInfos().size());
4342

4443
final Queue<CrawlerUrlDTO> subpages = createSubPageUrls(aTarget, topicInfo);
4544
if (!subpages.isEmpty()) {
@@ -65,7 +64,6 @@ private Queue<CrawlerUrlDTO> createSubPageUrls(
6564
break;
6665
}
6766
}
68-
//LOG.debug("Found {} subpage", subpages.size());
6967
return subpages;
7068
}
7169

‎src/main/java/de/mediathekview/mserver/crawler/basic/AbstractJsonRestTask.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ protected void processRestTarget(final D aDTO, final WebTarget aTarget) {
7777
}
7878
//log.debug("Too Many Requests - propsoal: {} waiting: {} ", proposalWaitMillis, waitMillis);
7979
Thread.sleep(waitMillis);
80+
crawler.getRateLimiter().acquire();
8081
continue;
8182
}
8283
handleHttpError(aDTO, aTarget.getUri(), response);

‎src/main/java/de/mediathekview/mserver/crawler/dw/parser/DwFilmDetailDeserializer.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -245,14 +245,8 @@ private Optional<Map<Resolution, FilmUrl>> getVideos(
245245
videoListe.remove(Resolution.HD);
246246
}
247247
}
248-
// TODO: add more resolutions etc - but for backwards compatibility - set to the current once
249-
final Map<Resolution, FilmUrl> videoListeBackwardsCompat = new ConcurrentHashMap<>();
250-
videoListeBackwardsCompat.put(Resolution.HD, videoListe.get(Resolution.WQHD));
251-
videoListeBackwardsCompat.put(Resolution.NORMAL, videoListe.get(Resolution.NORMAL));
252-
videoListeBackwardsCompat.put(Resolution.SMALL, videoListe.get(Resolution.SMALL));
253-
254248
if (videoListe.size() > 0) {
255-
return Optional.of(videoListeBackwardsCompat);
249+
return Optional.of(videoListe);
256250
}
257251
LOG.error("No video url for video: {}", videoid);
258252
return Optional.empty();

‎src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializerTest.java‎

Lines changed: 0 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,11 @@
55
import de.mediathekview.mserver.daten.Sender;
66
import de.mediathekview.mserver.base.messages.listener.MessageListener;
77
import de.mediathekview.mserver.base.config.MServerConfigManager;
8-
import de.mediathekview.mserver.crawler.ard.ArdConstants;
98
import de.mediathekview.mserver.crawler.ard.ArdCrawler;
109
import de.mediathekview.mserver.crawler.ard.ArdFilmDto;
11-
import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
1210
import de.mediathekview.mserver.progress.listeners.SenderProgressListener;
1311
import de.mediathekview.mserver.testhelper.AssertFilm;
1412
import de.mediathekview.mserver.testhelper.JsonFileReader;
15-
import org.hamcrest.Matchers;
1613
import org.junit.Test;
1714
import org.junit.runner.RunWith;
1815
import org.junit.runners.Parameterized;
@@ -45,7 +42,6 @@ public class ArdFilmDeserializerTest {
4542
private final String expectedDGSUrlHd;
4643
private final String expectedSubtitle;
4744
private final GeoLocations expectedGeo;
48-
private final ArdFilmInfoDto[] relatedFilms;
4945
private final Optional<Sender> additionalSender;
5046

5147
protected MServerConfigManager rootConfig = new MServerConfigManager("MServer-JUnit-Config.yaml");
@@ -68,7 +64,6 @@ public ArdFilmDeserializerTest(
6864
final String expectedDGSUrlHd,
6965
final String expectedSubtitle,
7066
final GeoLocations expectedGeo,
71-
final ArdFilmInfoDto[] relatedFilms,
7267
final Optional<Sender> additionalSender) {
7368
this.jsonFile = jsonFile;
7469
this.expectedTopic = expectedTopic;
@@ -87,7 +82,6 @@ public ArdFilmDeserializerTest(
8782
this.expectedDGSUrlHd = expectedDGSUrlHd;
8883
this.expectedSubtitle = expectedSubtitle;
8984
this.expectedGeo = expectedGeo;
90-
this.relatedFilms = relatedFilms;
9185
this.additionalSender = additionalSender;
9286
}
9387

@@ -113,7 +107,6 @@ public static Collection<Object[]> data() {
113107
/*DGShd*/ "https://mediandr-a.akamaihd.net/progressive_geo/2022/0104/TV-20220104-0902-5000.hd.mp4",
114108
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/ebutt/urn:ard:subtitle:eaa2ed13a677cd00",
115109
/*hd*/ GeoLocations.GEO_DE,
116-
/*related*/ new ArdFilmInfoDto[0],
117110
/*sender*/ Optional.of(Sender.KIKA)
118111
},
119112
{
@@ -134,16 +127,6 @@ public static Collection<Object[]> data() {
134127
/*DGShd*/ "https://pd-videos.daserste.de/int/2024/01/24/03247ab1-4dcc-427e-b577-a6ca25c1dffe/JOB_432151_sendeton_1920x1080-50p-5000kbit.mp4",
135128
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:7b0043ec0b358eb8.vtt",
136129
/*hd*/ GeoLocations.GEO_NONE,
137-
/*related*/ new ArdFilmInfoDto[] {
138-
new ArdFilmInfoDto(
139-
"Y3JpZDovL3dkci5kZS9CZWl0cmFnLThlNjczODVlLWZhZTktNDMwYi1iNzI1LTA0NjU1ZmRmMDljZQ",
140-
String.format(ArdConstants.ITEM_URL, "Y3JpZDovL3dkci5kZS9CZWl0cmFnLThlNjczODVlLWZhZTktNDMwYi1iNzI1LTA0NjU1ZmRmMDljZQ"),
141-
0),
142-
new ArdFilmInfoDto(
143-
"Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtZWRmMTRhM2UtNmM3Ny00NGZhLTg1ZWYtYTJkYmZmNzM0NTg5",
144-
String.format(ArdConstants.ITEM_URL, "Y3JpZDovL3dkci5kZS9CZWl0cmFnLXNvcGhvcmEtZWRmMTRhM2UtNmM3Ny00NGZhLTg1ZWYtYTJkYmZmNzM0NTg5"),
145-
0)
146-
},
147130
/*sender*/ Optional.of(Sender.ARD)
148131
},
149132
{
@@ -164,12 +147,6 @@ public static Collection<Object[]> data() {
164147
/*DGShd */ "",
165148
/*sub*/ "",
166149
/*hd*/ GeoLocations.GEO_NONE,
167-
/*related*/ new ArdFilmInfoDto[] {
168-
new ArdFilmInfoDto(
169-
"Y3JpZDovL2JyLmRlL3ZpZGVvLzkwZTA1Y2Y5LTA4ZDEtNGU4Zi1iNTQyLWNiYjIyYzcyZDA0Mw",
170-
String.format(ArdConstants.ITEM_URL, "Y3JpZDovL2JyLmRlL3ZpZGVvLzkwZTA1Y2Y5LTA4ZDEtNGU4Zi1iNTQyLWNiYjIyYzcyZDA0Mw"),
171-
0)
172-
},
173150
/*sender*/ Optional.of(Sender.BR)
174151
},
175152
{
@@ -190,12 +167,6 @@ public static Collection<Object[]> data() {
190167
/*DGShd */ "",
191168
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:c09c9cee3bf53db8.vtt",
192169
/*hd*/ GeoLocations.GEO_NONE,
193-
/*related*/ new ArdFilmInfoDto[] {
194-
new ArdFilmInfoDto(
195-
"Y3JpZDovL3RhZ2Vzc2NoYXUuZGUvNTBjOTc0OGUtMTIwYi00MjllLWI2ODEtZTkyMTY5ODEyNGI0X2dhbnplU2VuZHVuZw",
196-
String.format(ArdConstants.ITEM_URL, "Y3JpZDovL3RhZ2Vzc2NoYXUuZGUvNTBjOTc0OGUtMTIwYi00MjllLWI2ODEtZTkyMTY5ODEyNGI0X2dhbnplU2VuZHVuZw"),
197-
0)
198-
},
199170
/*sender*/ Optional.of(Sender.ARD),
200171
},
201172
{
@@ -216,7 +187,6 @@ public static Collection<Object[]> data() {
216187
/*DGShd */ "",
217188
/*sub*/ "",
218189
/*hd*/ GeoLocations.GEO_NONE,
219-
/*related*/ new ArdFilmInfoDto[0],
220190
/*sender*/ Optional.of(Sender.HR),
221191
},
222192
{
@@ -237,7 +207,6 @@ public static Collection<Object[]> data() {
237207
/*DGShd */ "",
238208
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:ea9ad6b71df1b8ed.vtt",
239209
/*hd*/ GeoLocations.GEO_NONE,
240-
/*related*/ new ArdFilmInfoDto[0],
241210
/*sender*/ Optional.of(Sender.NDR),
242211
},
243212
{
@@ -258,7 +227,6 @@ public static Collection<Object[]> data() {
258227
/*DGShd */ "",
259228
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:0567b031db73e4b9.vtt",
260229
/*hd*/ GeoLocations.GEO_DE,
261-
/*related*/ new ArdFilmInfoDto[0],
262230
/*sender*/ Optional.of(Sender.ONE),
263231
},
264232
{
@@ -279,7 +247,6 @@ public static Collection<Object[]> data() {
279247
/*DGShd */ "",
280248
/*sub*/ "",
281249
/*hd*/ GeoLocations.GEO_NONE,
282-
/*related*/ new ArdFilmInfoDto[0],
283250
/*sender*/ Optional.of(Sender.RBB),
284251
},
285252
{
@@ -300,7 +267,6 @@ public static Collection<Object[]> data() {
300267
/*DGShd */ "",
301268
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:a1d11ac623c7d120.vtt",
302269
/*hd*/ GeoLocations.GEO_NONE,
303-
/*related*/ new ArdFilmInfoDto[0],
304270
/*sender*/ Optional.of(Sender.ARD),
305271
},
306272
{
@@ -321,7 +287,6 @@ public static Collection<Object[]> data() {
321287
/*DGShd */ "",
322288
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/ebutt/urn:ard:subtitle:7d1c01087f8cae77",
323289
/*hd*/ GeoLocations.GEO_DE,
324-
/*related*/ new ArdFilmInfoDto[0],
325290
/*sender*/ Optional.of(Sender.MDR),
326291
},
327292
{
@@ -342,7 +307,6 @@ public static Collection<Object[]> data() {
342307
/*DGShd */ "",
343308
/*sub*/ "",
344309
/*hd*/ GeoLocations.GEO_DE,
345-
/*related*/ new ArdFilmInfoDto[0],
346310
/*sender*/ Optional.of(Sender.SWR),
347311
},
348312
{
@@ -363,7 +327,6 @@ public static Collection<Object[]> data() {
363327
/*DGShd */ "",
364328
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:efab8bf55007171e.vtt",
365329
/*hd*/ GeoLocations.GEO_DE,
366-
/*related*/ new ArdFilmInfoDto[0],
367330
/*sender*/ Optional.of(Sender.ARD),
368331
},
369332
{
@@ -384,7 +347,6 @@ public static Collection<Object[]> data() {
384347
/*DGShd */ "",
385348
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:d0e38dd26e6cc85e.vtt",
386349
/*hd*/ GeoLocations.GEO_DE,
387-
/*related*/ new ArdFilmInfoDto[0],
388350
/*sender*/ Optional.of(Sender.ONE),
389351
}
390352
});
@@ -422,7 +384,6 @@ public void test() {
422384
expectedADUrlNormal,
423385
expectedADUrlHd,
424386
expectedSubtitle);
425-
//assertThat(films[0].getRelatedFilms(), Matchers.containsInAnyOrder(relatedFilms));
426387
}
427388
}
428389

0 commit comments

Comments
 (0)