Skip to content

update maxSubpages and search by broadcastdate #1109 #1110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
codingPF wants to merge 1 commit into
base: master
Choose a base branch
from
+18 −29
Open

update maxSubpages and search by broadcastdate #1109 #1110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/main/java/mServer/crawler/sender/arte/ArteConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package mServer.crawler.sender.arte;

public class ArteConstants {
public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-creationDate&language=%s";
public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=creationDate&language=%s";
public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-broadcastBegin&language=%s";
public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=broadcastBegin&language=%s";
public static final String VIDEO_URL ="https://www.arte.tv/hbbtvv2/services/web/index.php/OPA/v3/streams/%s/%s/%s"; //PROGRAMID/KIND/LANG
public static final String API_TOKEN = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA";
public static final int MAX_POSSIBLE_SUBPAGES = 100;
private ArteConstants() {}

}
22 changes: 9 additions & 13 deletions src/main/java/mServer/crawler/sender/arte/ArteCrawler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {

final ArteVideoInfoTask aArteRestVideoInfoTask;
// DO NOT overload - maximumUrlsPerTask used to reduce threads to 4
aArteRestVideoInfoTask = new ArteVideoInfoTask(this, videoUrls);
aArteRestVideoInfoTask = new ArteVideoInfoTask(this, videoUrls, getMaxPagesForOverview(getLanguage().toString().toLowerCase()));
final ConcurrentLinkedQueue<ArteVideoInfoDto> videos = new ConcurrentLinkedQueue<>();
videos.addAll(aArteRestVideoInfoTask.fork().join());
//
Expand All @@ -71,29 +71,25 @@ protected RecursiveTask<Set<DatenFilm>> createCrawlerTask() {
}

private ConcurrentLinkedQueue<TopicUrlDTO> createVideosQueue(String language) {
int maxPages = getMaxPagesForOverview(language);
final ConcurrentLinkedQueue<TopicUrlDTO> root = new ConcurrentLinkedQueue<>();
String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, language);
root.add(new TopicUrlDTO("all videos1", rootUrl));
if (maxPages >= 100) {
String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language);
root.add(new TopicUrlDTO("all videos2", rootUrl2));
}
root.add(new TopicUrlDTO("all videos sorted up", rootUrl));
String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language);
root.add(new TopicUrlDTO("all videos sorted down", rootUrl2));
return root;
}

private int getMaxPagesForOverview(String lang) {
final int maxAvailablePages = getNumberOfAvailablePages(lang);
final int configuredMaxPages = getMaximumSubpages();
if (configuredMaxPages > maxAvailablePages) {
return Math.min(configuredMaxPages, maxAvailablePages / 2);
return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, maxAvailablePages / 2);
} else {
return Math.min(configuredMaxPages, configuredMaxPages / 2);
return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, configuredMaxPages / 2);
}
}

private int getNumberOfAvailablePages(String lang) {
final int naturalLimit = Math.min(100, getMaximumSubpages());
try {
String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, lang);
String[] path= {"meta", "videos", "pages"};
Expand All @@ -110,14 +106,14 @@ private int getNumberOfAvailablePages(String lang) {
} catch (IOException e) {
LOG.error("getMaxPagesForOverview", e);
}
return naturalLimit;
return ArteConstants.MAX_POSSIBLE_SUBPAGES;
}

private int getMaximumSubpages() {
if (CrawlerTool.loadLongMax()) {
return 10;
return 30;
} else {
return 1;
return 10;
}
}
}
Expand Down
20 changes: 6 additions & 14 deletions src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@ public class ArteVideoInfoTask
private static final long serialVersionUID = 1L;
protected final transient Logger log = LogManager.getLogger(this.getClass());
protected transient Optional<AbstractRecursivConverterTask<ArteVideoInfoDto, TopicUrlDTO>> nextPageTask = Optional.empty();

protected int maxSubpages;

public ArteVideoInfoTask(MediathekReader crawler, ConcurrentLinkedQueue<TopicUrlDTO> urlToCrawlDTOs) {
public ArteVideoInfoTask(MediathekReader crawler, ConcurrentLinkedQueue<TopicUrlDTO> urlToCrawlDTOs, int maxSubpages) {
super(crawler, urlToCrawlDTOs, Optional.of(ArteConstants.API_TOKEN));
this.maxSubpages = maxSubpages;
}

@Override
Expand All @@ -49,9 +50,8 @@ protected void postProcessingNextPage(PagedElementListDTO<ArteVideoInfoDto> aRes
if (aResponseObj.getNextPage().isEmpty()) {
return;
}
int maxPages = Math.min(100, getMaximumSubpages());
if (aResponseObj.getNextPage().get().contains("age="+maxPages)) {
log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxPages);
if (aResponseObj.getNextPage().get().contains("age="+maxSubpages)) {
log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxSubpages);
return;
}

Expand All @@ -61,14 +61,6 @@ protected void postProcessingNextPage(PagedElementListDTO<ArteVideoInfoDto> aRes
nextPageTask.get().fork();
}

private int getMaximumSubpages() {
if (CrawlerTool.loadLongMax()) {
return 10;
} else {
return 3;
}
}

protected void postProcessingElements(Set<ArteVideoInfoDto> elements) {
for (ArteVideoInfoDto element : elements) {
taskResults.add(element);
Expand All @@ -86,7 +78,7 @@ protected void postProcessing(PagedElementListDTO<ArteVideoInfoDto> aResponseObj
@Override
protected AbstractRecursivConverterTask<ArteVideoInfoDto, TopicUrlDTO> createNewOwnInstance(
ConcurrentLinkedQueue<TopicUrlDTO> aElementsToProcess) {
return new ArteVideoInfoTask(crawler, aElementsToProcess);
return new ArteVideoInfoTask(crawler, aElementsToProcess, maxSubpages);
}

@Override
Expand Down