From 8ea2a3ffb9d02587e7916f2ab351e3d2260d6f8e Mon Sep 17 00:00:00 2001 From: CodingPF Date: Fri, 19 Dec 2025 12:24:54 +0100 Subject: [PATCH] update maxSubpages and search by broadcastdate --- .../crawler/sender/arte/ArteConstants.java | 5 +++-- .../crawler/sender/arte/ArteCrawler.java | 22 ++++++++----------- .../sender/arte/tasks/ArteVideoInfoTask.java | 20 +++++------------ 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/src/main/java/mServer/crawler/sender/arte/ArteConstants.java b/src/main/java/mServer/crawler/sender/arte/ArteConstants.java index 9c21ac9c..445aac70 100644 --- a/src/main/java/mServer/crawler/sender/arte/ArteConstants.java +++ b/src/main/java/mServer/crawler/sender/arte/ArteConstants.java @@ -1,10 +1,11 @@ package mServer.crawler.sender.arte; public class ArteConstants { - public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-creationDate&language=%s"; - public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=creationDate&language=%s"; + public static final String VIDEOS_URL ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=-broadcastBegin&language=%s"; + public static final String VIDEOS_URL_ALT ="https://api.arte.tv/api/opa/v3/videos?limit=100&page=%s&sort=broadcastBegin&language=%s"; public static final String VIDEO_URL ="https://www.arte.tv/hbbtvv2/services/web/index.php/OPA/v3/streams/%s/%s/%s"; //PROGRAMID/KIND/LANG public static final String API_TOKEN = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA"; + public static final int MAX_POSSIBLE_SUBPAGES = 100; private ArteConstants() {} } diff --git a/src/main/java/mServer/crawler/sender/arte/ArteCrawler.java b/src/main/java/mServer/crawler/sender/arte/ArteCrawler.java index c2c23ed5..531d1f2f 100644 --- a/src/main/java/mServer/crawler/sender/arte/ArteCrawler.java +++ b/src/main/java/mServer/crawler/sender/arte/ArteCrawler.java @@ -50,7 +50,7 @@ protected RecursiveTask> createCrawlerTask() { final ArteVideoInfoTask aArteRestVideoInfoTask; // DO NOT overload - maximumUrlsPerTask used to reduce threads to 4 - aArteRestVideoInfoTask = new ArteVideoInfoTask(this, videoUrls); + aArteRestVideoInfoTask = new ArteVideoInfoTask(this, videoUrls, getMaxPagesForOverview(getLanguage().toString().toLowerCase())); final ConcurrentLinkedQueue videos = new ConcurrentLinkedQueue<>(); videos.addAll(aArteRestVideoInfoTask.fork().join()); // @@ -71,14 +71,11 @@ protected RecursiveTask> createCrawlerTask() { } private ConcurrentLinkedQueue createVideosQueue(String language) { - int maxPages = getMaxPagesForOverview(language); final ConcurrentLinkedQueue root = new ConcurrentLinkedQueue<>(); String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, language); - root.add(new TopicUrlDTO("all videos1", rootUrl)); - if (maxPages >= 100) { - String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language); - root.add(new TopicUrlDTO("all videos2", rootUrl2)); - } + root.add(new TopicUrlDTO("all videos sorted up", rootUrl)); + String rootUrl2 = String.format(ArteConstants.VIDEOS_URL_ALT, 1, language); + root.add(new TopicUrlDTO("all videos sorted down", rootUrl2)); return root; } @@ -86,14 +83,13 @@ private int getMaxPagesForOverview(String lang) { final int maxAvailablePages = getNumberOfAvailablePages(lang); final int configuredMaxPages = getMaximumSubpages(); if (configuredMaxPages > maxAvailablePages) { - return Math.min(configuredMaxPages, maxAvailablePages / 2); + return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, maxAvailablePages / 2); } else { - return Math.min(configuredMaxPages, configuredMaxPages / 2); + return Math.min(ArteConstants.MAX_POSSIBLE_SUBPAGES, configuredMaxPages / 2); } } private int getNumberOfAvailablePages(String lang) { - final int naturalLimit = Math.min(100, getMaximumSubpages()); try { String rootUrl = String.format(ArteConstants.VIDEOS_URL, 1, lang); String[] path= {"meta", "videos", "pages"}; @@ -110,14 +106,14 @@ private int getNumberOfAvailablePages(String lang) { } catch (IOException e) { LOG.error("getMaxPagesForOverview", e); } - return naturalLimit; + return ArteConstants.MAX_POSSIBLE_SUBPAGES; } private int getMaximumSubpages() { if (CrawlerTool.loadLongMax()) { - return 10; + return 30; } else { - return 1; + return 10; } } } diff --git a/src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java b/src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java index aaafbd5b..91dd2b4a 100644 --- a/src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java +++ b/src/main/java/mServer/crawler/sender/arte/tasks/ArteVideoInfoTask.java @@ -29,10 +29,11 @@ public class ArteVideoInfoTask private static final long serialVersionUID = 1L; protected final transient Logger log = LogManager.getLogger(this.getClass()); protected transient Optional> nextPageTask = Optional.empty(); - + protected int maxSubpages; - public ArteVideoInfoTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs) { + public ArteVideoInfoTask(MediathekReader crawler, ConcurrentLinkedQueue urlToCrawlDTOs, int maxSubpages) { super(crawler, urlToCrawlDTOs, Optional.of(ArteConstants.API_TOKEN)); + this.maxSubpages = maxSubpages; } @Override @@ -49,9 +50,8 @@ protected void postProcessingNextPage(PagedElementListDTO aRes if (aResponseObj.getNextPage().isEmpty()) { return; } - int maxPages = Math.min(100, getMaximumSubpages()); - if (aResponseObj.getNextPage().get().contains("age="+maxPages)) { - log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxPages); + if (aResponseObj.getNextPage().get().contains("age="+maxSubpages)) { + log.debug("stop at page url {} due to limit {}", aResponseObj.getNextPage().get(), maxSubpages); return; } @@ -61,14 +61,6 @@ protected void postProcessingNextPage(PagedElementListDTO aRes nextPageTask.get().fork(); } - private int getMaximumSubpages() { - if (CrawlerTool.loadLongMax()) { - return 10; - } else { - return 3; - } - } - protected void postProcessingElements(Set elements) { for (ArteVideoInfoDto element : elements) { taskResults.add(element); @@ -86,7 +78,7 @@ protected void postProcessing(PagedElementListDTO aResponseObj @Override protected AbstractRecursivConverterTask createNewOwnInstance( ConcurrentLinkedQueue aElementsToProcess) { - return new ArteVideoInfoTask(crawler, aElementsToProcess); + return new ArteVideoInfoTask(crawler, aElementsToProcess, maxSubpages); } @Override