From 3f678c49e5f94fe0bcae2f21ba4b4f28913614b4 Mon Sep 17 00:00:00 2001 From: Hamza Malkawi Date: Tue, 12 Aug 2025 16:06:01 +0200 Subject: [PATCH 1/4] feat: fix http requests rate limit using caching --- highcharts_excentis/highcharts/highcharts.py | 75 ++++++++++++++------ pyproject.toml | 3 +- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/highcharts_excentis/highcharts/highcharts.py b/highcharts_excentis/highcharts/highcharts.py index 45ec45d..c884642 100644 --- a/highcharts_excentis/highcharts/highcharts.py +++ b/highcharts_excentis/highcharts/highcharts.py @@ -1,31 +1,43 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals, absolute_import -from future.standard_library import install_aliases -install_aliases() -from past.builtins import basestring +from __future__ import absolute_import, unicode_literals -from jinja2 import Environment, PackageLoader +from future.standard_library import install_aliases -import json, uuid -import re +install_aliases() import datetime -import urllib.request, urllib.error, urllib.parse import html -from .options import BaseOptions, ChartOptions, ColorAxisOptions, \ - ColorsOptions, CreditsOptions, DrilldownOptions, ExportingOptions, \ - GlobalOptions, LabelsOptions, LangOptions, \ - LegendOptions, LoadingOptions, NavigationOptions, PaneOptions, \ - PlotOptions, SeriesData, SubtitleOptions, TitleOptions, \ - TooltipOptions, xAxisOptions, yAxisOptions, zAxisOptions, MultiAxis +import json +import re +import urllib.error +import urllib.parse +import urllib.request +import uuid +import diskcache as dc +from jinja2 import Environment, PackageLoader +from past.builtins import basestring + +from .common import (ArrayObject, ColorObject, CommonObject, CSSObject, + Formatter, JSfunction, Levels, RawJavaScriptText, + SVGObject) from .highchart_types import Series, SeriesOptions -from .common import Levels, Formatter, CSSObject, SVGObject, JSfunction, RawJavaScriptText, \ - CommonObject, ArrayObject, ColorObject +from .options import (BaseOptions, ChartOptions, ColorAxisOptions, + ColorsOptions, CreditsOptions, DrilldownOptions, + ExportingOptions, GlobalOptions, LabelsOptions, + LangOptions, LegendOptions, LoadingOptions, MultiAxis, + NavigationOptions, PaneOptions, PlotOptions, SeriesData, + SubtitleOptions, TitleOptions, TooltipOptions, + xAxisOptions, yAxisOptions, zAxisOptions) CONTENT_FILENAME = "content.html" PAGE_FILENAME = "page.html" +# 50MB limit, to ensure LRU evection +_CACHE = dc.Cache('./highcharts_cache', size_limit=50 * 1024**2) + +_CACHE.cull() # Clean up expired entries + pl = PackageLoader('highcharts_excentis.highcharts', 'templates') jinja2_env = Environment(lstrip_blocks=True, trim_blocks=True, loader=pl) @@ -347,15 +359,36 @@ def buildhtmlheader(self): opener = urllib.request.build_opener() opener.addheaders = [('User-Agent', 'Mozilla/5.0')] + + def _get_or_download(url: str, + ttl=60 * 60 * 24 * 7): # Default: 7 days + """Perform caching for Highcarts JS libraries to avoid HTTP rate limit.""" + if url in _CACHE: + print(f"Highcharts: Cache hit, pulling cached URL {url} ") + return _CACHE[url] + else: + try: + # your actual HTTP request + print(f"Highcharts: Downloading and caching url {url}") + content = opener.open(url).read().decode( + 'utf-8').replace('\n', '') + _CACHE.set(url, content, expire=ttl) # 1 week TTL + return content + except urllib.error.HTTPError as e: + print( + f"Highcharts: HTTP Error {e.code} for URL: {url}") + except urllib.error.URLError as e: + print( + f"Highcharts: URL Error: {e.reason} for URL: {url}" + ) + self.header_css = [ - '' for source in self.CSSsource + '' + for source in self.CSSsource ] self.header_js = [ - '' for source in self.JSsource ] diff --git a/pyproject.toml b/pyproject.toml index fcebc57..8592cb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,8 @@ maintainers = [ requires-python = ">=3.4" dependencies = [ 'future', - 'Jinja2', # for templates + 'diskcache', # to avoid http rate limit + 'Jinja2' # for templates ] keywords = [ 'python', From 580f7b8bb4b624abd68c748838c18a8fbc3fd2fd Mon Sep 17 00:00:00 2001 From: Hamza Malkawi Date: Wed, 13 Aug 2025 09:57:11 +0200 Subject: [PATCH 2/4] docs: describe caching approach used --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index 1586655..b9e8e79 100644 --- a/README.md +++ b/README.md @@ -596,6 +596,26 @@ chart.save_file() ``` +## 📦 HTTP Request Caching + +**Summary:** +Added persistent, on-disk caching for Highcharts JavaScript library downloads to avoid repeated network requests and prevent hitting Highcharts rate limits. + +**Key Changes:** +- Introduced [`diskcache`](https://pypi.org/project/diskcache/) to store downloaded JS files in `./highcharts_cache` (50 MB limit). +- Default **time-to-live (TTL)** for cached files: **7 days**. +- Requests now go through a `get_or_download(url)` helper: + - **Cache hit** → return stored content. + - **Cache miss** → download via `urllib.request`, store in cache, return result. +- Automatic cleanup of expired entries using `cache.cull()`. + +**What Users Should Know:** +- First run may download required files; subsequent runs reuse local cache. +- Cache directory path (`./highcharts_cache`) works with project’s CI/CD and artifact/cache strategy. + this directory is automatically generated in your working directory +- Prevents repeated HTTP calls between test runs and across pipelines when artifacts/cache are reused. +- Cache can be cleared manually by deleting `./highcharts_cache`. + ## Todo: * More examples From 0b9486e169b82d65aaf8e91dc533b476ebdab6c4 Mon Sep 17 00:00:00 2001 From: Hamza Malkawi Date: Mon, 18 Aug 2025 16:03:36 +0200 Subject: [PATCH 3/4] Apply suggestions from code review fix: prefix cache directory with . to make it hidden Co-authored-by: Tom Ghyselinck --- README.md | 6 +++--- highcharts_excentis/highcharts/highcharts.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b9e8e79..d037ab9 100644 --- a/README.md +++ b/README.md @@ -602,7 +602,7 @@ chart.save_file() Added persistent, on-disk caching for Highcharts JavaScript library downloads to avoid repeated network requests and prevent hitting Highcharts rate limits. **Key Changes:** -- Introduced [`diskcache`](https://pypi.org/project/diskcache/) to store downloaded JS files in `./highcharts_cache` (50 MB limit). +- Introduced [`diskcache`](https://pypi.org/project/diskcache/) to store downloaded JS files in `./.cache/highcharts-excentis` (50 MB limit). - Default **time-to-live (TTL)** for cached files: **7 days**. - Requests now go through a `get_or_download(url)` helper: - **Cache hit** → return stored content. @@ -611,10 +611,10 @@ Added persistent, on-disk caching for Highcharts JavaScript library downloads to **What Users Should Know:** - First run may download required files; subsequent runs reuse local cache. -- Cache directory path (`./highcharts_cache`) works with project’s CI/CD and artifact/cache strategy. +- Cache directory path (`./.cache/highcharts-excentis`) works with project’s CI/CD and artifact/cache strategy. this directory is automatically generated in your working directory - Prevents repeated HTTP calls between test runs and across pipelines when artifacts/cache are reused. -- Cache can be cleared manually by deleting `./highcharts_cache`. +- Cache can be cleared manually by deleting `./.cache/highcharts-excentis`. ## Todo: diff --git a/highcharts_excentis/highcharts/highcharts.py b/highcharts_excentis/highcharts/highcharts.py index c884642..67e53be 100644 --- a/highcharts_excentis/highcharts/highcharts.py +++ b/highcharts_excentis/highcharts/highcharts.py @@ -34,7 +34,7 @@ PAGE_FILENAME = "page.html" # 50MB limit, to ensure LRU evection -_CACHE = dc.Cache('./highcharts_cache', size_limit=50 * 1024**2) +_CACHE = dc.Cache('./.cache/highcharts-excentis', size_limit=50 * 1024**2) _CACHE.cull() # Clean up expired entries From ac0f37d3c4abd5f46d60b72f58653154ee0c0e35 Mon Sep 17 00:00:00 2001 From: Hamza Malkawi Date: Mon, 18 Aug 2025 16:04:08 +0200 Subject: [PATCH 4/4] fix: docs Co-authored-by: Tom Ghyselinck --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d037ab9..3b7470d 100644 --- a/README.md +++ b/README.md @@ -599,7 +599,7 @@ chart.save_file() ## 📦 HTTP Request Caching **Summary:** -Added persistent, on-disk caching for Highcharts JavaScript library downloads to avoid repeated network requests and prevent hitting Highcharts rate limits. +Added persistent, on-disk caching for Highcharts JavaScript library downloads to avoid repeated network requests and prevent hitting Highcharts CDN rate limits. **Key Changes:** - Introduced [`diskcache`](https://pypi.org/project/diskcache/) to store downloaded JS files in `./.cache/highcharts-excentis` (50 MB limit).