diff --git a/README.md b/README.md index 1586655..3b7470d 100644 --- a/README.md +++ b/README.md @@ -596,6 +596,26 @@ chart.save_file() ``` +## 📦 HTTP Request Caching + +**Summary:** +Added persistent, on-disk caching for Highcharts JavaScript library downloads to avoid repeated network requests and prevent hitting Highcharts CDN rate limits. + +**Key Changes:** +- Introduced [`diskcache`](https://pypi.org/project/diskcache/) to store downloaded JS files in `./.cache/highcharts-excentis` (50 MB limit). +- Default **time-to-live (TTL)** for cached files: **7 days**. +- Requests now go through a `get_or_download(url)` helper: + - **Cache hit** → return stored content. + - **Cache miss** → download via `urllib.request`, store in cache, return result. +- Automatic cleanup of expired entries using `cache.cull()`. + +**What Users Should Know:** +- First run may download required files; subsequent runs reuse local cache. +- Cache directory path (`./.cache/highcharts-excentis`) works with project’s CI/CD and artifact/cache strategy. + this directory is automatically generated in your working directory +- Prevents repeated HTTP calls between test runs and across pipelines when artifacts/cache are reused. +- Cache can be cleared manually by deleting `./.cache/highcharts-excentis`. + ## Todo: * More examples diff --git a/highcharts_excentis/highcharts/highcharts.py b/highcharts_excentis/highcharts/highcharts.py index 45ec45d..67e53be 100644 --- a/highcharts_excentis/highcharts/highcharts.py +++ b/highcharts_excentis/highcharts/highcharts.py @@ -1,31 +1,43 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals, absolute_import -from future.standard_library import install_aliases -install_aliases() -from past.builtins import basestring +from __future__ import absolute_import, unicode_literals -from jinja2 import Environment, PackageLoader +from future.standard_library import install_aliases -import json, uuid -import re +install_aliases() import datetime -import urllib.request, urllib.error, urllib.parse import html -from .options import BaseOptions, ChartOptions, ColorAxisOptions, \ - ColorsOptions, CreditsOptions, DrilldownOptions, ExportingOptions, \ - GlobalOptions, LabelsOptions, LangOptions, \ - LegendOptions, LoadingOptions, NavigationOptions, PaneOptions, \ - PlotOptions, SeriesData, SubtitleOptions, TitleOptions, \ - TooltipOptions, xAxisOptions, yAxisOptions, zAxisOptions, MultiAxis +import json +import re +import urllib.error +import urllib.parse +import urllib.request +import uuid +import diskcache as dc +from jinja2 import Environment, PackageLoader +from past.builtins import basestring + +from .common import (ArrayObject, ColorObject, CommonObject, CSSObject, + Formatter, JSfunction, Levels, RawJavaScriptText, + SVGObject) from .highchart_types import Series, SeriesOptions -from .common import Levels, Formatter, CSSObject, SVGObject, JSfunction, RawJavaScriptText, \ - CommonObject, ArrayObject, ColorObject +from .options import (BaseOptions, ChartOptions, ColorAxisOptions, + ColorsOptions, CreditsOptions, DrilldownOptions, + ExportingOptions, GlobalOptions, LabelsOptions, + LangOptions, LegendOptions, LoadingOptions, MultiAxis, + NavigationOptions, PaneOptions, PlotOptions, SeriesData, + SubtitleOptions, TitleOptions, TooltipOptions, + xAxisOptions, yAxisOptions, zAxisOptions) CONTENT_FILENAME = "content.html" PAGE_FILENAME = "page.html" +# 50MB limit, to ensure LRU evection +_CACHE = dc.Cache('./.cache/highcharts-excentis', size_limit=50 * 1024**2) + +_CACHE.cull() # Clean up expired entries + pl = PackageLoader('highcharts_excentis.highcharts', 'templates') jinja2_env = Environment(lstrip_blocks=True, trim_blocks=True, loader=pl) @@ -347,15 +359,36 @@ def buildhtmlheader(self): opener = urllib.request.build_opener() opener.addheaders = [('User-Agent', 'Mozilla/5.0')] + + def _get_or_download(url: str, + ttl=60 * 60 * 24 * 7): # Default: 7 days + """Perform caching for Highcarts JS libraries to avoid HTTP rate limit.""" + if url in _CACHE: + print(f"Highcharts: Cache hit, pulling cached URL {url} ") + return _CACHE[url] + else: + try: + # your actual HTTP request + print(f"Highcharts: Downloading and caching url {url}") + content = opener.open(url).read().decode( + 'utf-8').replace('\n', '') + _CACHE.set(url, content, expire=ttl) # 1 week TTL + return content + except urllib.error.HTTPError as e: + print( + f"Highcharts: HTTP Error {e.code} for URL: {url}") + except urllib.error.URLError as e: + print( + f"Highcharts: URL Error: {e.reason} for URL: {url}" + ) + self.header_css = [ - '' for source in self.CSSsource + '' + for source in self.CSSsource ] self.header_js = [ - '' for source in self.JSsource ] diff --git a/pyproject.toml b/pyproject.toml index fcebc57..8592cb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,8 @@ maintainers = [ requires-python = ">=3.4" dependencies = [ 'future', - 'Jinja2', # for templates + 'diskcache', # to avoid http rate limit + 'Jinja2' # for templates ] keywords = [ 'python',