From 22a70b30a90af5e75f194a98fdca41abaae35cef Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:42:44 +1000 Subject: [PATCH 1/7] Update dependencies --- Manifest.toml | 2 +- Project.toml | 1 + src/ReefGuideAPI.jl | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Manifest.toml b/Manifest.toml index 1514fd0..8a09058 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.5" manifest_format = "2.0" -project_hash = "d0ffae0ac62d29f3001ef8e43624c0daaa93de19" +project_hash = "23aa2c97b270f292dd6c68842031110f00d039d9" [[deps.AbstractFFTs]] deps = ["LinearAlgebra"] diff --git a/Project.toml b/Project.toml index 0a633f9..00cf1ef 100644 --- a/Project.toml +++ b/Project.toml @@ -35,6 +35,7 @@ Oxygen = "df9a0d86-3283-4920-82dc-4555fc0d1d8b" Proj = "c94c279d-25a6-4763-9509-64d165bea63e" ProtoBuf = "3349acd9-ac6a-5e09-bcdb-63829b23a429" Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index ff2ba1d..4eec399 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -5,6 +5,8 @@ using Glob, TOML +using Serialization + using DataFrames using OrderedCollections using Memoization From 54aa493b4b881e4f7cbb68e8c86d5749bce4896d Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:43:04 +1000 Subject: [PATCH 2/7] Switch bare print statements to `@info` --- src/ReefGuideAPI.jl | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 4eec399..192afab 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -141,22 +141,22 @@ function tile_size(config)::Tuple end function start_server(config_path) - println("Launching server...please wait") + @info "Launching server... please wait" - println("Parsing configuration from $(config_path)...") + @info "Parsing configuration from $(config_path)..." config = TOML.parsefile(config_path) - println("Successfully parsed configuration.") + @info "Successfully parsed configuration." - println("Setting up region routes...") + @info "Setting up region routes..." setup_region_routes(config) - println("Completed region routes setup.") + @info "Completed region routes setup." - println("Setting up tile routes...") + @info "Setting up tile routes..." setup_tile_routes() - println("Completed tile routes setup.") + @info "Completed tile routes setup." - println("Initialisation complete, starting server on port 8000.") - println("Starting with $(Threads.nthreads()) threads...") + @info "Initialisation complete, starting server on port 8000." + @info "Starting with $(Threads.nthreads()) threads..." if Threads.nthreads() > 1 serveparallel(host="0.0.0.0", port=8000) else From a42b2dc5637127715a10c365b2b0145f6a8a8877 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:44:43 +1000 Subject: [PATCH 3/7] Indicate updated config --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b074f0f..536ff3f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ These are currently the files/data created in Step/Script 1a in https://github.c PREPPED_DATA_DIR = "C:/some_path_to_data/MPA/" [server_config] -CACHE_DIR = "" +TIFF_CACHE_DIR = "" +REGIONAL_CACHE_DIR = "" DEBUG_MODE = "false" # Optional, disables file caching and displays debug logs COG_THREADS = "2" # Optional, Number of threads to use when creating COGs (defaults to 1) TILE_SIZE = "256" # Optional, tile block size to use (defaults to 256) From 2163c2a5eb8f58012d5267561fac364a84c89394 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:45:17 +1000 Subject: [PATCH 4/7] Add docstrings --- src/ReefGuideAPI.jl | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 192afab..4c027bb 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -104,13 +104,18 @@ function setup_regional_data(reef_data_path::String) return REGIONAL_DATA end -function _cache_location(config) +""" + _cache_location(config::Dict)::String + +Retrieve cache location for geotiffs. +""" +function _cache_location(config::Dict)::String cache_loc = try in_debug = "DEBUG_MODE" in config["server_config"] if in_debug && lowercase(config["server_config"]["DEBUG_MODE"]) == "true" mktempdir() else - config["server_config"]["CACHE_DIR"] + config["server_config"]["TIFF_CACHE_DIR"] end catch mktempdir() @@ -119,7 +124,12 @@ function _cache_location(config) return cache_loc end -function n_gdal_threads(config)::String +""" + n_gdal_threads(config::Dict)::String + +Retrieve the configured number of threads to use when writing COGs with GDAL. +""" +function n_gdal_threads(config::Dict)::String n_cog_threads = try config["server_config"]["COG_THREADS"] catch @@ -129,7 +139,12 @@ function n_gdal_threads(config)::String return n_cog_threads end -function tile_size(config)::Tuple +""" + tile_size(config::Dict)::Tuple + +Retrieve the configured size of map tiles in pixels (width and height / lon and lat). +""" +function tile_size(config::Dict)::Tuple tile_dims = try res = parse(Int, config["server_config"]["TILE_SIZE"]) (res, res) From cd74b3dce4fb728672f01347924544cab8d0e974 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:45:25 +1000 Subject: [PATCH 5/7] Add warmup function --- src/ReefGuideAPI.jl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 4c027bb..4a7bc38 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -155,6 +155,16 @@ function tile_size(config::Dict)::Tuple return tile_dims end +""" + warmup_cache(config_path::String) + +Invokes warm up of regional data cache to reduce later spin up times. +""" +function warmup_cache(config_path::String) + config = TOML.parsefile(config_path) + setup_regional_data(config) +end + function start_server(config_path) @info "Launching server... please wait" From 64776db042c7f35a5a54f0f142990620662522ec Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:47:51 +1000 Subject: [PATCH 6/7] Serialize data to disk-based cache --- src/ReefGuideAPI.jl | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 4a7bc38..1d8446e 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -39,18 +39,38 @@ function get_regions() return regions end +""" + setup_regional_data(config::Dict) + +Load regional data to act as an in-memory cache. + +# Arguments +- `config` : Configuration settings, typically loaded from a TOML file. +- `reef_data_path` : Path to pre-prepared reef data + +# Returns +OrderedDict of `RegionalCriteria` for each region. +""" function setup_regional_data(config::Dict) - return setup_regional_data(config["prepped_data"]["PREPPED_DATA_DIR"]) -end -function setup_regional_data(reef_data_path::String) if @isdefined(REGIONAL_DATA) @debug "Using previously generated regional data store." sleep(1) # Pause so message is noticeably visible return REGIONAL_DATA end + # Check disk-based store + reg_cache_dir = config["server_config"]["REGIONAL_CACHE_DIR"] + reg_cache_fn = joinpath(reg_cache_dir, "regional_cache.dat") + if isfile(reg_cache_fn) + @debug "Loading regional data cache from disk" + @eval const REGIONAL_DATA = deserialize($(reg_cache_fn)) + return REGIONAL_DATA + end + @debug "Setting up regional data store..." + reef_data_path = config["prepped_data"]["PREPPED_DATA_DIR"] + regional_assessment_data = OrderedDict{String, RegionalCriteria}() for reg in get_regions() data_paths = String[] @@ -98,6 +118,10 @@ function setup_regional_data(reef_data_path::String) ) end + # Store cache on disk to avoid excessive cold startup times + @debug "Saving regional data cache to disk" + serialize(reg_cache_fn, regional_assessment_data) + # Remember, `@eval` runs in global scope. @eval const REGIONAL_DATA = $(regional_assessment_data) From 175e2853ad9d59c1bfeee4ddff0a0ae38322740a Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Sun, 22 Sep 2024 12:48:12 +1000 Subject: [PATCH 7/7] Use more informative error message Remove use of bare `error()` --- src/ReefGuideAPI.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ReefGuideAPI.jl b/src/ReefGuideAPI.jl index 1d8446e..b453fc3 100644 --- a/src/ReefGuideAPI.jl +++ b/src/ReefGuideAPI.jl @@ -96,7 +96,8 @@ function setup_regional_data(config::Dict) elseif occursin("flat", string(dp)) flat_table = GeoParquet.read(parq_file) else - error("Unknown lookup found: $(parq_file)") + msg = "Unknown lookup found: $(parq_file). Must be 'slope' or 'flat'" + throw(ArgumentError(msg)) end end end