diff --git a/.github/decrypt_secret.sh b/.github/decrypt_secret.sh
deleted file mode 100755
index 60176fdf..00000000
--- a/.github/decrypt_secret.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/bin/sh
-
-# Decrypt the file
-
-mkdir -p $HOME/.config/earthengine/ndef/
-
-mkdir -p /home/rstudio/.config/earthengine/ndef/
-
-mkdir -p /github/home/config/earthengine
-
-mkdir -p ./secrets
-
-# --batch to prevent interactive command
-# --yes to assume "yes" for questions
-
-
-# Decrypt ee credentials (currently decrypting to a bunch of places hoping that earth engine finds one)
-
-gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output $HOME/.config/earthengine/ndef/credentials ./scratch_code/credentials.gpg
-
-gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output /home/rstudio/.config/earthengine/ndef/credentials ./scratch_code/credentials.gpg
-
-gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output ~/.config/earthengine/credentials ./scratch_code/credentials.gpg
-
-gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output /github/home/config/earthengine/credentials ./scratch_code/credentials.gpg
-
-
-# Decrypt google drive credentials
-gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output $HOME/.config/earthengine/ndef/20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner@gmail.com ./scratch_code/20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner@gmail.com.gpg
-
-gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output /home/rstudio/.config/earthengine/ndef/20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner@gmail.com ./scratch_code/20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner@gmail.com.gpg
-
-
-# Decrypt google drive credentials json creds
-
-#Note: directly encrypting hte .json provided by Google failed. Instead, I loaded it into R, re-saved it, THEN encrypted the new version.
-
-echo " json token "
-
-gpg --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
---output ./secrets/ee-wilsonlab-emma-ef416058504a.json ./scratch_code/ee-wilsonlab-emma-ef416058504a.json.gpg
-
-#Encrypting
-
- #gpg --output your-json-token.json.gpg --symmetric your-json-token.json
-
diff --git a/.github/decrypt_secret_gd.sh b/.github/decrypt_secret_gd.sh
deleted file mode 100644
index 77c91518..00000000
--- a/.github/decrypt_secret_gd.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Decrypt the file
-
-
-# --batch to prevent interactive command
-# --yes to assume "yes" for questions
-
-
-# Decrypt google drive credentials (newer version)
-gpg --quiet --batch --yes --decrypt --passphrase="$GD_SECRET" \
---output /home/rstudio/.config/earthengine/ndef/maitner-f590bfc7be54.json ./scratch_code/maitner-f590bfc7be54.json.gpg
-
-gpg --quiet --batch --yes --decrypt --passphrase="$GD_SECRET" \
---output $HOME/.config/earthengine/ndef/maitner-f590bfc7be54.json ./scratch_code/maitner-f590bfc7be54.json.gpg
-
-gpg --quiet --batch --yes --decrypt --passphrase="$GD_SECRET" \
---output $HOME/.config/earthengine/ndef/maitner-f590bfc7be54.json ./scratch_code/maitner-f590bfc7be54.json.gpg
-
diff --git a/.github/workflow-logs/failure-20260303-122355.log b/.github/workflow-logs/failure-20260303-122355.log
new file mode 100644
index 00000000..040f20ce
--- /dev/null
+++ b/.github/workflow-logs/failure-20260303-122355.log
@@ -0,0 +1,2 @@
+Error: unexpected symbol in "options(error = function() { traceback(2) quit"
+Execution halted
diff --git a/.github/workflows/targets.yaml b/.github/workflows/targets.yaml
index 314a3032..c167f288 100644
--- a/.github/workflows/targets.yaml
+++ b/.github/workflows/targets.yaml
@@ -30,289 +30,194 @@ on:
branches:
- main
- master
+ - dev-adam-appeears
- dev-jiyeon
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
schedule:
- # - cron: '1 0 * * 0' # runs every Sunday at 00:01
- - cron: '0 0 * * *' # runs every day at midnight
+ - cron: '0 0 * * 0' # runs every Sunday at midnight UTC
+ # - cron: '0 0 * * *' # runs every day at midnight
# - cron: '0 */6 * * *' # run every 6th hour
name: targets
jobs:
targets:
- runs-on: ubuntu-24.04 #ubuntu-22.04
+ runs-on: ubuntu-latest
container: adamwilsonlab/emma:latest
+ permissions:
+ contents: write
+ pull-requests: write
+ issues: write
env:
GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
- RENV_PATHS_ROOT: ~/.local/share/renv
+ R_KEEP_PKG_SOURCE: yes
+ EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
+ EARTHDATA_USER: ${{ secrets.EARTHDATA_USER }}
ACTIONS_RUNNER_DEBUG: true
- PYTHONPATH: /opt/conda/envs/r-reticulate/bin/python
- RETICULATE_PYTHON: /opt/conda/envs/r-reticulate/bin/python
- # GARGLE_AUTH_FILE: secrets/ee-wilsonlab-emma-ef416058504a.json
- GOOGLE_APPLICATION_CREDENTIALS: secrets/ee-wilsonlab-emma-ef416058504a.json
steps:
- uses: actions/checkout@v2
with:
- ref: main
+ ref: dev-adam-appeears
lfs: true
- name: Whitelist directory
run: git config --global --add safe.directory /__w/emma_envdata/emma_envdata
- - name: Checkout LFS objects
- run: git lfs checkout
- continue-on-error: true
- # - name: List files in scratch_code
- # run: ls -lh scratch_code/
-# - name: Setup Python
-# uses: actions/setup-python@v2
-# with:
-# python-version: '3.x'
-# - name: Install Python Dependencies
-# run: |
-# pip install earthengine-api
-# pip install -r requirements.txt # If you have other dependencies listed in a requirements file
-# - name: Verify Python Installation
-# run: |
-# python -m pip show earthengine-api
-# python -c "import ee; print(ee.__version__)"
-# # - name: Install Linux system dependencies
-# # if: runner.os == 'Linux'
-# # run: |
-# # sudo apt-get update
- - name: Install Google Cloud SDK in container
+ - name: Install system deps for keyring
run: |
apt-get update
- apt-get install -y curl apt-transport-https ca-certificates gnupg
- echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" \
- | tee /etc/apt/sources.list.d/google-cloud-sdk.list
- curl https://packages.cloud.google.com/apt/doc/apt-key.gpg \
- | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
- apt-get update
- apt-get install -y google-cloud-cli=492.0.0-0
- - name: DECRYPT rgee SECRETS
- run: ./.github/decrypt_secret.sh
- env:
- RGEE_SECRET: ${{ secrets.RGEE_SECRET }}
- continue-on-error: true
- # - name: List files in secrets
- # run: ls -lh secrets/
-# - name: Check git settings
-# run:
-# git config --get http.postBuffer
-# continue-on-error: true
-
- # - name: Display Service Account JSON
- # run: |
- # echo "=== Service Account JSON ==="
- # cat secrets/ee-wilsonlab-emma-ef416058504a.json
- # echo "============================"
-
- - name: Install Earth Engine API 1.5.4 in r-reticulate env
- run: |
- /opt/conda/envs/r-reticulate/bin/pip install --upgrade earthengine-api==0.1.370 #0.1.381
- # conda install -n r-reticulate -y -c conda-forge earthengine-api=1.5.4
-
- - name: Verify EE API version
- run: |
- /opt/conda/envs/r-reticulate/bin/pip show earthengine-api
-
+ apt-get install -y libsecret-1-0 libsecret-1-dev
- name: Install R Package Dependencies
run: |-
Rscript -e "questionr::qscan(list.files(pattern='*.R',recursive=T), load = TRUE, detail = TRUE)" # this scans all scripts and installs any needed packages
- # - name: Upgrade rgee to latest from GitHub
- # run: |
- # Rscript -e "if (!requireNamespace('devtools', quietly = TRUE)) install.packages('devtools')"
- # Rscript -e 'devtools::install_github("r-spatial/rgee", ref="v.1.1.5")'
-
- # - name: Install rgee v1.1.5 from CRAN Archive
- # run: |
- # Rscript -e 'if (!requireNamespace("remotes", quietly=TRUE)) install.packages("remotes", repos="https://cloud.r-project.org")'
- # Rscript -e 'remotes::install_version("rgee", version="1.1.5", repos="https://cloud.r-project.org")'
-
- - name: Install rgee from bmaitner/rgee
- run: Rscript -e 'devtools::install_github(repo = "bmaitner/rgee", ref = "noninteractive_auth")'
- name: Parse _targets.R
run: Rscript -e "parse('_targets.R')"
- - name: Cache packages
+ - name: Cache targets store
uses: actions/cache@v3
with:
- path: ${{ env.RENV_PATHS_ROOT }}
- key: ${{ runner.os }}-renv-${{ hashFiles('**/renv.lock') }}
- restore-keys: ${{ runner.os }}-renv-
-
- - name: Authenticate to GCP
+ path: _targets/
+ key: targets-${{ hashFiles('_targets.R', 'R/**') }}
+ restore-keys: targets-
+ - name: Run targets pipeline
run: |
- # write the JSON secret to disk
- # activate the service account (no --scopes flag)
- gcloud auth activate-service-account \
- --key-file="secrets/ee-wilsonlab-emma-ef416058504a.json" \
- --project=ee-wilsonlab-emma \
- --quiet
-
- - name: Add r-reticulate env to PATH
- run: echo "/opt/conda/envs/r-reticulate/bin" >> $GITHUB_PATH
-
- # - name: Authenticate Earth Engine
- # run: earthengine authenticate \
- # --service_account \
- # --quiet \
- # #--key_file=${{ runner.temp }}/gee-key.json
-
- # - name: Authenticate to GCP (ADC with scopes)
- # run: |
- # gcloud auth application-default login \
- # --project=ee-wilsonlab-emma \
- # --scopes=https://www.googleapis.com/auth/cloud-platform \
- # --quiet
-
- # - name: Restore packages
- # shell: Rscript {0}
- # run: |
- # if (!requireNamespace("renv", quietly = TRUE)) install.packages("renv")
- # renv::restore()
-
- - name: Check if previous runs exists
- id: runs-exist
- run: git ls-remote --exit-code --heads origin targets-runs
- continue-on-error: true
-
- - name: Checkout previous run
- if: steps.runs-exist.outcome == 'success'
- uses: actions/checkout@v2
- with:
- ref: targets-runs
- fetch-depth: 1
- path: .targets-runs
- lfs: 'true' #attempting to fix issue with target-committed lfs files not being treated as lfs
-
- - name: Restore output files from the previous run
- if: steps.runs-exist.outcome == 'success'
+ Rscript -e "options(error = function() { traceback(2) quit(status = 1)}) targets::tar_make()" 2>&1 | tee targets-output.log
+ # Rscript -e "targets::tar_make()" 2>&1 | tee targets-output.log
+ shell: bash
+ - name: Export final data products to releases
+ if: github.ref == 'refs/heads/main'
run: |
- for (dest in scan(".targets-runs/.targets-files", what = character())) {
- source <- file.path(".targets-runs", dest)
- if (!file.exists(dirname(dest))) dir.create(dirname(dest), recursive = TRUE)
- if (file.exists(source)) file.rename(source, dest)
+ # Load final targets and export to data/releases/
+ targets::tar_load_everything()
+
+ # Create releases directory
+ dir.create("data/releases", recursive = TRUE, showWarnings = FALSE)
+
+ # Export domain as gpkg (example - adjust to your actual target names)
+ if (exists("domain")) {
+ sf::st_write(
+ sf::st_as_sf(domain),
+ "data/releases/domain.gpkg",
+ delete_dsn = TRUE
+ )
}
+
+ # Add other final outputs here as needed
+ # Example: write.csv(final_data, "data/releases/final_data.csv")
+
shell: Rscript {0}
- # - name: Find credential JSON
- # run: |
- # echo "Searching for ee-wilsonlab-emma-ef416058504a.json ..."
- # find / -name 'ee-wilsonlab-emma-ef416058504a.json' 2>/dev/null || true
-# - name: Run targets pipeline
-# run: |
-# Sys.setenv(HOME="/home/rstudio")
-# cmdstanr::set_cmdstan_path("/home/rstudio/.cmdstanr/cmdstan-2.28.1")
-# cmdstanr::check_cmdstan_toolchain()
-# #cmdstanr::install_cmdstan()
-# targets::tar_make()
-# shell: Rscript {0}
-# - name: Run targets pt 1
-# run: |
-# Sys.setenv(HOME="/home/rstudio")
-# cmdstanr::set_cmdstan_path("/home/rstudio/.cmdstanr/cmdstan-2.28.1")
-# cmdstanr::check_cmdstan_toolchain()
-# #cmdstanr::install_cmdstan()
-# shell: Rscript {0}
- - name: Verify credentials
- run: |
- ./.github/decrypt_secret.sh
-# ls ~/.config/earthengine
-# /usr/bin/earthengine -h
-# echo " main dir"
-# ls
-# echo "scratch dir"
-# ls scratch_code
-# echo "secrets dir"
-# ls secrets
- env:
- RGEE_SECRET: ${{ secrets.RGEE_SECRET }}
- GD_SECRET: ${{ secrets.GD_SECRET }}
continue-on-error: true
- # - name: Install custom rgee
- # run: |
- # # added below on april 9 from https://github.com/r-spatial/rgee/issues/353#issuecomment-1983765552
- # library(reticulate)
- # py_config() # see the name of your conda (python) environment, in my case "r-reticulate"
- # reticulate::py_install('earthengine-api==0.1.370', envname='r-reticulate')
- # # Check the installation of "earthengine-api" with
- # py_list_packages()
- # pyl <- py_list_packages()
- # pyl[pyl$package == "earthengine-api", ]
- # # check python version with
- # py_run_string("import sys; print(sys.version)")
- # devtools::install_github(repo = "bmaitner/rgee", ref = "noninteractive_auth")
- # - name: Install custom rgee
- # run: |
- # Rscript -e "library(reticulate)"
- # Rscript -e "reticulate::py_install('earthengine-api==0.1.370', envname='r-reticulate')"
- # Rscript -e "pyl <- py_list_packages(); print(pyl[pyl$package == 'earthengine-api', ])"
- # Rscript -e "reticulate::py_run_string('import sys; print(sys.version)')"
- # Rscript -e "devtools::install_github(repo = 'bmaitner/rgee', ref = 'noninteractive_auth')"
-# shell: Rscript {0}
- # - name: Print environment
- # run: env
- # - name: Print R session info
- # run: Rscript -e 'sessionInfo()'
- # - name: βSet up gcloud CLIβ
- # uses: google-github-actions/setup-gcloud@v1
- # with:
- # project_id: ${{ secrets.GCP_PROJECT }}
- # service_account_key: ${{ secrets.GCP_SA_KEY }}
- - name: Run targets pt 2
+
+ - name: Upload to GitHub Releases
+ if: github.ref == 'refs/heads/main'
+ uses: softprops/action-gh-release@v1
+ with:
+ tag_name: latest
+ files: |
+ data/releases/*
+ body: |
+ Latest data products from targets pipeline
+ Updated: ${{ github.event.head_commit.timestamp }}
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ - name: Install GitHub CLI
+ if: failure() && github.event_name == 'push'
run: |
- #reticulate::use_python('/usr/bin/python3')
- #rgee::ee_set_pyenv('/usr/bin/python3','r-reticulate', confirm = F)
- #targets::tar_destroy(destroy = "all",ask = FALSE) #uncomment to reset
- targets::tar_make()
- shell: Rscript {0}
- - name: Identify files that the targets pipeline produced
- run: git ls-files -mo --exclude=renv > .targets-files
- - name: Create the runs branch if it does not already exist
- if: steps.runs-exist.outcome != 'success'
- run: git checkout --orphan targets-runs
- - name: Put the worktree in the runs branch if the latter already exists
- if: steps.runs-exist.outcome == 'success'
+ apt-get update
+ apt-get install -y gh
+
+ - name: Create review PR on failure
+ if: failure() && github.event_name == 'push'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
- rm -r .git
- mv .targets-runs/.git .
- rm -r .targets-runs
+ # Create error log if it doesn't exist
+ if [ ! -f targets-output.log ]; then
+ echo "No targets-output.log file found. Pipeline may have failed during execution." > targets-output.log
+ fi
+ # Get last 100 lines of error
+ tail -100 targets-output.log > error-summary.txt
+
+ # Create branch
+ BRANCH="auto-fix-$(date +%s)"
+ git config user.name "github-actions[bot]"
+ git config user.email "github-actions[bot]@users.noreply.github.com"
+ git checkout -b "$BRANCH"
+
+ # Add error log
+ mkdir -p .github/workflow-logs
+ cp targets-output.log .github/workflow-logs/failure-$(date +%Y%m%d-%H%M%S).log
+ git add .github/workflow-logs/
+ git commit -m "Add workflow failure log for review" || true
+ git push origin "$BRANCH"
+
+ # Create PR (Copilot will auto-review it)
+ gh pr create \
+ --title "π¨ Targets workflow failed - Review needed" \
+ --body "## Workflow Failure
- - name: update md
- run: |
- webshot::install_phantomjs()
- knitr::knit("README.Rmd")
- shell: Rscript {0}
- - name: Upload latest run
- run: |
- git config --local user.name "GitHub Actions"
- git config --local user.email "actions@github.com"
- rm -r .gitignore .github/workflows
- git lfs track _targets/objects/* #use git LFS to track the targets output (larger files)
- git lfs track data/raw_data/*
- git lfs track data/* #use git LFS to track the data output (larger files)
- git lfs track "raw_data/**" #use git LFS to track raw_data output (larger files)
- git lfs track "data/**"
- git lfs track _targets/metadata/* #track metadata
- git lfs track *.tif
- git add .gitattributes
- git add --all -- ':!renv' ':!*json'
- for file in $(git ls-files ':!*.json' -mo --exclude=renv)
- do
- git add -f $file
- done
- git commit -am "Run pipeline"
- git push origin targets-runs
-# Move the loop below up two lines (after git add) to commit large files.
- - name: Prepare failure artifact
- if: failure()
- run: rm -rf .git .github .targets-files .targets-runs
+ The targets pipeline failed. GitHub Copilot will review this PR.
+
+ ### Error Summary
+ \`\`\`
+ $(cat error-summary.txt)
+ \`\`\`
+
+ ### Full Log
+ See \`.github/workflow-logs/\` for complete output.
+
+ **Next steps:**
+ 1. Review Copilot's suggestions in the PR
+ 2. Apply fixes
+ 3. Close this PR once resolved" \
+ --base main \
+ --head "$BRANCH" \
+ --label "bug"
+
+ # Request Copilot review
+ gh pr comment $BRANCH --body "@github-copilot review"
+
+ # Request Copilot coding agent to fix the issue
+ gh pr comment $BRANCH --body "@github-copilot-agent Please analyze the error log and create fixes for this targets pipeline failure"
+
- name: Post failure artifact
if: failure()
uses: actions/upload-artifact@main
with:
- name: ${{ runner.os }}-r${{ matrix.config.r }}-results
+ name: ${{ runner.os }}-results
path: .
+
+ - name: Prepare failure artifact
+ if: failure()
+ run: rm -rf .git .github
+
+ - name: Comment on existing PR with status
+ if: always() && github.event_name == 'pull_request'
+ env:
+ GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: |
+ STATUS="${{ steps.targets.outcome }}"
+ if [ "$STATUS" = "success" ]; then
+ EMOJI="β
"
+ MESSAGE="Targets workflow completed successfully!"
+ else
+ EMOJI="β"
+ MESSAGE="Targets workflow failed. See logs below."
+ fi
+
+ if [ ! -f targets-output.log ]; then
+ echo "No targets-output.log file found. Pipeline completed without explicit log output." > targets-output.log
+ fi
+ gh pr comment ${{ github.event.pull_request.number }} --body "## $EMOJI Targets Workflow Result
+
+ $MESSAGE
+
+
+ Show last 50 lines of output
+
+ \`\`\`
+ $(tail -50 targets-output.log)
+ \`\`\`
+ "
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 889aa851..fc6328d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,53 +1,76 @@
-# History files
+# R session files
.Rhistory
.Rapp.history
-# Session Data files
.RData
-# User-specific files
.Ruserdata
-# Example code in package build process
+
+# RStudio files
+.Rproj.user/
+
+# Targets workflow - track meta/meta for pipeline state
+_targets/objects/
+_targets/user/
+_targets/workspaces/
+
+# Data directories
+data/temp/
+data/other_data/
+data/raw_data/
+data/manual_download/*.gpkg
+data/manual_download/NVM*
+data/raw*
+data/.tar_cache/
+data/target_outputs/
+
+# Terra auxiliary files (from terra_preserve_metadata)
+*.tif.aux.xml
+*.tif.zip
+*.ovr
+
+# Secrets & credentials
+.Renviron
+.httr-oauth
+~/.boxr-oauth
+.secrets
+secrets/
+debugging_auth.R
+
+# JSON files - specific patterns only
+secrets/*.json
+scratch_code/*.json
+
+# AppEEARS/NASA downloads
+*.hdf
+*.h5
+
+# Test scripts
+test_*.R
+
+# R package build
*-Ex.R
-# Output files from R CMD build
/*.tar.gz
-# Output files from R CMD check
/*.Rcheck/
-# RStudio files
-.Rproj.user/
-# produced vignettes
+
+# Vignettes
vignettes/*.html
vignettes/*.pdf
-# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3
-.httr-oauth
-# knitr and R markdown default cache directories
+
+# knitr and R markdown
*_cache/
/cache/
-# Temporary files created by R markdown
*.utf8.md
*.knit.md
-# R Environment Variables
-.Renviron
-# targets related workflow stuff
-~/.boxr-oauth
index.md
-index_files/*
-firemodel_predict
-#raw_data/
+index_files/
+
+# Output/legacy directories
processed_data/
-#data/*
-*.gpkg
-data/other_data/
-*/other_data/*
-.DS_Store
-#*/meta
-img/network.html
+firemodel_predict
+
+# Image outputs
+img/*_files
+img/*.html
img/network*
-scratch_code/*.json
-scratch_code/!*.json.gpg
-.secrets
-secrets
-secrets/
-secrets/*
-secrets/*.json
-secrets/!*.json.gpg
-debugging_auth.R
-*.json
+
+# OS files
+.DS_Store
diff --git a/.targets-files b/.targets-files
new file mode 100644
index 00000000..50cfbbde
--- /dev/null
+++ b/.targets-files
@@ -0,0 +1,2443 @@
+.targets-files
+.targets-runs/
+Rplots.pdf
+_targets/meta/.gitignore
+_targets/meta/meta
+_targets/meta/process
+_targets/meta/progress
+_targets/objects/alos
+_targets/objects/burn_date_to_last_burned_date
+_targets/objects/climate_chelsa
+_targets/objects/clouds_wilson
+_targets/objects/correct_fire_proj
+_targets/objects/correct_kndvi_proj
+_targets/objects/correct_ndvi_date_proj
+_targets/objects/correct_ndvi_proj
+_targets/objects/country
+_targets/objects/data
+_targets/objects/domain
+_targets/objects/elevation_nasadem
+_targets/objects/fire_dates_to_parquet
+_targets/objects/fire_doy_to_unix_date
+_targets/objects/fire_modis
+_targets/objects/group_data
+_targets/objects/kndvi_modis
+_targets/objects/landcover_za
+_targets/objects/model
+_targets/objects/model_fit
+_targets/objects/most_recent_fire_dates_to_parquet
+_targets/objects/ndvi_dates_modis
+_targets/objects/ndvi_modis
+_targets/objects/ndvi_relative_days_since_fire
+_targets/objects/ndvi_to_parquet
+_targets/objects/posterior_summary
+_targets/objects/precipitation_chelsa
+_targets/objects/projected_alos
+_targets/objects/projected_climate_chelsa
+_targets/objects/projected_clouds_wilson
+_targets/objects/projected_elevation_nasadem
+_targets/objects/projected_landcover_za
+_targets/objects/projected_precipitation_chelsa
+_targets/objects/raw_data
+_targets/objects/stan_data
+_targets/objects/template
+_targets/objects/test_tif
+_targets/objects/vegmap
+data/domain.gpkg
+data/model_data.csv
+data/processed_data/alos/alos_chili.tif
+data/processed_data/alos/alos_mtpi.tif
+data/processed_data/alos/alos_topographic_diversity.tif
+data/processed_data/alos/landforms.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_01_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_02_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_03_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_04_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_05_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_06_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_07_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_08_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_09_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_10_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_11_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_12_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_13_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_14_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_15_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_16_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_17_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_18_V1.2_clipped.tif
+data/processed_data/climate_chelsa/CHELSA_bio10_19_V1.2_clipped.tif
+data/processed_data/clouds_wilson/MODCF_interannualSD.tif
+data/processed_data/clouds_wilson/MODCF_intraannualSD.tif
+data/processed_data/clouds_wilson/MODCF_meanannual.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_01.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_02.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_03.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_04.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_05.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_06.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_07.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_08.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_09.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_10.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_11.tif
+data/processed_data/clouds_wilson/MODCF_monthlymean_12.tif
+data/processed_data/clouds_wilson/MODCF_seasonality_concentration.tif
+data/processed_data/clouds_wilson/MODCF_seasonality_rgb.tif
+data/processed_data/clouds_wilson/MODCF_seasonality_theta.tif
+data/processed_data/clouds_wilson/MODCF_seasonality_visct.tif
+data/processed_data/clouds_wilson/MODCF_spatialSD_1deg.tif
+data/processed_data/elevation_nasadem/nasadem.tif
+data/processed_data/fire_dates/2000_11_01.tif
+data/processed_data/fire_dates/2000_12_01.tif
+data/processed_data/fire_dates/2001_01_01.tif
+data/processed_data/fire_dates/2001_02_01.tif
+data/processed_data/fire_dates/2001_03_01.tif
+data/processed_data/fire_dates/2001_04_01.tif
+data/processed_data/fire_dates/2001_05_01.tif
+data/processed_data/fire_dates/2001_06_01.tif
+data/processed_data/fire_dates/2001_07_01.tif
+data/processed_data/fire_dates/2001_08_01.tif
+data/processed_data/fire_dates/2001_09_01.tif
+data/processed_data/fire_dates/2001_10_01.tif
+data/processed_data/fire_dates/2001_11_01.tif
+data/processed_data/fire_dates/2001_12_01.tif
+data/processed_data/fire_dates/2002_01_01.tif
+data/processed_data/fire_dates/2002_02_01.tif
+data/processed_data/fire_dates/2002_03_01.tif
+data/processed_data/fire_dates/2002_04_01.tif
+data/processed_data/fire_dates/2002_05_01.tif
+data/processed_data/fire_dates/2002_06_01.tif
+data/processed_data/fire_dates/2002_07_01.tif
+data/processed_data/fire_dates/2002_08_01.tif
+data/processed_data/fire_dates/2002_09_01.tif
+data/processed_data/fire_dates/2002_10_01.tif
+data/processed_data/fire_dates/2002_11_01.tif
+data/processed_data/fire_dates/2002_12_01.tif
+data/processed_data/fire_dates/2003_01_01.tif
+data/processed_data/fire_dates/2003_02_01.tif
+data/processed_data/fire_dates/2003_03_01.tif
+data/processed_data/fire_dates/2003_04_01.tif
+data/processed_data/fire_dates/2003_05_01.tif
+data/processed_data/fire_dates/2003_06_01.tif
+data/processed_data/fire_dates/2003_07_01.tif
+data/processed_data/fire_dates/2003_08_01.tif
+data/processed_data/fire_dates/2003_09_01.tif
+data/processed_data/fire_dates/2003_10_01.tif
+data/processed_data/fire_dates/2003_11_01.tif
+data/processed_data/fire_dates/2003_12_01.tif
+data/processed_data/fire_dates/2004_01_01.tif
+data/processed_data/fire_dates/2004_02_01.tif
+data/processed_data/fire_dates/2004_03_01.tif
+data/processed_data/fire_dates/2004_04_01.tif
+data/processed_data/fire_dates/2004_05_01.tif
+data/processed_data/fire_dates/2004_06_01.tif
+data/processed_data/fire_dates/2004_07_01.tif
+data/processed_data/fire_dates/2004_08_01.tif
+data/processed_data/fire_dates/2004_09_01.tif
+data/processed_data/fire_dates/2004_10_01.tif
+data/processed_data/fire_dates/2004_11_01.tif
+data/processed_data/fire_dates/2004_12_01.tif
+data/processed_data/fire_dates/2005_01_01.tif
+data/processed_data/fire_dates/2005_02_01.tif
+data/processed_data/fire_dates/2005_03_01.tif
+data/processed_data/fire_dates/2005_04_01.tif
+data/processed_data/fire_dates/2005_05_01.tif
+data/processed_data/fire_dates/2005_06_01.tif
+data/processed_data/fire_dates/2005_07_01.tif
+data/processed_data/fire_dates/2005_08_01.tif
+data/processed_data/fire_dates/2005_09_01.tif
+data/processed_data/fire_dates/2005_10_01.tif
+data/processed_data/fire_dates/2005_11_01.tif
+data/processed_data/fire_dates/2005_12_01.tif
+data/processed_data/fire_dates/2006_01_01.tif
+data/processed_data/fire_dates/2006_02_01.tif
+data/processed_data/fire_dates/2006_03_01.tif
+data/processed_data/fire_dates/2006_04_01.tif
+data/processed_data/fire_dates/2006_05_01.tif
+data/processed_data/fire_dates/2006_06_01.tif
+data/processed_data/fire_dates/2006_07_01.tif
+data/processed_data/fire_dates/2006_08_01.tif
+data/processed_data/fire_dates/2006_09_01.tif
+data/processed_data/fire_dates/2006_10_01.tif
+data/processed_data/fire_dates/2006_11_01.tif
+data/processed_data/fire_dates/2006_12_01.tif
+data/processed_data/fire_dates/2007_01_01.tif
+data/processed_data/fire_dates/2007_02_01.tif
+data/processed_data/fire_dates/2007_03_01.tif
+data/processed_data/fire_dates/2007_04_01.tif
+data/processed_data/fire_dates/2007_05_01.tif
+data/processed_data/fire_dates/2007_06_01.tif
+data/processed_data/fire_dates/2007_07_01.tif
+data/processed_data/fire_dates/2007_08_01.tif
+data/processed_data/fire_dates/2007_09_01.tif
+data/processed_data/fire_dates/2007_10_01.tif
+data/processed_data/fire_dates/2007_11_01.tif
+data/processed_data/fire_dates/2007_12_01.tif
+data/processed_data/fire_dates/2008_01_01.tif
+data/processed_data/fire_dates/2008_02_01.tif
+data/processed_data/fire_dates/2008_03_01.tif
+data/processed_data/fire_dates/2008_04_01.tif
+data/processed_data/fire_dates/2008_05_01.tif
+data/processed_data/fire_dates/2008_06_01.tif
+data/processed_data/fire_dates/2008_07_01.tif
+data/processed_data/fire_dates/2008_08_01.tif
+data/processed_data/fire_dates/2008_09_01.tif
+data/processed_data/fire_dates/2008_10_01.tif
+data/processed_data/fire_dates/2008_11_01.tif
+data/processed_data/fire_dates/2008_12_01.tif
+data/processed_data/fire_dates/2009_01_01.tif
+data/processed_data/fire_dates/2009_02_01.tif
+data/processed_data/fire_dates/2009_03_01.tif
+data/processed_data/fire_dates/2009_04_01.tif
+data/processed_data/fire_dates/2009_05_01.tif
+data/processed_data/fire_dates/2009_06_01.tif
+data/processed_data/fire_dates/2009_07_01.tif
+data/processed_data/fire_dates/2009_08_01.tif
+data/processed_data/fire_dates/2009_09_01.tif
+data/processed_data/fire_dates/2009_10_01.tif
+data/processed_data/fire_dates/2009_11_01.tif
+data/processed_data/fire_dates/2009_12_01.tif
+data/processed_data/fire_dates/2010_01_01.tif
+data/processed_data/fire_dates/2010_02_01.tif
+data/processed_data/fire_dates/2010_03_01.tif
+data/processed_data/fire_dates/2010_04_01.tif
+data/processed_data/fire_dates/2010_05_01.tif
+data/processed_data/fire_dates/2010_06_01.tif
+data/processed_data/fire_dates/2010_07_01.tif
+data/processed_data/fire_dates/2010_08_01.tif
+data/processed_data/fire_dates/2010_09_01.tif
+data/processed_data/fire_dates/2010_10_01.tif
+data/processed_data/fire_dates/2010_11_01.tif
+data/processed_data/fire_dates/2010_12_01.tif
+data/processed_data/fire_dates/2011_01_01.tif
+data/processed_data/fire_dates/2011_02_01.tif
+data/processed_data/fire_dates/2011_03_01.tif
+data/processed_data/fire_dates/2011_04_01.tif
+data/processed_data/fire_dates/2011_05_01.tif
+data/processed_data/fire_dates/2011_06_01.tif
+data/processed_data/fire_dates/2011_07_01.tif
+data/processed_data/fire_dates/2011_08_01.tif
+data/processed_data/fire_dates/2011_09_01.tif
+data/processed_data/fire_dates/2011_10_01.tif
+data/processed_data/fire_dates/2011_11_01.tif
+data/processed_data/fire_dates/2011_12_01.tif
+data/processed_data/fire_dates/2012_01_01.tif
+data/processed_data/fire_dates/2012_02_01.tif
+data/processed_data/fire_dates/2012_03_01.tif
+data/processed_data/fire_dates/2012_04_01.tif
+data/processed_data/fire_dates/2012_05_01.tif
+data/processed_data/fire_dates/2012_06_01.tif
+data/processed_data/fire_dates/2012_07_01.tif
+data/processed_data/fire_dates/2012_08_01.tif
+data/processed_data/fire_dates/2012_09_01.tif
+data/processed_data/fire_dates/2012_10_01.tif
+data/processed_data/fire_dates/2012_11_01.tif
+data/processed_data/fire_dates/2012_12_01.tif
+data/processed_data/fire_dates/2013_01_01.tif
+data/processed_data/fire_dates/2013_02_01.tif
+data/processed_data/fire_dates/2013_03_01.tif
+data/processed_data/fire_dates/2013_04_01.tif
+data/processed_data/fire_dates/2013_05_01.tif
+data/processed_data/fire_dates/2013_06_01.tif
+data/processed_data/fire_dates/2013_07_01.tif
+data/processed_data/fire_dates/2013_08_01.tif
+data/processed_data/fire_dates/2013_09_01.tif
+data/processed_data/fire_dates/2013_10_01.tif
+data/processed_data/fire_dates/2013_11_01.tif
+data/processed_data/fire_dates/2013_12_01.tif
+data/processed_data/fire_dates/2014_01_01.tif
+data/processed_data/fire_dates/2014_02_01.tif
+data/processed_data/fire_dates/2014_03_01.tif
+data/processed_data/fire_dates/2014_04_01.tif
+data/processed_data/fire_dates/2014_05_01.tif
+data/processed_data/fire_dates/2014_06_01.tif
+data/processed_data/fire_dates/2014_07_01.tif
+data/processed_data/fire_dates/2014_08_01.tif
+data/processed_data/fire_dates/2014_09_01.tif
+data/processed_data/fire_dates/2014_10_01.tif
+data/processed_data/fire_dates/2014_11_01.tif
+data/processed_data/fire_dates/2014_12_01.tif
+data/processed_data/fire_dates/2015_01_01.tif
+data/processed_data/fire_dates/2015_02_01.tif
+data/processed_data/fire_dates/2015_03_01.tif
+data/processed_data/fire_dates/2015_04_01.tif
+data/processed_data/fire_dates/2015_05_01.tif
+data/processed_data/fire_dates/2015_06_01.tif
+data/processed_data/fire_dates/2015_07_01.tif
+data/processed_data/fire_dates/2015_08_01.tif
+data/processed_data/fire_dates/2015_09_01.tif
+data/processed_data/fire_dates/2015_10_01.tif
+data/processed_data/fire_dates/2015_11_01.tif
+data/processed_data/fire_dates/2015_12_01.tif
+data/processed_data/fire_dates/2016_01_01.tif
+data/processed_data/fire_dates/2016_02_01.tif
+data/processed_data/fire_dates/2016_03_01.tif
+data/processed_data/fire_dates/2016_04_01.tif
+data/processed_data/fire_dates/2016_05_01.tif
+data/processed_data/fire_dates/2016_06_01.tif
+data/processed_data/fire_dates/2016_07_01.tif
+data/processed_data/fire_dates/2016_08_01.tif
+data/processed_data/fire_dates/2016_09_01.tif
+data/processed_data/fire_dates/2016_10_01.tif
+data/processed_data/fire_dates/2016_11_01.tif
+data/processed_data/fire_dates/2016_12_01.tif
+data/processed_data/fire_dates/2017_01_01.tif
+data/processed_data/fire_dates/2017_02_01.tif
+data/processed_data/fire_dates/2017_03_01.tif
+data/processed_data/fire_dates/2017_04_01.tif
+data/processed_data/fire_dates/2017_05_01.tif
+data/processed_data/fire_dates/2017_06_01.tif
+data/processed_data/fire_dates/2017_07_01.tif
+data/processed_data/fire_dates/2017_08_01.tif
+data/processed_data/fire_dates/2017_09_01.tif
+data/processed_data/fire_dates/2017_10_01.tif
+data/processed_data/fire_dates/2017_11_01.tif
+data/processed_data/fire_dates/2017_12_01.tif
+data/processed_data/fire_dates/2018_01_01.tif
+data/processed_data/fire_dates/2018_02_01.tif
+data/processed_data/fire_dates/2018_03_01.tif
+data/processed_data/fire_dates/2018_04_01.tif
+data/processed_data/fire_dates/2018_05_01.tif
+data/processed_data/fire_dates/2018_06_01.tif
+data/processed_data/fire_dates/2018_07_01.tif
+data/processed_data/fire_dates/2018_08_01.tif
+data/processed_data/fire_dates/2018_09_01.tif
+data/processed_data/fire_dates/2018_10_01.tif
+data/processed_data/fire_dates/2018_11_01.tif
+data/processed_data/fire_dates/2018_12_01.tif
+data/processed_data/fire_dates/2019_01_01.tif
+data/processed_data/fire_dates/2019_02_01.tif
+data/processed_data/fire_dates/2019_03_01.tif
+data/processed_data/fire_dates/2019_04_01.tif
+data/processed_data/fire_dates/2019_05_01.tif
+data/processed_data/fire_dates/2019_06_01.tif
+data/processed_data/fire_dates/2019_07_01.tif
+data/processed_data/fire_dates/2019_08_01.tif
+data/processed_data/fire_dates/2019_09_01.tif
+data/processed_data/fire_dates/2019_10_01.tif
+data/processed_data/fire_dates/2019_11_01.tif
+data/processed_data/fire_dates/2019_12_01.tif
+data/processed_data/fire_dates/2020_01_01.tif
+data/processed_data/fire_dates/2020_02_01.tif
+data/processed_data/fire_dates/2020_03_01.tif
+data/processed_data/fire_dates/2020_04_01.tif
+data/processed_data/fire_dates/2020_05_01.tif
+data/processed_data/fire_dates/2020_06_01.tif
+data/processed_data/fire_dates/2020_07_01.tif
+data/processed_data/fire_dates/2020_08_01.tif
+data/processed_data/fire_dates/2020_09_01.tif
+data/processed_data/fire_dates/2020_10_01.tif
+data/processed_data/fire_dates/2020_11_01.tif
+data/processed_data/fire_dates/2020_12_01.tif
+data/processed_data/fire_dates/2021_01_01.tif
+data/processed_data/fire_dates/2021_02_01.tif
+data/processed_data/fire_dates/2021_03_01.tif
+data/processed_data/fire_dates/2021_04_01.tif
+data/processed_data/fire_dates/2021_05_01.tif
+data/processed_data/fire_dates/2021_06_01.tif
+data/processed_data/fire_dates/2021_07_01.tif
+data/processed_data/fire_dates/2021_08_01.tif
+data/processed_data/fire_dates/2021_09_01.tif
+data/processed_data/fire_dates/2021_10_01.tif
+data/processed_data/fire_dates/2021_11_01.tif
+data/processed_data/fire_dates/2021_12_01.tif
+data/processed_data/landcover_za/SA_NLC_2020_GEO.tif
+data/processed_data/landcover_za/SA_NLC_2020_GEO.tif.vat.cpg
+data/processed_data/landcover_za/SA_NLC_2020_GEO.tif.vat.dbf
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11262.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11292.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11323.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11354.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11382.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11413.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11443.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11474.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11504.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11535.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11566.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11627.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11657.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11688.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11719.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11747.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11778.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11808.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11839.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11869.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11900.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11931.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11961.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/11992.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12022.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12053.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12084.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12112.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12143.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12173.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12204.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12234.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12265.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12296.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12326.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12387.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12418.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12449.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12478.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12509.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12539.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12570.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12600.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12631.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12662.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12723.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12753.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12784.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12815.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12843.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12874.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12904.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12935.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12965.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/12996.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13027.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13057.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13088.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13118.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13180.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13208.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13239.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13269.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13300.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13330.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13361.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13392.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13422.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13453.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13483.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13514.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13545.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13573.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13604.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13634.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13665.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13695.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13726.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13757.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13787.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13848.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13879.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13910.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13939.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/13970.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14000.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14031.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14061.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14092.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14123.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14153.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14184.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14214.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14276.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14304.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14335.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14365.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14396.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14426.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14457.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14488.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14518.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14549.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14579.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14641.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14669.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14700.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14730.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14761.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14791.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14822.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14853.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14883.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14914.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14944.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/14975.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15006.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15034.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15065.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15095.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15126.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15156.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15187.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15218.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15248.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15340.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15371.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15400.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15431.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15461.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15492.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15522.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15553.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15584.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15614.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15645.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15675.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15737.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15765.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15796.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15826.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15857.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15887.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15918.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15949.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/15979.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16010.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16040.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16102.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16130.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16161.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16191.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16222.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16252.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16283.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16314.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16344.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16375.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16436.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16467.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16495.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16526.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16556.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16587.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16617.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16648.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16679.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16709.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16801.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16832.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16861.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16892.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16922.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16953.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/16983.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17014.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17045.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17075.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17106.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17136.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17198.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17226.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17257.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17287.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17318.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17348.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17379.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17410.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17440.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17471.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17501.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17563.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17591.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17622.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17652.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17683.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17713.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17744.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17775.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17805.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17836.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17897.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17928.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17956.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/17987.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18017.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18048.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18078.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18109.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18140.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18170.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18201.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18262.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18322.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18353.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18383.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18414.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18444.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18475.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18506.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18536.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18567.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18597.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18659.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18687.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18718.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18748.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18779.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18809.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18840.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18871.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18901.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18932.gz.parquet
+data/processed_data/model_data/dynamic_parquet/most_recent_burn_dates/18962.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11005.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11021.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11037.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11053.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11069.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11085.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11101.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11117.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11133.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11165.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11181.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11197.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11213.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11229.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11323.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11339.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11355.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11371.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11387.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11403.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11419.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11435.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11451.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11467.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11483.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11499.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11515.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11531.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11547.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11563.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11579.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11595.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11611.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11627.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11643.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11659.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11675.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11688.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11704.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11720.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11736.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11752.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11768.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11784.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11800.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11816.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11832.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11848.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11864.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11880.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11896.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11912.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11928.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11944.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11960.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11976.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/11992.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12008.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12024.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12040.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12053.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12069.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12085.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12101.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12117.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12133.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12165.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12181.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12197.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12213.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12229.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12325.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12341.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12373.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12389.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12418.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12434.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12450.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12466.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12482.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12498.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12514.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12530.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12546.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12562.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12578.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12594.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12626.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12642.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12658.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12674.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12690.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12784.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12800.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12816.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12832.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12848.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12864.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12880.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12896.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12912.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12928.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12944.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12960.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12976.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/12992.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13008.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13024.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13040.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13056.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13072.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13088.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13104.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13120.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13136.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13165.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13181.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13197.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13213.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13229.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13325.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13341.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13373.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13389.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13421.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13437.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13453.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13469.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13485.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13501.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13514.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13530.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13546.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13562.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13578.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13594.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13626.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13642.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13658.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13674.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13690.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13786.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13802.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13834.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13850.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13879.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13895.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13911.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13927.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13943.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13959.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13975.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/13991.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14007.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14023.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14039.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14055.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14087.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14103.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14119.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14135.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14151.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14325.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14341.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14373.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14389.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14421.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14437.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14453.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14469.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14485.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14501.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14517.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14533.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14549.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14565.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14581.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14597.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14626.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14642.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14658.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14674.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14690.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14786.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14802.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14834.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14850.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14882.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14898.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14914.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14930.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14946.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14962.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14975.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/14991.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15007.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15023.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15039.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15055.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15087.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15103.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15119.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15135.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15151.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15247.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15263.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15295.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15311.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15327.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15340.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15356.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15372.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15388.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15404.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15420.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15436.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15452.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15468.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15484.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15500.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15516.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15548.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15564.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15580.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15612.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15786.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15802.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15834.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15850.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15882.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15898.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15914.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15930.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15946.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15962.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15978.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/15994.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16010.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16026.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16042.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16058.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16087.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16103.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16119.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16135.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16151.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16247.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16263.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16295.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16311.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16327.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16343.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16359.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16375.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16391.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16407.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16423.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16436.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16452.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16468.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16484.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16500.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16516.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16548.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16564.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16580.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16612.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16708.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16724.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16756.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16772.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16788.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16801.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16817.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16833.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16849.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16865.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16881.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16897.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16913.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16929.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16945.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16961.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16977.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/16993.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17009.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17025.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17041.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17057.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17073.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17089.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17105.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17121.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17137.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17153.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17247.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17263.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17295.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17311.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17327.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17343.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17359.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17375.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17391.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17407.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17423.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17439.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17455.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17471.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17487.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17503.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17519.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17548.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17564.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17580.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17612.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17708.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17724.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17756.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17772.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17788.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17804.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17820.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17836.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17852.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17868.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17884.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17897.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17913.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17929.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17945.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17961.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17977.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/17993.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18009.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18025.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18041.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18057.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18073.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18089.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18105.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18121.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18137.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18153.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18169.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18185.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18201.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18217.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18233.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18249.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18262.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18278.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18294.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18310.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18326.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18342.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18358.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18374.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18390.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18406.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18422.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18438.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18454.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18470.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18486.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18502.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18518.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18534.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18550.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18566.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18582.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18598.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18614.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18708.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18724.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18756.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18772.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18788.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18804.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18820.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18836.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18852.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18868.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18884.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18900.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18916.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18932.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18948.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18964.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18980.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/18993.gz.parquet
+data/processed_data/model_data/dynamic_parquet/ndvi/19009.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11005.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11021.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11037.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11053.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11069.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11085.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11101.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11117.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11133.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11165.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11181.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11197.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11213.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11229.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11323.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11339.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11355.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11371.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11387.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11403.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11419.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11435.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11451.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11467.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11483.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11499.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11515.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11531.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11547.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11563.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11579.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11595.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11611.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11627.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11643.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11659.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11675.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11688.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11704.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11720.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11736.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11752.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11768.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11784.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11800.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11816.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11832.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11848.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11864.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11880.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11896.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11912.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11928.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11944.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11960.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11976.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/11992.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12008.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12024.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12040.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12053.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12069.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12085.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12101.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12117.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12133.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12165.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12181.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12197.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12213.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12229.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12325.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12341.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12373.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12389.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12418.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12434.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12450.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12466.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12482.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12498.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12514.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12530.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12546.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12562.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12578.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12594.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12626.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12642.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12658.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12674.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12690.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12784.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12800.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12816.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12832.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12848.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12864.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12880.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12896.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12912.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12928.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12944.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12960.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12976.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/12992.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13008.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13024.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13040.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13056.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13072.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13088.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13104.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13120.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13136.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13149.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13165.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13181.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13197.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13213.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13229.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13325.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13341.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13373.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13389.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13421.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13437.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13453.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13469.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13485.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13501.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13514.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13530.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13546.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13562.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13578.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13594.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13626.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13642.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13658.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13674.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13690.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13786.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13802.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13834.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13850.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13879.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13895.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13911.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13927.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13943.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13959.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13975.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/13991.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14007.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14023.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14039.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14055.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14087.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14103.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14119.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14135.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14151.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14245.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14261.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14277.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14293.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14309.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14325.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14341.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14357.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14373.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14389.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14405.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14421.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14437.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14453.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14469.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14485.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14501.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14517.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14533.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14549.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14565.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14581.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14597.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14610.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14626.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14642.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14658.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14674.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14690.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14786.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14802.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14834.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14850.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14882.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14898.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14914.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14930.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14946.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14962.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14975.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/14991.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15007.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15023.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15039.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15055.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15087.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15103.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15119.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15135.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15151.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15247.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15263.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15295.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15311.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15327.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15340.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15356.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15372.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15388.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15404.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15420.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15436.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15452.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15468.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15484.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15500.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15516.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15548.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15564.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15580.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15612.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15706.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15722.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15738.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15754.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15770.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15786.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15802.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15818.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15834.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15850.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15866.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15882.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15898.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15914.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15930.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15946.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15962.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15978.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/15994.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16010.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16026.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16042.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16058.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16071.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16087.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16103.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16119.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16135.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16151.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16247.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16263.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16295.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16311.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16327.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16343.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16359.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16375.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16391.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16407.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16423.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16436.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16452.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16468.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16484.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16500.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16516.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16548.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16564.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16580.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16612.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16708.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16724.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16756.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16772.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16788.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16801.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16817.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16833.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16849.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16865.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16881.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16897.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16913.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16929.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16945.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16961.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16977.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/16993.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17009.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17025.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17041.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17057.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17073.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17089.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17105.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17121.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17137.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17153.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17167.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17183.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17199.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17215.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17231.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17247.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17263.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17279.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17295.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17311.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17327.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17343.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17359.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17375.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17391.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17407.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17423.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17439.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17455.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17471.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17487.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17503.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17519.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17532.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17548.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17564.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17580.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17596.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17612.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17708.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17724.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17756.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17772.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17788.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17804.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17820.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17836.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17852.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17868.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17884.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17897.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17913.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17929.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17945.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17961.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17977.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/17993.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18009.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18025.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18041.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18057.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18073.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18089.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18105.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18121.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18137.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18153.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18169.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18185.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18201.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18217.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18233.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18249.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18262.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18278.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18294.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18310.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18326.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18342.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18358.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18374.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18390.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18406.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18422.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18438.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18454.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18470.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18486.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18502.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18518.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18534.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18550.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18566.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18582.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18598.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18614.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18628.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18644.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18660.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18676.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18692.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18708.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18724.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18740.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18756.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18772.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18788.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18804.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18820.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18836.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18852.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18868.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18884.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18900.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18916.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18932.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18948.gz.parquet
+data/processed_data/model_data/dynamic_parquet/time_since_fire/18964.gz.parquet
+data/processed_data/model_data/stable_data.gz.parquet
+data/processed_data/most_recent_burn_dates/2000_11_01.tif
+data/processed_data/most_recent_burn_dates/2000_12_01.tif
+data/processed_data/most_recent_burn_dates/2001_01_01.tif
+data/processed_data/most_recent_burn_dates/2001_02_01.tif
+data/processed_data/most_recent_burn_dates/2001_03_01.tif
+data/processed_data/most_recent_burn_dates/2001_04_01.tif
+data/processed_data/most_recent_burn_dates/2001_05_01.tif
+data/processed_data/most_recent_burn_dates/2001_06_01.tif
+data/processed_data/most_recent_burn_dates/2001_07_01.tif
+data/processed_data/most_recent_burn_dates/2001_08_01.tif
+data/processed_data/most_recent_burn_dates/2001_09_01.tif
+data/processed_data/most_recent_burn_dates/2001_10_01.tif
+data/processed_data/most_recent_burn_dates/2001_11_01.tif
+data/processed_data/most_recent_burn_dates/2001_12_01.tif
+data/processed_data/most_recent_burn_dates/2002_01_01.tif
+data/processed_data/most_recent_burn_dates/2002_02_01.tif
+data/processed_data/most_recent_burn_dates/2002_03_01.tif
+data/processed_data/most_recent_burn_dates/2002_04_01.tif
+data/processed_data/most_recent_burn_dates/2002_05_01.tif
+data/processed_data/most_recent_burn_dates/2002_06_01.tif
+data/processed_data/most_recent_burn_dates/2002_07_01.tif
+data/processed_data/most_recent_burn_dates/2002_08_01.tif
+data/processed_data/most_recent_burn_dates/2002_09_01.tif
+data/processed_data/most_recent_burn_dates/2002_10_01.tif
+data/processed_data/most_recent_burn_dates/2002_11_01.tif
+data/processed_data/most_recent_burn_dates/2002_12_01.tif
+data/processed_data/most_recent_burn_dates/2003_01_01.tif
+data/processed_data/most_recent_burn_dates/2003_02_01.tif
+data/processed_data/most_recent_burn_dates/2003_03_01.tif
+data/processed_data/most_recent_burn_dates/2003_04_01.tif
+data/processed_data/most_recent_burn_dates/2003_05_01.tif
+data/processed_data/most_recent_burn_dates/2003_06_01.tif
+data/processed_data/most_recent_burn_dates/2003_07_01.tif
+data/processed_data/most_recent_burn_dates/2003_08_01.tif
+data/processed_data/most_recent_burn_dates/2003_09_01.tif
+data/processed_data/most_recent_burn_dates/2003_10_01.tif
+data/processed_data/most_recent_burn_dates/2003_11_01.tif
+data/processed_data/most_recent_burn_dates/2003_12_01.tif
+data/processed_data/most_recent_burn_dates/2004_01_01.tif
+data/processed_data/most_recent_burn_dates/2004_02_01.tif
+data/processed_data/most_recent_burn_dates/2004_03_01.tif
+data/processed_data/most_recent_burn_dates/2004_04_01.tif
+data/processed_data/most_recent_burn_dates/2004_05_01.tif
+data/processed_data/most_recent_burn_dates/2004_06_01.tif
+data/processed_data/most_recent_burn_dates/2004_07_01.tif
+data/processed_data/most_recent_burn_dates/2004_08_01.tif
+data/processed_data/most_recent_burn_dates/2004_09_01.tif
+data/processed_data/most_recent_burn_dates/2004_10_01.tif
+data/processed_data/most_recent_burn_dates/2004_11_01.tif
+data/processed_data/most_recent_burn_dates/2004_12_01.tif
+data/processed_data/most_recent_burn_dates/2005_01_01.tif
+data/processed_data/most_recent_burn_dates/2005_02_01.tif
+data/processed_data/most_recent_burn_dates/2005_03_01.tif
+data/processed_data/most_recent_burn_dates/2005_04_01.tif
+data/processed_data/most_recent_burn_dates/2005_05_01.tif
+data/processed_data/most_recent_burn_dates/2005_06_01.tif
+data/processed_data/most_recent_burn_dates/2005_07_01.tif
+data/processed_data/most_recent_burn_dates/2005_08_01.tif
+data/processed_data/most_recent_burn_dates/2005_09_01.tif
+data/processed_data/most_recent_burn_dates/2005_10_01.tif
+data/processed_data/most_recent_burn_dates/2005_11_01.tif
+data/processed_data/most_recent_burn_dates/2005_12_01.tif
+data/processed_data/most_recent_burn_dates/2006_01_01.tif
+data/processed_data/most_recent_burn_dates/2006_02_01.tif
+data/processed_data/most_recent_burn_dates/2006_03_01.tif
+data/processed_data/most_recent_burn_dates/2006_04_01.tif
+data/processed_data/most_recent_burn_dates/2006_05_01.tif
+data/processed_data/most_recent_burn_dates/2006_06_01.tif
+data/processed_data/most_recent_burn_dates/2006_07_01.tif
+data/processed_data/most_recent_burn_dates/2006_08_01.tif
+data/processed_data/most_recent_burn_dates/2006_09_01.tif
+data/processed_data/most_recent_burn_dates/2006_10_01.tif
+data/processed_data/most_recent_burn_dates/2006_11_01.tif
+data/processed_data/most_recent_burn_dates/2006_12_01.tif
+data/processed_data/most_recent_burn_dates/2007_01_01.tif
+data/processed_data/most_recent_burn_dates/2007_02_01.tif
+data/processed_data/most_recent_burn_dates/2007_03_01.tif
+data/processed_data/most_recent_burn_dates/2007_04_01.tif
+data/processed_data/most_recent_burn_dates/2007_05_01.tif
+data/processed_data/most_recent_burn_dates/2007_06_01.tif
+data/processed_data/most_recent_burn_dates/2007_07_01.tif
+data/processed_data/most_recent_burn_dates/2007_08_01.tif
+data/processed_data/most_recent_burn_dates/2007_09_01.tif
+data/processed_data/most_recent_burn_dates/2007_10_01.tif
+data/processed_data/most_recent_burn_dates/2007_11_01.tif
+data/processed_data/most_recent_burn_dates/2007_12_01.tif
+data/processed_data/most_recent_burn_dates/2008_01_01.tif
+data/processed_data/most_recent_burn_dates/2008_02_01.tif
+data/processed_data/most_recent_burn_dates/2008_03_01.tif
+data/processed_data/most_recent_burn_dates/2008_04_01.tif
+data/processed_data/most_recent_burn_dates/2008_05_01.tif
+data/processed_data/most_recent_burn_dates/2008_06_01.tif
+data/processed_data/most_recent_burn_dates/2008_07_01.tif
+data/processed_data/most_recent_burn_dates/2008_08_01.tif
+data/processed_data/most_recent_burn_dates/2008_09_01.tif
+data/processed_data/most_recent_burn_dates/2008_10_01.tif
+data/processed_data/most_recent_burn_dates/2008_11_01.tif
+data/processed_data/most_recent_burn_dates/2008_12_01.tif
+data/processed_data/most_recent_burn_dates/2009_01_01.tif
+data/processed_data/most_recent_burn_dates/2009_02_01.tif
+data/processed_data/most_recent_burn_dates/2009_03_01.tif
+data/processed_data/most_recent_burn_dates/2009_04_01.tif
+data/processed_data/most_recent_burn_dates/2009_05_01.tif
+data/processed_data/most_recent_burn_dates/2009_06_01.tif
+data/processed_data/most_recent_burn_dates/2009_07_01.tif
+data/processed_data/most_recent_burn_dates/2009_08_01.tif
+data/processed_data/most_recent_burn_dates/2009_09_01.tif
+data/processed_data/most_recent_burn_dates/2009_10_01.tif
+data/processed_data/most_recent_burn_dates/2009_11_01.tif
+data/processed_data/most_recent_burn_dates/2009_12_01.tif
+data/processed_data/most_recent_burn_dates/2010_01_01.tif
+data/processed_data/most_recent_burn_dates/2010_02_01.tif
+data/processed_data/most_recent_burn_dates/2010_03_01.tif
+data/processed_data/most_recent_burn_dates/2010_04_01.tif
+data/processed_data/most_recent_burn_dates/2010_05_01.tif
+data/processed_data/most_recent_burn_dates/2010_06_01.tif
+data/processed_data/most_recent_burn_dates/2010_07_01.tif
+data/processed_data/most_recent_burn_dates/2010_08_01.tif
+data/processed_data/most_recent_burn_dates/2010_09_01.tif
+data/processed_data/most_recent_burn_dates/2010_10_01.tif
+data/processed_data/most_recent_burn_dates/2010_11_01.tif
+data/processed_data/most_recent_burn_dates/2010_12_01.tif
+data/processed_data/most_recent_burn_dates/2011_01_01.tif
+data/processed_data/most_recent_burn_dates/2011_02_01.tif
+data/processed_data/most_recent_burn_dates/2011_03_01.tif
+data/processed_data/most_recent_burn_dates/2011_04_01.tif
+data/processed_data/most_recent_burn_dates/2011_05_01.tif
+data/processed_data/most_recent_burn_dates/2011_06_01.tif
+data/processed_data/most_recent_burn_dates/2011_07_01.tif
+data/processed_data/most_recent_burn_dates/2011_08_01.tif
+data/processed_data/most_recent_burn_dates/2011_09_01.tif
+data/processed_data/most_recent_burn_dates/2011_10_01.tif
+data/processed_data/most_recent_burn_dates/2011_11_01.tif
+data/processed_data/most_recent_burn_dates/2011_12_01.tif
+data/processed_data/most_recent_burn_dates/2012_01_01.tif
+data/processed_data/most_recent_burn_dates/2012_02_01.tif
+data/processed_data/most_recent_burn_dates/2012_03_01.tif
+data/processed_data/most_recent_burn_dates/2012_04_01.tif
+data/processed_data/most_recent_burn_dates/2012_05_01.tif
+data/processed_data/most_recent_burn_dates/2012_06_01.tif
+data/processed_data/most_recent_burn_dates/2012_07_01.tif
+data/processed_data/most_recent_burn_dates/2012_08_01.tif
+data/processed_data/most_recent_burn_dates/2012_09_01.tif
+data/processed_data/most_recent_burn_dates/2012_10_01.tif
+data/processed_data/most_recent_burn_dates/2012_11_01.tif
+data/processed_data/most_recent_burn_dates/2012_12_01.tif
+data/processed_data/most_recent_burn_dates/2013_01_01.tif
+data/processed_data/most_recent_burn_dates/2013_02_01.tif
+data/processed_data/most_recent_burn_dates/2013_03_01.tif
+data/processed_data/most_recent_burn_dates/2013_04_01.tif
+data/processed_data/most_recent_burn_dates/2013_05_01.tif
+data/processed_data/most_recent_burn_dates/2013_06_01.tif
+data/processed_data/most_recent_burn_dates/2013_07_01.tif
+data/processed_data/most_recent_burn_dates/2013_08_01.tif
+data/processed_data/most_recent_burn_dates/2013_09_01.tif
+data/processed_data/most_recent_burn_dates/2013_10_01.tif
+data/processed_data/most_recent_burn_dates/2013_11_01.tif
+data/processed_data/most_recent_burn_dates/2013_12_01.tif
+data/processed_data/most_recent_burn_dates/2014_01_01.tif
+data/processed_data/most_recent_burn_dates/2014_02_01.tif
+data/processed_data/most_recent_burn_dates/2014_03_01.tif
+data/processed_data/most_recent_burn_dates/2014_04_01.tif
+data/processed_data/most_recent_burn_dates/2014_05_01.tif
+data/processed_data/most_recent_burn_dates/2014_06_01.tif
+data/processed_data/most_recent_burn_dates/2014_07_01.tif
+data/processed_data/most_recent_burn_dates/2014_08_01.tif
+data/processed_data/most_recent_burn_dates/2014_09_01.tif
+data/processed_data/most_recent_burn_dates/2014_10_01.tif
+data/processed_data/most_recent_burn_dates/2014_11_01.tif
+data/processed_data/most_recent_burn_dates/2014_12_01.tif
+data/processed_data/most_recent_burn_dates/2015_01_01.tif
+data/processed_data/most_recent_burn_dates/2015_02_01.tif
+data/processed_data/most_recent_burn_dates/2015_03_01.tif
+data/processed_data/most_recent_burn_dates/2015_04_01.tif
+data/processed_data/most_recent_burn_dates/2015_05_01.tif
+data/processed_data/most_recent_burn_dates/2015_06_01.tif
+data/processed_data/most_recent_burn_dates/2015_07_01.tif
+data/processed_data/most_recent_burn_dates/2015_08_01.tif
+data/processed_data/most_recent_burn_dates/2015_09_01.tif
+data/processed_data/most_recent_burn_dates/2015_10_01.tif
+data/processed_data/most_recent_burn_dates/2015_11_01.tif
+data/processed_data/most_recent_burn_dates/2015_12_01.tif
+data/processed_data/most_recent_burn_dates/2016_01_01.tif
+data/processed_data/most_recent_burn_dates/2016_02_01.tif
+data/processed_data/most_recent_burn_dates/2016_03_01.tif
+data/processed_data/most_recent_burn_dates/2016_04_01.tif
+data/processed_data/most_recent_burn_dates/2016_05_01.tif
+data/processed_data/most_recent_burn_dates/2016_06_01.tif
+data/processed_data/most_recent_burn_dates/2016_07_01.tif
+data/processed_data/most_recent_burn_dates/2016_08_01.tif
+data/processed_data/most_recent_burn_dates/2016_09_01.tif
+data/processed_data/most_recent_burn_dates/2016_10_01.tif
+data/processed_data/most_recent_burn_dates/2016_11_01.tif
+data/processed_data/most_recent_burn_dates/2016_12_01.tif
+data/processed_data/most_recent_burn_dates/2017_01_01.tif
+data/processed_data/most_recent_burn_dates/2017_02_01.tif
+data/processed_data/most_recent_burn_dates/2017_03_01.tif
+data/processed_data/most_recent_burn_dates/2017_04_01.tif
+data/processed_data/most_recent_burn_dates/2017_05_01.tif
+data/processed_data/most_recent_burn_dates/2017_06_01.tif
+data/processed_data/most_recent_burn_dates/2017_07_01.tif
+data/processed_data/most_recent_burn_dates/2017_08_01.tif
+data/processed_data/most_recent_burn_dates/2017_09_01.tif
+data/processed_data/most_recent_burn_dates/2017_10_01.tif
+data/processed_data/most_recent_burn_dates/2017_11_01.tif
+data/processed_data/most_recent_burn_dates/2017_12_01.tif
+data/processed_data/most_recent_burn_dates/2018_01_01.tif
+data/processed_data/most_recent_burn_dates/2018_02_01.tif
+data/processed_data/most_recent_burn_dates/2018_03_01.tif
+data/processed_data/most_recent_burn_dates/2018_04_01.tif
+data/processed_data/most_recent_burn_dates/2018_05_01.tif
+data/processed_data/most_recent_burn_dates/2018_06_01.tif
+data/processed_data/most_recent_burn_dates/2018_07_01.tif
+data/processed_data/most_recent_burn_dates/2018_08_01.tif
+data/processed_data/most_recent_burn_dates/2018_09_01.tif
+data/processed_data/most_recent_burn_dates/2018_10_01.tif
+data/processed_data/most_recent_burn_dates/2018_11_01.tif
+data/processed_data/most_recent_burn_dates/2018_12_01.tif
+data/processed_data/most_recent_burn_dates/2019_01_01.tif
+data/processed_data/most_recent_burn_dates/2019_02_01.tif
+data/processed_data/most_recent_burn_dates/2019_03_01.tif
+data/processed_data/most_recent_burn_dates/2019_04_01.tif
+data/processed_data/most_recent_burn_dates/2019_05_01.tif
+data/processed_data/most_recent_burn_dates/2019_06_01.tif
+data/processed_data/most_recent_burn_dates/2019_07_01.tif
+data/processed_data/most_recent_burn_dates/2019_08_01.tif
+data/processed_data/most_recent_burn_dates/2019_09_01.tif
+data/processed_data/most_recent_burn_dates/2019_10_01.tif
+data/processed_data/most_recent_burn_dates/2019_11_01.tif
+data/processed_data/most_recent_burn_dates/2019_12_01.tif
+data/processed_data/most_recent_burn_dates/2020_01_01.tif
+data/processed_data/most_recent_burn_dates/2020_02_01.tif
+data/processed_data/most_recent_burn_dates/2020_03_01.tif
+data/processed_data/most_recent_burn_dates/2020_04_01.tif
+data/processed_data/most_recent_burn_dates/2020_05_01.tif
+data/processed_data/most_recent_burn_dates/2020_06_01.tif
+data/processed_data/most_recent_burn_dates/2020_07_01.tif
+data/processed_data/most_recent_burn_dates/2020_08_01.tif
+data/processed_data/most_recent_burn_dates/2020_09_01.tif
+data/processed_data/most_recent_burn_dates/2020_10_01.tif
+data/processed_data/most_recent_burn_dates/2020_11_01.tif
+data/processed_data/most_recent_burn_dates/2020_12_01.tif
+data/processed_data/most_recent_burn_dates/2021_01_01.tif
+data/processed_data/most_recent_burn_dates/2021_02_01.tif
+data/processed_data/most_recent_burn_dates/2021_03_01.tif
+data/processed_data/most_recent_burn_dates/2021_04_01.tif
+data/processed_data/most_recent_burn_dates/2021_05_01.tif
+data/processed_data/most_recent_burn_dates/2021_06_01.tif
+data/processed_data/most_recent_burn_dates/2021_07_01.tif
+data/processed_data/most_recent_burn_dates/2021_08_01.tif
+data/processed_data/most_recent_burn_dates/2021_09_01.tif
+data/processed_data/most_recent_burn_dates/2021_10_01.tif
+data/processed_data/most_recent_burn_dates/2021_11_01.tif
+data/processed_data/most_recent_burn_dates/2021_12_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_03_05.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_03_21.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_04_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_04_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_05_08.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_05_24.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_06_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_06_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_07_11.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_07_27.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_08_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_08_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_09_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_09_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_10_15.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_10_31.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_11_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_12_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2000_12_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2001_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2002_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2003_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_03_05.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_03_21.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_04_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_04_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_05_08.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_05_24.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_06_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_06_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_07_11.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_07_27.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_08_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_08_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_09_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_09_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_10_15.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_10_31.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_11_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_12_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2004_12_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2005_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2006_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2007_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_03_05.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_03_21.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_04_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_04_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_05_08.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_05_24.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_06_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_06_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_07_11.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_07_27.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_08_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_08_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_09_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_09_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_10_15.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_10_31.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_11_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_12_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2008_12_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2009_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2010_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2011_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_03_05.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_03_21.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_04_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_04_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_05_08.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_05_24.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_06_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_06_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_07_11.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_07_27.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_08_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_08_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_09_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_09_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_10_15.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_10_31.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_11_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_12_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2012_12_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2013_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2014_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2015_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_03_05.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_03_21.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_04_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_04_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_05_08.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_05_24.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_06_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_06_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_07_11.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_07_27.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_08_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_08_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_09_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_09_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_10_15.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_10_31.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_11_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_12_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2016_12_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2017_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2018_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_12_03.tif
+data/processed_data/ndvi_relative_time_since_fire/2019_12_19.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_03_05.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_03_21.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_04_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_04_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_05_08.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_05_24.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_06_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_06_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_07_11.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_07_27.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_08_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_08_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_09_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_09_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_10_15.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_10_31.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_11_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_12_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2020_12_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_01_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_01_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_02_02.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_02_18.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_03_06.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_03_22.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_04_07.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_04_23.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_05_09.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_05_25.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_06_10.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_06_26.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_07_12.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_07_28.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_08_13.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_08_29.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_09_14.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_09_30.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_10_16.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_11_01.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_11_17.tif
+data/processed_data/ndvi_relative_time_since_fire/2021_12_03.tif
+data/processed_data/precipitation_chelsa/CHELSA_prec_01_V1.2_land_clipped.tif
+data/processed_data/precipitation_chelsa/CHELSA_prec_07_V1.2_land_clipped.tif
+data/raw_data/alos/alos_chili.tif
+data/raw_data/alos/alos_mtpi.tif
+data/raw_data/alos/alos_topographic_diversity.tif
+data/raw_data/alos/landforms.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_01_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_02_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_03_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_04_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_05_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_06_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_07_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_08_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_09_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_10_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_11_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_12_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_13_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_14_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_15_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_16_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_17_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_18_V1.2_clipped.tif
+data/raw_data/climate_chelsa/bio/bio_V1.2/clipped/CHELSA_bio10_19_V1.2_clipped.tif
+data/raw_data/climate_chelsa/chelsa_citation.bib
+data/raw_data/clouds_wilson/MODCF_interannualSD.tif
+data/raw_data/clouds_wilson/MODCF_intraannualSD.tif
+data/raw_data/clouds_wilson/MODCF_meanannual.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_01.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_02.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_03.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_04.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_05.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_06.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_07.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_08.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_09.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_10.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_11.tif
+data/raw_data/clouds_wilson/MODCF_monthlymean_12.tif
+data/raw_data/clouds_wilson/MODCF_seasonality_concentration.tif
+data/raw_data/clouds_wilson/MODCF_seasonality_rgb.tif
+data/raw_data/clouds_wilson/MODCF_seasonality_theta.tif
+data/raw_data/clouds_wilson/MODCF_seasonality_visct.tif
+data/raw_data/clouds_wilson/MODCF_spatialSD_1deg.tif
+data/raw_data/elevation_nasadem/nasadem.tif
+data/raw_data/fire_modis/2021_11_01.tif
+data/raw_data/fire_modis/2021_12_01.tif
+data/raw_data/kndvi_modis/.tif
+data/raw_data/landcover_za/SA_NLC_2020_GEO.tif
+data/raw_data/landcover_za/SA_NLC_2020_GEO.tif.vat.cpg
+data/raw_data/landcover_za/SA_NLC_2020_GEO.tif.vat.dbf
+data/raw_data/ndvi_dates_modis/.tif
+data/raw_data/ndvi_dates_modis/2022_01_01.tif
+data/raw_data/ndvi_dates_modis/2022_01_17.tif
+data/raw_data/ndvi_modis/.tif
+data/raw_data/ndvi_modis/2022_01_01.tif
+data/raw_data/ndvi_modis/2022_01_17.tif
+data/raw_data/precipitation_chelsa/chelsa_citation.bib
+data/raw_data/precipitation_chelsa/prec/prec_V1.2/clipped/CHELSA_prec_01_V1.2_land_clipped.grd
+data/raw_data/precipitation_chelsa/prec/prec_V1.2/clipped/CHELSA_prec_01_V1.2_land_clipped.gri
+data/raw_data/precipitation_chelsa/prec/prec_V1.2/clipped/CHELSA_prec_01_V1.2_land_clipped.tif
+data/raw_data/precipitation_chelsa/prec/prec_V1.2/clipped/CHELSA_prec_07_V1.2_land_clipped.grd
+data/raw_data/precipitation_chelsa/prec/prec_V1.2/clipped/CHELSA_prec_07_V1.2_land_clipped.gri
+data/raw_data/precipitation_chelsa/prec/prec_V1.2/clipped/CHELSA_prec_07_V1.2_land_clipped.tif
+data/remnant_distance.tif
+data/remnants.tif
+firemodel_predict
+index.md
+index_files/figure-gfm/p1-1.png
+index_files/figure-gfm/plot-1.png
+scratch_code/test.zip
+scratch_code/wc2.1_10m_tmin_01.tif
+data/raw_data/fire_modis/log.csv
+data/raw_data/ndvi_dates_modis/log.csv
+data/raw_data/ndvi_modis/log.csv
diff --git a/DESCRIPTION b/DESCRIPTION
index 9bf7a592..53bf1796 100644
--- a/DESCRIPTION
+++ b/DESCRIPTION
@@ -1,10 +1,10 @@
Package: EMMAv1
Type: Project
Title: EMMA Targets Workflow
-Version: 0.1.0
+Version: 0.2.0
Author: Adam M. Wilson
Maintainer: Adam M. Wilson
- Description: Example targets workflow for GitHub actions
+ Description: EMMA data workflow using targets, AppEEARS, and GitHub releases
License: MIT + file LICENSE
Encoding: UTF-8
LazyData: true
@@ -14,12 +14,33 @@ Imports:
viridis,
colourvalues,
sf,
- raster,
+ terra,
+ stars,
testthat,
kableExtra,
knitr,
- cmdstanr,
- posterior,
- bayesplot
+ appeears,
+ piggyback,
+ lubridate,
+ jsonlite,
+ dplyr,
+ arrow,
+ targets,
+ tarchetypes,
+ visNetwork,
+ rdryad,
+ keyring,
+ filelock,
+ sfarrow,
+ geotargets,
+ janitor,
+ ncdf4,
+ devtools,
+ smoothr,
+ piggyback,
+ questionr
Suggests:
+ rgee,
+ reticulate,
+ raster,
RoxygenNote: 7.1.1
diff --git a/IMPLEMENTATION_SUMMARY.txt b/IMPLEMENTATION_SUMMARY.txt
new file mode 100644
index 00000000..4fb37614
--- /dev/null
+++ b/IMPLEMENTATION_SUMMARY.txt
@@ -0,0 +1,190 @@
+# MODIS VI Monthly Download Implementation - Summary
+
+## Implementation Date
+February 23, 2026
+
+## Overview
+Refactored MODIS VI data downloading from a single monolithic AppEEARS request to a month-by-month approach with dynamic branching in the targets workflow.
+
+## Changes Made
+
+### 1. New File: `R/modis_vi_monthly_helpers.R`
+**Purpose**: Helper functions for month management and aggregation
+
+**Functions**:
+- `generate_month_sequence()` - Creates all months within a date range
+- `identify_missing_months()` - Detects which months need downloading
+- `aggregate_modis_vi_monthly()` - Combines monthly files into single NetCDF
+- `create_modis_vi_monthly_index()` - Creates parquet index of monthly files
+
+**Key Features**:
+- Supports both "prime" (all months) and "update" (recent months) modes
+- Automatic detection of missing months based on output directory
+- Parallelization-ready: dynamic branching identifies independent work units
+
+### 2. Updated File: `R/get_modis_vi.R`
+**Added Functions**:
+- `submit_modis_vi_month()` - Submits single month to AppEEARS (new)
+- `download_modis_vi_month()` - Downloads and processes single month (new)
+
+**Kept Functions** (for backward compatibility):
+- `submit_modis_vi_task()` - Original monolithic submission (deprecated)
+- `download_modis_vi_results()` - Original monolithic download (deprecated)
+
+**Size**: Added ~280 lines of code (functions well-documented with roxygen2 comments)
+
+### 3. Updated File: `_targets.R`
+**Replaced Targets**:
+- `modis_vi_task_id` β New dynamic branching workflow
+- `modis_vi` β New monthly download targets
+
+**New Targets** (lines 288-387):
+1. **`modis_vi_months_to_download`** - Identifies missing months
+2. **`modis_vi_month_task_ids`** - Dynamic branching: submits each month
+3. **`modis_vi_monthly_files`** - Dynamic branching: downloads each month
+4. **`modis_vi_monthly_index`** - Creates parquet index of all files
+5. **`modis_vi`** (commented) - Optional aggregation to single file
+
+**Improvements**:
+- Added descriptive comment block explaining the new approach
+- Dynamic branching enables parallelization (2+ workers)
+- Monthly granularity enables natural incremental updates
+- Resilience: Single month failure doesn't cascade
+
+### 4. New File: `R/MODIS_VI_MONTHLY_README.md`
+**Purpose**: User documentation and implementation guide
+
+**Sections**:
+- How It Works (overview of 5-step pipeline)
+- File Structure (output directory layout)
+- Migration notes (old vs new approach)
+- Parallelization instructions
+- Downstream usage examples
+- Troubleshooting guide
+- Future enhancements
+
+## Technical Details
+
+### Month Sequence Generation
+```
+2000-02 (49 days) β 2000-03, 2000-04, ... β 2026-02
+Total: 313 months
+```
+
+### Dynamic Branching Pattern
+```r
+tar_target(months_to_download, ...) %>% # Identify work
+ tar_target(monthly_tasks, ..., # One task per month
+ pattern = map(months_to_download)
+ ) %>%
+ tar_target(monthly_files, ..., # One download per task
+ pattern = map(monthly_tasks)
+ )
+```
+
+### File Naming Convention
+- Input: Monthly NetCDF files from AppEEARS
+- Output: `modis_vi_YYYY_MM.nc` (e.g., `modis_vi_2000_02.nc`)
+- Index: `modis_vi_monthly_index.parquet`
+
+### Parallelization Potential
+- **Current**: Single-processor (sequential)
+- **Potential with GitHub Actions**: 2 workers β ~2x speedup
+- **Potential on CCR cluster**: 8+ workers β ~8x speedup
+
+## Backward Compatibility
+
+- Old functions (`submit_modis_vi_task`, `download_modis_vi_results`) remain available
+- Can revert to monolithic approach if needed by uncommenting old targets
+- New functions follow consistent naming and parameter patterns
+
+## Testing
+
+β **Syntax validation**: `_targets.R` parses successfully
+β **Function loading**: All new functions source without errors
+β **Month sequence**: Generates 313 months correctly (2000-02 to 2026-02)
+β **Code organization**: Follows existing project patterns
+
+## Next Steps for User
+
+### Option 1: Immediate Testing (Recommended)
+```r
+# Test month identification locally
+source("R/modis_vi_monthly_helpers.R")
+missing_months <- identify_missing_months(
+ output_dir = "data/target_outputs/modis_vi_monthly",
+ start_date = "2025-01-01",
+ end_date = "2025-12-31"
+)
+# Should return months not yet in output directory
+```
+
+### Option 2: Run Full Pipeline (Prime Mode)
+```bash
+cd /Users/adamw/Documents/repos/emma/emma_envdata
+Rscript -e "targets::tar_make()"
+```
+- Downloads all months from 2000-02 to present
+- First run creates 313 monthly files (~1-2 TB total)
+- Subsequent runs only add new months
+
+### Option 3: Enable Parallelization
+Edit `.github/workflows/targets.yaml` line ~97:
+```yaml
+# Old: Single processor
+Rscript -e "targets::tar_make()"
+
+# New: Two processors (GitHub Actions)
+Rscript -e "targets::tar_make_future(workers = 2)"
+```
+
+### Option 4: Keep Monthly Files Separate
+Default behavior keeps monthly NetCDF files separate. To aggregate:
+1. Uncomment the `modis_vi` target in `_targets.R` (lines 389-407)
+2. Run pipeline: `targets::tar_make()`
+3. Creates `modis_vi_combined.nc` with all months stacked
+
+## Performance Characteristics
+
+### Prime Mode (First Run)
+- 313 months, ~5-10 GB per month per format
+- Sequential: ~24-48 hours (depends on AppEEARS load)
+- With 2 workers: ~12-24 hours
+- Network: Heavy (100+ GB total download)
+
+### Update Mode (GitHub Actions)
+- Last 3 months only (~20 GB)
+- Sequential: ~30-60 minutes
+- With 2 workers: ~15-30 minutes
+- Network: Light per run, but frequent (weekly)
+
+### File Storage
+- Monthly files: ~500-600 GB for 26 years (5-10 GB/month Γ 313)
+- Index file: ~10 KB (parquet)
+- Optional combined file: 150-200 GB (if aggregated)
+
+## Potential Issues & Solutions
+
+### Issue 1: AppEEARS Timeout
+**Symptom**: "Task polling timed out after 120 minutes"
+**Solution**: Increase `max_retries` in `download_modis_vi_month()` or wait for AppEEARS recovery
+
+### Issue 2: Memory Issues with Aggregation
+**Symptom**: "Cannot allocate vector of size X GB"
+**Solution**: Keep monthly files separate (default); aggregate only if needed
+
+### Issue 3: Inconsistent Results Between Months
+**Symptom**: Different projection/grid in different months
+**Solution**: All months use same domain template (`domain_raster`); shouldn't occur
+
+## Documentation
+- Inline comments in code explain each function
+- MODIS_VI_MONTHLY_README.md provides user guide
+- This summary document (IMPLEMENTATION_SUMMARY.txt)
+
+## Rollback Plan
+If issues arise, can revert to old approach:
+1. Restore old targets in `_targets.R` (lines 287-305 commented out)
+2. Remove monthly targets (lines 288-387)
+3. Re-enable: `modis_vi_task_id` and `modis_vi`
+4. Keep monthly files in `modis_vi_monthly/` directory (separate from pipeline)
diff --git a/OBJECTS_RELEASE.md b/OBJECTS_RELEASE.md
new file mode 100644
index 00000000..fa67a3de
--- /dev/null
+++ b/OBJECTS_RELEASE.md
@@ -0,0 +1,129 @@
+# Target Objects Release (`objects_current`)
+
+This GitHub release contains cached computational artifacts from the emma_envdata workflow pipeline using content-addressable storage. Objects are stored with hash-based filenames for deduplication and efficient caching, enabling the workflow to avoid redundant computation of expensive intermediate datasets.
+
+## Contents
+
+This release stores two types of objects:
+
+### File Outputs (Spatial Data)
+Published directly as files on the release such as parquet and netCDF files. However, due to the targets workflow's content-addressable storage system, these files are referenced by their content hash in the workflow metadata. The targets workflow automatically manages hash lookups and downloads based on content.
+
+### Serialized R Objects (Intermediate Results)
+Stored with hash-based filenames (content-addressable storage) for efficient deduplication. These intermediate objects are serialized as QS files (`.qs`) and referenced by content hash on the release page. The targets workflow automatically manages hash lookups and downloads based on content.
+
+
+## How the Workflow Uses These Objects
+
+### Prime Mode (Full Processing)
+When running on the analysis server (`run_mode = "prime"`), the workflow:
+1. **Retrieves** objects from this release using the `tar_github_release_repo()` backend
+2. **Caches locally** in `data/target_outputs/.tar_cache/` for speed
+3. **Recomputes** targets only if:
+ - Source code has changed
+ - Input data has changed
+ - Objects are manually invalidated with `tar_invalidate()`
+4. **Uploads** new/modified targets back to this release
+
+### Update Mode (GitHub Actions)
+When running on GitHub Actions (`run_mode = "update"`), the workflow:
+1. **Retrieves** all cached objects from this release
+2. **Never recomputes** (cue mode = "never") to save CI/CD time
+3. **Uses cached objects** for downstream operations only
+4. **Skips** expensive computations (elevation API calls, climate downloads) unless manually invalidated
+
+This separation allows:
+- **Local development** with full control and recomputation
+- **Efficient CI/CD** that leverages pre-computed intermediate results
+- **Reproducibility** by pinning exact object versions in the release
+
+## File Formats & Storage
+
+### Spatial Files (NetCDF)
+Published with human-readable filenames on the release:
+- `.nc` files are CF-1.8 compliant netCDF4 format
+- Include geospatial metadata (CRS, bounds, variable attributes)
+- Can be read with standard tools (GDAL, xarray, R terra/ncdf4)
+
+### Serialized Objects (QS Format)
+Stored with hash-based filenames (content-addressable storage):
+- `.qs` files are R object serializations (fast, lossless)
+- Filenames are SHA-256 hashes of content
+- Hash naming enables deduplication: identical objects share one file
+- Managed transparently by targetsβhumans don't interact with hashes directly
+- Only the workflow's metadata tracks which hash corresponds to which target
+
+## When Objects Are Updated
+
+Objects in this release are regenerated and pushed automatically when:
+1. Running `tar_make()` on the analysis server with changes to:
+ - R functions in `R/` folder
+ - Data download URLs or APIs
+ - Target definitions in `_targets.R`
+2. Manual `tar_make(targets = "target_name")` calls
+3. Scheduled workflows or CI/CD pipelines
+
+## Accessing Objects
+
+### Automatic (Preferred)
+Objects are automatically retrieved by `tar_load()` and `tar_read()`:
+```r
+tar_load(elevation) # Loads from cache or downloads from release
+```
+
+### Manual Download
+To download specific objects manually:
+```bash
+gh release download objects_current --dir data/target_outputs
+```
+
+## Cache Management
+
+The local cache in `data/target_outputs/.tar_cache/` can be cleared to force re-downloads:
+```r
+unlink("data/target_outputs/.tar_cache", recursive = TRUE)
+tar_make() # Will re-download from release
+```
+
+## Hash-to-File Mapping
+
+The targets metadata stores the relationship between hash-based filenames and human-readable target names. To view this mapping:
+
+```r
+# Show all targets with their store information
+tar_manifest() %>%
+ select(name, type, path, repository) %>%
+ filter(!is.na(path)) # Only file-based targets
+```
+
+This shows:
+- **name**: Target name (e.g., `domain_boundary.parquet`, `elevation`)
+- **type**: Object type (e.g., "file", "qs")
+- **path**: Output file path (for file targets like NetCDF)
+- **repository**: Storage location (gh_repo for GitHub release objects)
+
+For serialized R objects without human-readable filenames, the mapping is stored in `_targets/meta/objects/` as metadata files that track content hashes.
+
+## Troubleshooting
+
+**Objects not loading?**
+- Check GitHub credentials: `gitcreds::gitcreds_set()`
+- Verify network connectivity
+- Clear cache and retry: `unlink("data/target_outputs/.tar_cache", recursive = TRUE)`
+
+**Out-of-sync objects?**
+- Invalidate and recompute: `tar_invalidate(target_name)`
+- Rebuild all: `tar_destroy(); tar_make()`
+
+**Need to recompute everything?**
+```r
+unlink("_targets", recursive = TRUE) # Clear all metadata
+unlink("data/target_outputs/.tar_cache", recursive = TRUE) # Clear cache
+tar_make() # Recompute all targets
+```
+
+## Related Files
+
+- [`_targets.R`](https://github.com/AdamWilsonLab/emma_envdata/blob/main/_targets.R) - Workflow pipeline definition
+- [`R/`](https://github.com/AdamWilsonLab/emma_envdata/tree/main/R) - R functions that generate these objects
+- [DESCRIPTION](https://github.com/AdamWilsonLab/emma_envdata/blob/main/DESCRIPTION) - Package dependencies
diff --git a/R/MODIS_VI_MONTHLY_README.md b/R/MODIS_VI_MONTHLY_README.md
new file mode 100644
index 00000000..c93765fc
--- /dev/null
+++ b/R/MODIS_VI_MONTHLY_README.md
@@ -0,0 +1,151 @@
+# MODIS VI Monthly Download Implementation Guide
+
+## Overview
+
+As of February 2026, MODIS VI data is downloaded **month-by-month** instead of in a single monolithic request. This improves resilience, enables parallelization, and simplifies incremental updates.
+
+## How It Works
+
+### 1. Month Identification (`modis_vi_months_to_download`)
+- Generates full sequence of months from start date to present
+- Checks which months already exist in `data/target_outputs/modis_vi_monthly/`
+- Returns only missing months for downstream processing
+- **Prime mode**: Downloads all months from 2000-02-18 to present
+- **Update mode**: Downloads last ~3 months (to recapture any missed dates)
+
+### 2. Task Submission (`modis_vi_month_task_ids`)
+- **Dynamic branching**: One task per missing month
+- Each task submits a single month to AppEEARS API
+- Returns task ID for downstream polling
+- With 2 workers: Can submit 2 months simultaneously
+
+### 3. Download & Processing (`modis_vi_monthly_files`)
+- **Dynamic branching**: Polls and downloads each month independently
+- Applies QA masking to EVI data
+- Projects to domain CRS and grid
+- Outputs: Individual NetCDF file per month (format: `modis_vi_YYYY_MM.nc`)
+- With 2 workers: Can download 2 months simultaneously
+
+### 4. Indexing (`modis_vi_monthly_index`)
+- Creates parquet summary of all monthly files
+- Maps: month β file path, size, creation date
+- Useful for downstream analysis tools to discover available data
+
+### 5. Optional Aggregation (Commented Out)
+- Can combine all months into single NetCDF file
+- Uncomment the `modis_vi` target in `_targets.R` if needed
+- Trade-off: Single file easier to use, but loses monthly granularity
+
+## File Structure
+
+```
+data/
+βββ target_outputs/
+β βββ modis_vi_monthly/ # Monthly NetCDF files
+β β βββ modis_vi_2000_02.nc
+β β βββ modis_vi_2000_03.nc
+β β βββ ... (1 file per month)
+β βββ modis_vi_monthly_index.parquet # Index of all monthly files
+```
+
+## Migration from Old Approach
+
+**Old approach** (commented-out in current code):
+- Single AppEEARS request for entire date range (26+ years)
+- Returns one massive download
+- Failure means entire run fails
+- "Update mode" creates separate file and requires reconciliation
+
+**New approach**:
+- 312 independent AppEEARS requests (one per month)
+- Each month ~5-10 GB per format (NDVI, EVI, QA)
+- Failure of month N doesn't affect other months
+- "Update mode" naturally only downloads missing months
+- Targets automatically parallelizes: with 2 workers, ~156 parallel jobs instead of 1 serial job
+
+## Parallelization
+
+The current GitHub Actions workflow runs single-processor. To enable parallelization:
+
+In `_targets.R` line ~97, change:
+```r
+Rscript -e "targets::tar_make()"
+```
+
+To:
+```r
+Rscript -e "targets::tar_make_future(workers = 2)"
+```
+
+This will download up to 2 months simultaneously, ~2x speedup for full historical runs.
+
+## Resuming Interrupted Runs
+
+If a GitHub Actions run times out mid-way:
+1. The completed monthly files are already saved
+2. On next run, `modis_vi_months_to_download` will detect and skip them
+3. Only missing months are re-downloaded
+4. No re-submission of already-completed AppEEARS tasks needed
+
+## Downstream Usage
+
+### Access All Monthly Data
+```r
+# Load the index
+library(arrow)
+monthly_index <- read_parquet("data/target_outputs/modis_vi_monthly_index.parquet")
+
+# Load specific month(s)
+ndvi_jan_2020 <- terra::rast(
+ monthly_index$file_path[monthly_index$month == "2020-01-01"]
+)
+```
+
+### Access Time Series Data
+```r
+# Read all months as stacked raster
+monthly_files <- sort(list.files(
+ "data/target_outputs/modis_vi_monthly",
+ pattern = "modis_vi_.*\\.nc$",
+ full.names = TRUE
+))
+ndvi_timeseries <- terra::rast(monthly_files) # Stacked raster
+```
+
+### Aggregate After Pipeline Completes
+If needed, aggregate monthly files manually:
+```r
+# After all months downloaded, aggregate to single file
+r::source("R/modis_vi_monthly_helpers.R")
+
+aggregate_modis_vi_monthly(
+ monthly_files = sort(list.files(...)),
+ out_file = "data/target_outputs/modis_vi_combined.nc"
+)
+```
+
+## Future Enhancements
+
+Potential improvements:
+1. **Parallel download speeds**: Currently 2 workers (limited by GitHub Actions CPU); CCR cluster supports more
+2. **Tighter incremental updates**: Current "update mode" re-downloads last 3 months; could be refined to only new data
+3. **QA statistics**: Track which months had failed pixels, guide re-processing
+4. **Data subsetting**: Support downloading specific regions instead of full domain
+5. **Format flexibility**: Consider keeping data as monthly parquet files instead of NetCDF for model consumption
+
+## Troubleshooting
+
+### "No NetCDF files downloaded from AppEEARS"
+- AppEEARS task failed to complete
+- Check AppEEARS API status (may be temporarily down)
+- Verify domain geometry is valid (simplify step in `submit_modis_vi_month()`)
+- Manual retry: Delete month file from `modis_vi_monthly/` and re-run targets
+
+### "modis_vi_monthly_index not found"
+- Monthly downloads still in progress (check GitHub Actions logs)
+- Or all months already exist and no work was done (expected in updates)
+
+### Memory issues with aggregation
+- Monthly files are large (~5-10 GB each)
+- If aggregating many months, may need to increase memory or process subset
+- Better solution: Keep monthly files separate, access via index
diff --git a/R/appeears_auth.R b/R/appeears_auth.R
new file mode 100644
index 00000000..2ec66a98
--- /dev/null
+++ b/R/appeears_auth.R
@@ -0,0 +1,42 @@
+library(appeears)
+
+# AppEEARS authentication via keyring (no .netrc)
+earthdata_user <- Sys.getenv("EARTHDATA_USER")
+earthdata_pass <- Sys.getenv("EARTHDATA_PASSWORD")
+
+if (earthdata_user != "" && earthdata_pass != "") {
+ message("Setting up NASA EarthData authentication (keyring file backend)")
+
+ # Configure file keyring backend and location
+ if (Sys.getenv("R_KEYRING_PASSWORD") == "") {
+ Sys.setenv(R_KEYRING_PASSWORD = earthdata_pass)
+ }
+ if (Sys.getenv("R_KEYRING_FILE") == "") {
+ Sys.setenv(R_KEYRING_FILE = path.expand("~/.config/r-keyring/appeears.keyring"))
+ }
+ options(keyring_backend = "file")
+ suppressWarnings(dir.create(dirname(Sys.getenv("R_KEYRING_FILE")), recursive = TRUE, showWarnings = FALSE))
+
+ kr_name <- "appeears"
+ kr_pwd <- Sys.getenv("R_KEYRING_PASSWORD")
+
+ # Create keyring only if missing
+ existing_kr <- tryCatch(keyring::keyring_list()$keyring, error = function(e) character(0))
+ if (!(kr_name %in% existing_kr)) {
+ keyring::keyring_create(kr_name, password = kr_pwd)
+ }
+
+ # Unlock if locked (non-interactive using env password)
+ if (keyring::keyring_is_locked(kr_name)) {
+ keyring::keyring_unlock(kr_name, password = kr_pwd)
+ }
+
+ # Store credentials for appeears token refresh
+ suppressMessages(appeears::rs_set_key(user = earthdata_user, password = earthdata_pass))
+
+ # Authenticate (reads from keyring)
+ rstoken <- appeears::rs_login(earthdata_user)
+ message("AppEEARS authentication configured")
+} else {
+ warning("EARTHDATA credentials not found. Set EARTHDATA_USER and EARTHDATA_PASSWORD environment variables.")
+}
diff --git a/R/ccr_startup.sh b/R/ccr_startup.sh
new file mode 100644
index 00000000..6a187c46
--- /dev/null
+++ b/R/ccr_startup.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+
+## Script to launch an interactive Apptainer session on CCR
+
+ssh vortex.ccr.buffalo.edu
+salloc --cluster=faculty --qos=adamw --partition=adamw \
+ --job-name=InteractiveJob --nodes=1 --ntasks=4 \
+ --mem=10G -C INTEL --time=24:00:00
+
+
+export GROUP="adamw"
+export PROJECT_FOLDER="/projects/academic/"$GROUP
+export APPTAINER_CACHEDIR="/vscratch/grp-adamw/"$USER"/singularity"
+export SIF_PATH=$PROJECT_FOLDER"/users/"$USER"/singularity"
+export SIF_FILE="AdamWilsonLab-emma_docker-latest.sif"
+
+# set singularity cache and tmp directories to the same as apptainer
+# needed because CCR is still using singularity and it will use these directories
+export SINGULARITY_CACHEDIR=$APPTAINER_CACHEDIR
+export SINGULARITY_TMPDIR=$APPTAINER_TMPDIR
+export SINGULARITY_LOCALCACHEDIR=$APPTAINER_LOCALCACHEDIR
+
+
+apptainer run \
+ --bind $PROJECT_FOLDER:$PROJECT_FOLDER \
+ --bind $APPTAINER_CACHEDIR/tmp:/tmp \
+ --bind $APPTAINER_CACHEDIR/run:/run \
+ $SIF_PATH/$SIF_FILE R
diff --git a/R/data_chelsa.R b/R/data_chelsa.R
new file mode 100644
index 00000000..b00b86fe
--- /dev/null
+++ b/R/data_chelsa.R
@@ -0,0 +1,195 @@
+#R script to download climate data (CHELSA)
+
+library(terra)
+library(ncdf4)
+
+#' @author Brian Maitner & Adam Wilson
+#' @description This function will download CHELSA climate data if it isn't present, and (invisibly) return a NULL if it is present
+#' @param temp_directory Where to save the files, defaults to "data/raw_data/climate_chelsa/"
+#' @param domain domain (sf polygon) used for masking
+#' @param tag Tag for the release
+#' @param cleanup Logical. If TRUE (default, for GitHub Actions), clean temp directory before/after. If FALSE (local development), preserve cached files.
+#' @import terra
+get_chelsa <- function(temp_directory = "data/temp/raw_data/climate_chelsa/",
+ tag = "raw_static",
+ domain,
+ cleanup = TRUE){
+
+ #ensure temp directory is empty only if cleanup mode is enabled
+
+ if(cleanup && dir.exists(temp_directory)){
+ unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
+ }
+
+ #make a directory if one doesn't exist yet
+
+ if(!dir.exists(temp_directory)){
+ dir.create(temp_directory,recursive = TRUE)
+ }
+
+
+ #Make sure there is a release by attempting to create one. If it already exists, this will fail
+
+ tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
+ tag = tag),
+ error = function(e){message("Previous release found")})
+
+ #Adjust the download timeout duration (this needs to be large enough to allow the download to complete)
+
+ if(getOption('timeout') < 1000){
+ options(timeout = 1000)
+ }
+
+
+ #Transform domain to wgs84 to get the coordinates
+
+ # domain_extent <-
+ # domain %>%
+ # st_transform(crs("+proj=longlat +datum=WGS84 +no_defs +ellps=WGS84 +towgs84=0,0,0")@projargs)%>%
+ # extent()
+
+ domain_tf <-
+ domain %>%
+ st_as_sf() %>%
+ sf::st_transform(crs("+proj=longlat +datum=WGS84 +no_defs +ellps=WGS84 +towgs84=0,0,0"))
+
+ # Download the data
+ # Note that it would be useful to clip these to a polygon to save space
+ # It would also be useful if only the relevant data could be downloaded (rather than downloading and THEN pruning)
+
+ # CF-compliant metadata for CHELSA bioclimatic variables
+ bio_metadata <- tribble(
+ ~bio_name, ~long_name, ~units,
+ "bio01", "Annual Mean Temperature", "Β°C * 10",
+ "bio02", "Mean Diurnal Range", "Β°C * 10",
+ "bio03", "Isothermality", "%",
+ "bio04", "Temperature Seasonality", "Β°C * 10",
+ "bio05", "Max Temperature of Warmest Month", "Β°C * 10",
+ "bio06", "Min Temperature of Coldest Month", "Β°C * 10",
+ "bio07", "Temperature Annual Range", "Β°C * 10",
+ "bio08", "Mean Temperature of Wettest Quarter", "Β°C * 10",
+ "bio09", "Mean Temperature of Driest Quarter", "Β°C * 10",
+ "bio10", "Mean Temperature of Warmest Quarter", "Β°C * 10",
+ "bio11", "Mean Temperature of Coldest Quarter", "Β°C * 10",
+ "bio12", "Annual Precipitation", "mm",
+ "bio13", "Precipitation of Wettest Month", "mm",
+ "bio14", "Precipitation of Driest Month", "mm",
+ "bio15", "Precipitation Seasonality", "%",
+ "bio16", "Precipitation of Wettest Quarter", "mm",
+ "bio17", "Precipitation of Driest Quarter", "mm",
+ "bio18", "Precipitation of Warmest Quarter", "mm",
+ "bio19", "Precipitation of Coldest Quarter", "mm"
+ )
+
+ # Record download date
+ download_date <- Sys.Date()
+
+ for(idx in 1:nrow(bio_metadata)){
+ i <- bio_metadata$bio_name[idx]
+
+ # Construct filename
+ tif_filename <- file.path(temp_directory, paste("CHELSA_bio", sprintf("%02d", idx), "_1981-2010_V.2.1.tif", sep = ""))
+
+ # Skip download if file already exists (when cleanup = FALSE, running locally)
+ if (!cleanup && file.exists(tif_filename)) {
+ message("File already exists, skipping download: ", basename(tif_filename))
+ } else {
+ # Download the file
+ robust_download_file(
+ url = paste("https://os.unil.cloud.switch.ch/chelsa02/chelsa/global/bioclim/", i, "/1981-2010/CHELSA_bio", sprintf("%02d", idx), "_1981-2010_V.2.1.tif", sep = ""),
+ destfile = tif_filename,
+ max_attempts = 10,
+ sleep_time = 10
+ )
+ }
+
+ # load
+ rast_i <- terra::rast(tif_filename)
+
+ # crop
+
+ rast_i <- terra::crop(x = rast_i,
+ y = ext(domain_tf))
+
+ # mask
+ rast_i <-
+ terra::mask(rast_i,
+ mask = terra::vect(domain_tf))
+
+ # Write as NetCDF with CF-compliant metadata
+ nc_filename <- file.path(temp_directory, paste("CHELSA_", i, "_1981-2010_V.2.1.nc", sep = ""))
+
+ # Use terra's writeCDF function which creates NetCDF4 files
+ terra::writeCDF(x = rast_i,
+ filename = nc_filename,
+ overwrite = TRUE,
+ compression = 9)
+
+ # Add CF-compliant metadata using ncdf4 package
+ nc_file <- ncdf4::nc_open(nc_filename, write = TRUE)
+
+ # Get variable name (should be the first variable in the file)
+ var_name <- names(rast_i)
+ if (is.null(var_name) || var_name == "") {
+ var_name <- i
+ }
+
+ # Get metadata for this bioclimatic variable
+ long_name <- bio_metadata$long_name[idx]
+ units <- bio_metadata$units[idx]
+
+ # Add global attributes
+ ncdf4::ncatt_put(nc_file, 0, "title",
+ paste("CHELSA Bioclimatic Variable", i, sep = " "))
+ ncdf4::ncatt_put(nc_file, 0, "source", "CHELSA v.2.1 (Climatologies at high resolution for the earth land areas)")
+ ncdf4::ncatt_put(nc_file, 0, "dataset_url", "https://chelsa-climate.org/")
+ ncdf4::ncatt_put(nc_file, 0, "download_date", as.character(download_date))
+ ncdf4::ncatt_put(nc_file, 0, "temporal_range", "1981-2010")
+ ncdf4::ncatt_put(nc_file, 0, "Conventions", "CF-1.8")
+ ncdf4::ncatt_put(nc_file, 0, "history",
+ paste("Downloaded on", as.character(download_date),
+ "and clipped to domain. Processed using terra and ncdf4 R packages. "))
+
+ # Add variable attributes (long_name and units)
+ ncdf4::ncatt_put(nc_file, 1, "long_name", long_name)
+ ncdf4::ncatt_put(nc_file, 1, "units", units)
+ ncdf4::ncatt_put(nc_file, 1, "standard_name", paste("bioclimatic_variable_", i, sep = ""))
+
+ ncdf4::nc_close(nc_file)
+
+ rm(rast_i)
+
+ }
+
+ rm(i)
+
+ # release
+ to_release <-
+ list.files(path = file.path(temp_directory),
+ recursive = TRUE,
+ full.names = TRUE)
+
+
+ to_release <-
+ to_release[grepl(pattern = "CHELSA",
+ ignore.case = TRUE,
+ x = basename(to_release))]
+
+ # Filter for NetCDF files only
+ to_release <- to_release[grepl(pattern = "\\.nc$", x = to_release)]
+
+ # pb_upload(repo = "AdamWilsonLab/emma_envdata",
+ # file = to_release,
+ # tag = tag)
+
+ # delete directory and contents
+ # unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
+
+
+
+ message("CHELSA climate files downloaded")
+ return(Sys.Date())
+
+
+} # end fx
+
diff --git a/R/data_vegmap.R b/R/data_vegmap.R
new file mode 100644
index 00000000..7ac34596
--- /dev/null
+++ b/R/data_vegmap.R
@@ -0,0 +1,165 @@
+## Process Vegmap to add field related to biome type
+
+data_vegmap <- function(domain_raster,
+ vegmap_shp,
+ disagg_factor = 10,
+ out_file = "data/raw/vegmap.nc") {
+
+ # Load domain raster (may be passed as file path or raster object)
+ domain <- if (is.character(domain_raster)) {
+ # If it's a file path, load the 'domain' variable from NetCDF
+ rast(domain_raster, subds = "domain")
+ } else {
+ domain_raster
+ }
+
+ # Load and prep vegmap
+ vegmap_sf <- st_read(vegmap_shp, quiet = TRUE) %>%
+ janitor::clean_names() %>%
+ st_make_valid() %>%
+ st_transform(st_crs(domain, proj = TRUE)) %>%
+ st_intersection(st_as_sfc(st_bbox(domain))) |> #crop to domain
+ mutate(t_vegtypeid = as.numeric(factor(t_mapcode))) #create numeric vegtype ID
+
+ # Create fine template to reduce sliver effects, then modal aggregate back
+ template_fine <- disagg(domain, disagg_factor)
+
+ rasterize_modal <- function(field_name) {
+ r_fine <- terra::rasterize(
+ x = vect(vegmap_sf),
+ y = template_fine,
+ field = field_name,
+ touches = TRUE
+ )
+ aggregated <- terra::aggregate(r_fine, disagg_factor, fun = "modal")
+ # Resample to exactly match domain grid (nearest neighbor for categorical)
+ terra::resample(aggregated, domain, method = "near")
+ }
+
+ biome_raster <- rasterize_modal("t_biomeid")
+ bioregion_raster <- rasterize_modal("t_brgnid")
+ vegtype_raster <- rasterize_modal("t_vegtypeid")
+
+ # Combine into multiband raster
+ multiband <- c(biome_raster, bioregion_raster, vegtype_raster)
+ names(multiband) <- c("vegbiome", "vegbioregion", "vegtype")
+
+ # Mask to domain (set to NA where domain is NA)
+ domain_mask <- !is.na(domain)
+ multiband <- terra::mask(multiband, domain_mask, maskvalues = 0)
+
+ # Lookup table for IDs -> names
+ lookup_tbl <- vegmap_sf %>%
+ st_drop_geometry() %>%
+ dplyr::select(t_biomeid, t_brgnid, t_vegtypeid, t_mapcode, t_name, t_biome, t_bioregio) %>%
+ dplyr::rename(vegbiome = t_biomeid, vegbioregion = t_brgnid, vegtype = t_vegtypeid) %>%
+ dplyr::distinct()
+
+ # Create output file path (ensure directory exists)
+ output_file <- out_file
+ dir.create(dirname(output_file), recursive = TRUE, showWarnings = FALSE)
+
+ # Get spatial extent and resolution for dimensions
+ ext <- ext(multiband)
+ dx <- res(multiband)[1]
+ dy <- res(multiband)[2]
+
+ x_vals <- seq(ext$xmin + dx/2, ext$xmax - dx/2, by = dx)
+ y_vals <- seq(ext$ymax - dy/2, ext$ymin + dy/2, by = -dy)
+
+ # Define dimensions with coordinate vectors
+ dim_x <- ncdf4::ncdim_def(name = "easting", units = "meter", vals = x_vals, longname = "easting")
+ dim_y <- ncdf4::ncdim_def(name = "northing", units = "meter", vals = y_vals, longname = "northing")
+
+ # Define variables with compression level 9 for categorical data (storage as short integers)
+ var_biome <- ncdf4::ncvar_def(
+ name = "vegbiome",
+ units = "dimensionless",
+ dim = list(dim_x, dim_y),
+ longname = "Biome ID (biomeid_18)",
+ missval = -32768,
+ prec = "short",
+ compression = 9
+ )
+
+ var_bioregion <- ncdf4::ncvar_def(
+ name = "vegbioregion",
+ units = "dimensionless",
+ dim = list(dim_x, dim_y),
+ longname = "Bioregion ID (brgnid_18)",
+ missval = -32768,
+ prec = "short",
+ compression = 9
+ )
+
+ var_vegtype <- ncdf4::ncvar_def(
+ name = "vegtype",
+ units = "dimensionless",
+ dim = list(dim_x, dim_y),
+ longname = "Vegetation type code (mapcode18)",
+ missval = -32768,
+ prec = "short",
+ compression = 9
+ )
+
+ # Create NetCDF file with all variables
+ unlink(output_file)
+ nc <- ncdf4::nc_create(
+ filename = output_file,
+ vars = list(var_biome, var_bioregion, var_vegtype),
+ force_v4 = TRUE
+ )
+
+ # Convert rasters to matrices, transpose to match dimension order, replace NAs with fill values
+ biome_matrix <- t(as.matrix(biome_raster, wide = TRUE))
+ biome_matrix <- as.integer(biome_matrix)
+ biome_matrix[is.na(biome_matrix)] <- -32768
+
+ bioregion_matrix <- t(as.matrix(bioregion_raster, wide = TRUE))
+ bioregion_matrix <- as.integer(bioregion_matrix)
+ bioregion_matrix[is.na(bioregion_matrix)] <- -32768
+
+ vegtype_matrix <- t(as.matrix(vegtype_raster, wide = TRUE))
+ vegtype_matrix <- as.integer(vegtype_matrix)
+ vegtype_matrix[is.na(vegtype_matrix)] <- -32768
+
+ # Write data to variables
+ ncdf4::ncvar_put(nc, var_biome, biome_matrix)
+ ncdf4::ncvar_put(nc, var_bioregion, bioregion_matrix)
+ ncdf4::ncvar_put(nc, var_vegtype, vegtype_matrix)
+
+ # Add global attributes
+ ncdf4::ncatt_put(nc, 0, "title", "Vegetation Map - Biome, Bioregion, and Vegetation Type")
+ ncdf4::ncatt_put(nc, 0, "source", "South Africa National Vegetation Map 2024")
+ ncdf4::ncatt_put(nc, 0, "history", paste0("created: ", Sys.time()))
+ ncdf4::ncatt_put(nc, 0, "crs", as.character(crs(multiband)))
+ ncdf4::ncatt_put(nc, 0, "Conventions", "CF-1.8")
+ ncdf4::ncatt_put(nc, 0, "lookup_table_json", jsonlite::toJSON(lookup_tbl, dataframe = "rows", auto_unbox = TRUE))
+
+ # Add CRS variable for CF compliance and GIS compatibility
+ crs_var <- ncdf4::ncvar_def("crs", "", list(), prec = "integer")
+ nc <- ncdf4::ncvar_add(nc, crs_var)
+
+ crs_wkt <- as.character(crs(multiband))
+# ncdf4::ncatt_put(nc, "crs", "grid_mapping_name", "albers_conical_equal_area")
+ ncdf4::ncatt_put(nc, "crs", "crs_wkt", crs_wkt)
+ ncdf4::ncatt_put(nc, "crs", "spatial_ref", crs_wkt)
+
+ # Add geotransform for GDAL compatibility
+ ncdf4::ncatt_put(nc, "crs", "GeoTransform", paste(ext$xmin, dx, 0, ext$ymax, 0, -dy))
+
+ # Add grid_mapping to data variables
+ ncdf4::ncatt_put(nc, "vegbiome", "grid_mapping", "crs")
+ ncdf4::ncatt_put(nc, "vegbioregion", "grid_mapping", "crs")
+ ncdf4::ncatt_put(nc, "vegtype", "grid_mapping", "crs")
+
+ # Close file
+ ncdf4::nc_close(nc)
+
+ return(output_file)
+}
+
+if(F){
+ test=rast(output_file)
+ plot(test)
+}
\ No newline at end of file
diff --git a/R/domain_bbox.R b/R/domain_bbox.R
new file mode 100644
index 00000000..ff64d241
--- /dev/null
+++ b/R/domain_bbox.R
@@ -0,0 +1,34 @@
+# Create buffered domain bounding box and write GeoParquet
+
+#' Create a buffered bounding box around the domain
+#'
+#' @param domain_parquet Path to domain polygon GeoParquet (output of domain_define).
+#' @param buffer_m Numeric buffer distance in meters applied to the bbox (default 50 km).
+#' @param out_file Output GeoParquet path (default data/raw/domain_bbox.parquet).
+#' @return Character path to the written GeoParquet file.
+#' @details Reads the domain polygon, builds its bounding box, converts to sf, buffers, and writes to GeoParquet.
+make_domain_bbox <- function(domain_parquet, buffer_m = 50000, out_file = "data/target_outputs/domain_bbox.parquet") {
+ domain_sf <- sfarrow::st_read_parquet(domain_parquet)
+ bbox_geom <- domain_sf |> sf::st_bbox() |> sf::st_as_sfc() |> sf::st_buffer(buffer_m)
+ bbox_sf <- sf::st_as_sf(bbox_geom)
+
+ # Ensure directory exists
+ dir.create(dirname(out_file), recursive = TRUE, showWarnings = FALSE)
+
+ # Try writing with sfarrow, fall back to sf::st_write if needed
+ tryCatch({
+ sfarrow::st_write_parquet(bbox_sf, out_file)
+ message("[domain_bbox] Wrote with sfarrow: ", out_file)
+ }, error = function(e) {
+ message("[domain_bbox] sfarrow failed, trying sf::st_write: ", conditionMessage(e))
+ sf::st_write(bbox_sf, out_file, quiet = TRUE, delete_dsn = TRUE)
+ })
+
+ # Verify file exists
+ if (!file.exists(out_file)) {
+ stop("[domain_bbox] Failed to create output file: ", out_file)
+ }
+
+ message("[domain_bbox] File created: ", out_file, " (", file.size(out_file), " bytes)")
+ out_file
+}
diff --git a/R/domain_define.R b/R/domain_define.R
index 44659341..2c8d4659 100644
--- a/R/domain_define.R
+++ b/R/domain_define.R
@@ -1,54 +1,46 @@
-# Make Domain
+# Define buffered project domain
#' @author Adam M. Wilson
+#' @description Build a smoothed, buffered domain polygon from the 2018 vegetation map and country boundary, then write it to GeoParquet.
+#' @param vegmap_shp Path to the vegetation map shapefile (e.g., VEGMAP2018).
+#' @param country Path to the country boundary GeoParquet file.
+#' @return Character path to the written GeoParquet file (`data/raw/domain_boundary.parquet`).
+#' @details Filters to target biomes, unions polygons, simplifies (500 m), buffers (50 km), smooths (ksmooth, smoothness=120), intersects with country, and writes to GeoParquet.
-# Process 2018 Vegetation dataset to define project domain
+domain_define <- function(vegmap_shp, country){
-
-#' @param vegmap is the domains of interest from the 2018 national vegetation map
-#' @param vegmap_shp is the path to the 2018 national vegetation map - used to get national boundary
-#' @param buffer size of domain buffer (in m)
-
-domain_define <- function(vegmap, country){
+ # Always read country from a parquet path
+ if (!is.character(country)) {
+ stop("country must be a path to a GeoParquet file")
+ }
+ country <- sfarrow::st_read_parquet(country)
biomes = c("Fynbos")#,"Succulent Karoo")#,"Albany Thicket")
- vegmap_union=vegmap %>%
- filter(biome_18 %in% biomes ) %>% #filter to list above
+ vegmap_union=st_read(vegmap_shp) %>%
+ janitor::clean_names() %>%
+ filter(t_biome %in% biomes ) %>% #filter to list above
st_union() # union all polygons into one multipolygon, dissolving internal boundaries
-
- #buffer domain biomes
- vegmap_buffer= vegmap_union %>%
- st_simplify(dTolerance=500) %>%
- st_buffer(50000) %>%
- st_simplify(dTolerance=100)
-
-
-
-
+
+vegmap_buffer = vegmap_union %>%
+ st_simplify(dTolerance=500) %>%
+ st_buffer(50000) %>%
+ smoothr::smooth(method="ksmooth",smoothness=120) #%>%
+
+country= st_as_sf(country) %>%
+ st_transform(crs=st_crs(vegmap_buffer))
+
domain <-
vegmap_buffer %>%
- st_intersection(st_transform(country,crs=st_crs(vegmap))) %>% #only keep land areas of buffer - no ocean
+ st_intersection(st_transform(country,crs=st_crs(vegmap_union))) %>% #only keep land areas of buffer - no ocean
st_as_sf() %>%
mutate(domain=1)
- #Delete file if it exists (this should be handled by the line below, but that seems to fail on github)
- if(file.exists("data/domain.gpkg")){file.remove("data/domain.gpkg")}
-
- # save the files
- st_write(domain,dsn="data/domain.gpkg", append = F, delete_layer = TRUE)# overwrite layer if it exists (caused an error in targets)
-
- #release the domain
- piggyback::pb_upload(file = "data/domain.gpkg",
- repo = "AdamWilsonLab/emma_envdata",
- tag = "raw_static",
- overwrite = TRUE)
-
- return(domain)
-
-}
-
-
-
+ # Write to GeoParquet
+ out_file <- "data/raw/domain_boundary.parquet"
+ sfarrow::st_write_parquet(domain, out_file)
+
+ return(out_file)
+}
\ No newline at end of file
diff --git a/R/domain_define_bioscape.R b/R/domain_define_bioscape.R
new file mode 100644
index 00000000..5ce32bc4
--- /dev/null
+++ b/R/domain_define_bioscape.R
@@ -0,0 +1,52 @@
+# Make Domain
+
+#' @author Adam M. Wilson
+
+# Process 2018 Vegetation dataset to define project domain
+
+
+#' @param vegmap is the domains of interest from the 2018 national vegetation map
+#' @param vegmap_shp is the path to the 2018 national vegetation map - used to get national boundary
+#' @param buffer size of domain buffer (in m)
+
+domain_define_bioscape <- function(vegmap, country){
+
+ require(smoothr)
+
+ biomes = c("Fynbos")#,"Succulent Karoo")#,"Albany Thicket")
+
+
+ vegmap_union=vegmap %>%
+ filter(biome_18 %in% biomes ) %>% #filter to list above
+ st_union() # union all polygons into one multipolygon, dissolving internal boundaries
+
+ #buffer domain biomes
+ vegmap_buffer= vegmap_union %>%
+ st_simplify(dTolerance=500) %>%
+ st_buffer(set_units(set_units(100,km),m)) #%>%
+
+# v2<- vegmap_buffer %>%
+# smooth(method = "ksmooth",smoothness=25)
+#
+# plot(vegmap_union); plot(v2,add=T)
+
+ domain <-
+ vegmap_buffer %>%
+ smooth(method = "ksmooth",smoothness=50) %>%
+# st_intersection(st_transform(country,crs=st_crs(vegmap))) %>% #only keep land areas of buffer - no ocean
+ st_as_sf() %>%
+ mutate(domain=1) %>%
+ st_make_valid() %>%
+ st_transform(4326)
+
+ # save the files
+ st_write(domain,dsn="data/bioscape_domain.gpkg",append=F)
+ file.remove("data/bioscape_domain.geojson"); st_write(domain,dsn="data/bioscape_domain.geojson",append=F)
+
+ return(domain)
+
+}
+
+
+
+
diff --git a/R/domain_rasterize.R b/R/domain_rasterize.R
index 862c3e15..f48f9512 100644
--- a/R/domain_rasterize.R
+++ b/R/domain_rasterize.R
@@ -1,28 +1,194 @@
-# Rasterize Domain
+# Rasterize Domain and Remnants
+#' @title Rasterize domain with pixel IDs, remnants, and distance to remnants
#' @author Adam M. Wilson
+#' @description Creates a multivariate NetCDF file with four variables: domain mask, pixel IDs, remnant indicators, and distance to nearest remnant. Each variable is written separately with maximum compression and CF-1.8 compliant metadata.
+#' @param domain sf or SpatVector object defining the study area (typically from domain_define).
+#' @param remnants_shp Path to remnant vegetation shapefile.
+#' @param dx Numeric x-resolution in CRS units (default 500 m).
+#' @param dy Numeric y-resolution in CRS units (default 500 m).
+#' @param out_file Character path for output NetCDF file (default "data/raw/domain.nc").
+#' @return Character path to the written NetCDF file.
+#' @details Generates a raster template from domain bounding box, rasterizes domain and remnants, computes pixel IDs (sequential within domain) and Euclidean distance to nearest remnant (in km). Writes four variables (domain, pid, remnants, remnants_distance) to NetCDF with FORMAT=NC4, COMPRESS=DEFLATE, ZLEVEL=9, SHUFFLE=YES. Adds CF-compliant metadata via ncdf4 including long_name, units, CRS, history, and Conventions attributes.
-# Rasterize domain to common grid to define the raster domain
+domain_rasterize <- function(domain, remnants_shp, dx = 500, dy = 500, out_file = "data/raw/domain.nc") {
-#' @param domain vector file of study domain
-#' @param dx x resolution
-#' @param dy y resolution
+ # Generate raster template and rasterize domain with terra (touches = TRUE)
+ domain_template <- rast(st_as_stars(st_bbox(domain), dx = dx, dy = dy))
-domain_rasterize <- function(domain,dx = 500, dy = 500){
+ domain_raster <- domain %>%
+ st_as_sf() %>%
+ mutate(domain = 1) %>%
+ vect() %>%
+ terra::rasterize(
+ x = .,
+ y = domain_template,
+ field = "domain",
+ touches = TRUE
+ )
+
+ # Ensure pixels outside domain are NA (not 0)
+ domain_raster[domain_raster == 0] <- NA
- # generate raster version of domain
- domain_template=st_as_stars(st_bbox(domain), dx = dx, dy = dy)
- domain_raster <- domain %>%
- dplyr::select(biomeid_18) %>%
- st_rasterize(template = domain_template)
+## Process remnants to add fields related to whether the cell is in a remnant and distance to remnant
- domain_raster
- #writeRaster(domain_raster,file="data/domain.tif")
+# Load remnants file
+ remnants <- st_read(remnants_shp) %>%
+ janitor::clean_names() %>%
+ st_transform(crs = crs(domain)) %>%
+ st_crop(st_as_sfc(st_bbox(domain))) #crop to domain box
+# st_union() %>%
+# st_make_valid()
-}
+ remnants_raster <- remnants %>%
+ st_as_sf() |>
+ mutate(remnant=1) %>%
+ vect() %>%
+ rasterize(x = .,
+ y = domain_raster,
+ field = "remnant",
+ touches = T,
+ cover = T)|>
+ terra::mask(mask=domain_raster)*100 #set to NA outside domain and convert to integer
+
+ remnants_distance <- remnants_raster |>
+ terra::app(fun=function(x) ifelse(is.na(x),0,1)) |>
+ terra::distance(target=1)|>
+ terra::mask(mask=domain_raster) #set to NA outside domain
+
+ # Create pixel ID raster: 1:ncell where domain=1, NA elsewhere
+ pid_raster <- domain_raster
+ pid_values <- rep(NA, ncell(pid_raster))
+ domain_cells <- which(!is.na(values(domain_raster)))
+ pid_values[domain_cells] <- seq_along(domain_cells)
+ values(pid_raster) <- pid_values
+ # Prepare layers for per-variable write
+ layers <- list(
+ domain = domain_raster,
+ pid = pid_raster,
+ remnants = remnants_raster,
+ remnants_distance = remnants_distance
+ )
+
+ # Get spatial extent and create dimensions for NetCDF
+ ext <- ext(domain_raster)
+ x_vals <- seq(ext$xmin + dx/2, ext$xmax - dx/2, by = dx)
+ y_vals <- seq(ext$ymax - dy/2, ext$ymin + dy/2, by = -dy)
+
+ # Define dimensions
+ dim_x <- ncdf4::ncdim_def(name = "easting", units = "meter", vals = x_vals, longname = "easting")
+ dim_y <- ncdf4::ncdim_def(name = "northing", units = "meter", vals = y_vals, longname = "northing")
+
+ # Define variables with optimal data types and compression
+ var_domain <- ncdf4::ncvar_def(
+ name = "domain",
+ units = "dimensionless",
+ dim = list(dim_x, dim_y),
+ longname = "Domain mask (1 = in domain, NA = outside)",
+ missval = -128,
+ prec = "byte",
+ compression = 9
+ )
+
+ var_pid <- ncdf4::ncvar_def(
+ name = "pid",
+ units = "dimensionless",
+ dim = list(dim_x, dim_y),
+ longname = "Pixel ID for domain grid cells",
+ missval = -2147483648,
+ prec = "integer",
+ compression = 9
+ )
+
+ var_remnants <- ncdf4::ncvar_def(
+ name = "remnants",
+ units = "dimensionless",
+ dim = list(dim_x, dim_y),
+ longname = "Remnant vegetation proportion (100 = full remnant, 5 = 5% remnant, NA = not remnant)",
+ missval = -128,
+ prec = "byte",
+ compression = 9
+ )
+
+ var_dist <- ncdf4::ncvar_def(
+ name = "remnants_distance",
+ units = "meters",
+ dim = list(dim_x, dim_y),
+ longname = "Distance to nearest remnant vegetation",
+ missval = -2147483648,
+ prec = "integer",
+ compression = 9
+ )
+
+ # Create NetCDF file with all variables
+unlink(out_file)
+
+ nc <- ncdf4::nc_create(
+ filename = out_file,
+ vars = list(var_domain, var_pid, var_remnants, var_dist),
+ force_v4 = TRUE
+ )
+
+ # Convert rasters to matrices and replace NAs with fill values
+ # Note: as.matrix() from terra returns (nrow, ncol), but ncdf4 expects (ncol, nrow) for (x, y) dims
+ domain_matrix <- t(as.matrix(layers$domain, wide = TRUE))
+ domain_matrix[is.na(domain_matrix)] <- -128
+
+ pid_matrix <- t(as.matrix(layers$pid, wide = TRUE))
+ pid_matrix[is.na(pid_matrix)] <- -2147483648
+
+ remnants_matrix <- t(as.matrix(layers$remnants, wide = TRUE))
+ remnants_matrix[is.na(remnants_matrix)] <- -128
+
+ dist_matrix <- t(as.matrix(layers$remnants_distance, wide = TRUE))
+ dist_matrix[is.na(dist_matrix)] <- -2147483648
+
+ # Write data to variables
+ ncdf4::ncvar_put(nc, var_domain, domain_matrix)
+ ncdf4::ncvar_put(nc, var_pid, pid_matrix)
+ ncdf4::ncvar_put(nc, var_remnants, remnants_matrix)
+ ncdf4::ncvar_put(nc, var_dist, dist_matrix)
+
+ # Add global attributes
+ ncdf4::ncatt_put(nc, 0, "title", "Rasterized domain with remnants and distance")
+ ncdf4::ncatt_put(nc, 0, "history", paste0("created: ", Sys.time()))
+ ncdf4::ncatt_put(nc, 0, "Conventions", "CF-1.8")
+
+ # Add CRS variable with comprehensive attributes for GIS compatibility
+ crs_var <- ncdf4::ncvar_def("crs", "", list(), prec = "integer")
+ nc <- ncdf4::ncvar_add(nc, crs_var)
+
+ # Get CRS as WKT string (most reliable for terra)
+ crs_wkt <- as.character(crs(domain_raster))
+
+ # Add CRS attributes
+ ncdf4::ncatt_put(nc, "crs", "grid_mapping_name", "albers_conical_equal_area")
+ ncdf4::ncatt_put(nc, "crs", "crs_wkt", crs_wkt)
+ ncdf4::ncatt_put(nc, "crs", "spatial_ref", crs_wkt)
+
+ # Add geotransform for GDAL compatibility
+ ext_vals <- ext(domain_raster)
+ geotransform <- paste(ext_vals$xmin, dx, 0, ext_vals$ymax, 0, -dy)
+ ncdf4::ncatt_put(nc, "crs", "GeoTransform", geotransform)
+
+ # Add grid_mapping attribute to all data variables
+ ncdf4::ncatt_put(nc, "domain", "grid_mapping", "crs")
+ ncdf4::ncatt_put(nc, "pid", "grid_mapping", "crs")
+ ncdf4::ncatt_put(nc, "remnants", "grid_mapping", "crs")
+ ncdf4::ncatt_put(nc, "remnants_distance", "grid_mapping", "crs")
+
+ # Close file
+ ncdf4::nc_close(nc)
+
+ out_file
+}
+if(F){
+test=rast(out_file)
+plot(test$domain)
+}
\ No newline at end of file
diff --git a/R/domain_to_geoparquet.R b/R/domain_to_geoparquet.R
new file mode 100644
index 00000000..8bcf576f
--- /dev/null
+++ b/R/domain_to_geoparquet.R
@@ -0,0 +1,44 @@
+#' @title Convert domain raster to geoparquet
+#' @description Converts the domain raster (with pid and coordinates) to a geoparquet file
+#' for use as a spatial reference for dynamic data
+#' @param domain_raster_file Path to domain NetCDF file (from domain_rasterize)
+#' @param out_file Output geoparquet file path
+#' @param verbose Logical for progress messages
+#' @return Character path to the written geoparquet file
+#' @details Extracts the pid layer from the domain raster, converts to a point geometry
+#' with coordinates, and writes as a geoparquet file for efficient spatial queries
+#' @import terra
+#' @import sf
+#' @import sfarrow
+
+domain_to_geoparquet <- function(
+ domain_raster_file,
+ out_file = "data/target_outputs/domain.parquet",
+ verbose = TRUE
+) {
+
+ if (verbose) message("Loading domain raster from: ", domain_raster_file)
+
+ # Load the pid layer from the domain raster
+ domain_raster <- terra::rast(domain_raster_file, subds = "pid")
+
+ # Convert raster to dataframe with coordinates
+ # This gives us x, y coordinates and the pid value for each non-NA cell
+ df <- terra::as.data.frame(domain_raster, xy = TRUE, na.rm = TRUE)
+ colnames(df) <- c("x", "y", "pid")
+
+ # Convert to spatial points dataframe
+ sp_df <- sf::st_as_sf(df, coords = c("x", "y"), crs = terra::crs(domain_raster))
+
+ if (verbose) message("Converting to geoparquet: ", nrow(sp_df), " pixels")
+
+ # Ensure output directory exists
+ dir.create(dirname(out_file), recursive = TRUE, showWarnings = FALSE)
+
+ # Write as geoparquet using sfarrow
+ sfarrow::st_write_parquet(sp_df, out_file)
+
+ if (verbose) message("Domain geoparquet written to: ", out_file)
+
+ invisible(out_file)
+}
diff --git a/R/download_vegmap_release.R b/R/download_vegmap_release.R
new file mode 100644
index 00000000..a3648164
--- /dev/null
+++ b/R/download_vegmap_release.R
@@ -0,0 +1,46 @@
+#' Download vegetation map release and return shapefile path
+#'
+#' Downloads a zipped vegetation map from a GitHub release using piggyback,
+#' unzips it into a local directory, and returns the path to the `.shp` file
+#' for use in targets.
+#'
+#' @param repo character. GitHub repository in "owner/repo" format.
+#' @param tag character. Release tag (e.g., "latest" or "v1.0.0").
+#' @param file character. Name of the release asset file to download (e.g., "vegmap.zip").
+#' @param local_dir character. Local directory to store the unzipped files.
+#' @param shapefile_name character. Name of the shapefile to return (e.g., "NVM2024Final_IEM5_12_07012025.shp").
+#'
+#' @return character. Full path to the shapefile on disk.
+#'
+#' @details If the shapefile already exists at `local_dir`, no download
+#' is performed. Otherwise, the zip file is downloaded via piggyback and extracted.
+#'
+#'
+#' @importFrom piggyback pb_download
+#' @importFrom utils unzip
+#' @export
+download_vegmap_release <- function(repo, tag, file, local_dir, shapefile_name) {
+ dir.create(local_dir, recursive = TRUE, showWarnings = FALSE)
+
+ # Check if shapefile already exists in the expected location (with shapefile subdirectory)
+ shp_file_correct <- file.path(local_dir, "shapefile", shapefile_name)
+
+ if (!file.exists(shp_file_correct)) {
+ message("Downloading vegmap from GitHub release using piggyback...")
+ piggyback::pb_download(
+ file = file,
+ repo = repo,
+ tag = tag,
+ dest = local_dir,
+ overwrite = FALSE
+ )
+ message("Unzipping vegmap...")
+ list.files(local_dir,recursive = T)
+ zip_file <- file.path(local_dir, file)
+ utils::unzip(zip_file, exdir = local_dir)
+ unlink(zip_file)
+ }
+
+ # Return the correct path (shapefile is in a subdirectory after extraction)
+ shp_file_correct
+}
diff --git a/R/functions.R b/R/functions.R
index 18e71e6f..353e7f47 100644
--- a/R/functions.R
+++ b/R/functions.R
@@ -1,3 +1,33 @@
+# Load all packages listed in DESCRIPTION file (quietly, suppressing startup messages)
+load_description_packages <- function(description_file = "DESCRIPTION", quietly = TRUE, verbose = FALSE) {
+ # Read DESCRIPTION file
+ dcf <- read.dcf(description_file)
+
+ # Extract Imports and Depends fields
+ imports <- if ("Imports" %in% colnames(dcf)) dcf[1, "Imports"] else ""
+ depends <- if ("Depends" %in% colnames(dcf)) dcf[1, "Depends"] else ""
+
+ # Combine, split, and clean
+ all_text <- paste(imports, depends, sep = ",")
+ packages <- trimws(strsplit(all_text, ",")[[1]])
+
+ # Remove version specifications (e.g., "ggplot2 (>= 3.0)" -> "ggplot2")
+ packages <- sub("\\s*\\(.*\\).*", "", packages)
+
+ # Remove empty strings and R itself
+ packages <- packages[nzchar(packages) & packages != "R"]
+
+ # Load each package with suppressed startup messages
+ invisible(sapply(packages, function(pkg) {
+ suppressPackageStartupMessages(
+ library(pkg, character.only = TRUE, quietly = quietly)
+ )
+ }))
+
+ if (verbose) message(paste("Loaded", length(packages), "packages from DESCRIPTION"))
+ invisible(packages)
+}
+
#tidy up
clean_data <- function(raw_data_file){
raw_data_file %>%
diff --git a/R/generate_release_manifest.R b/R/generate_release_manifest.R
new file mode 100644
index 00000000..01e6d436
--- /dev/null
+++ b/R/generate_release_manifest.R
@@ -0,0 +1,47 @@
+#' Generate a human-readable manifest of all targets for the GitHub release
+#'
+#' Creates a JSON file mapping target names to descriptions and file hashes
+#' from the targets store. Useful for cross-referencing hash-based filenames
+#' in the GitHub release with human-readable target names.
+#'
+#' @return Path to manifest file
+#' @export
+#'
+generate_release_manifest <- function() {
+
+ # Target descriptions
+ descriptions <- c(
+ vegmap_shp = "Vegetation map shapefile (from GitHub release vegmap2024)",
+ remnants_shp = "Vegetation remnants shapefile (manual download)",
+ capenature_fires_shp = "Fire extent shapefile (manual download)",
+ country.parquet = "Country boundary geometry (derived from geodata)",
+ domain_boundary.parquet = "Study domain boundary (intersection of vegetation map and country)",
+ domain_bbox.parquet = "50km-buffered download boundary (locked to prevent re-downloads)",
+ domain_nc = "Domain raster grid with pixel IDs, remnants, and distance-to-remnants",
+ vegmap_nc = "Vegetation map rasterized to analysis grid",
+ climate_chelsa = "CHELSA bioclimatic variables (19 NetCDF files: bio01-bio19)",
+ elevation_task_id = "AppEEARS task ID for NASADEM elevation download (task submission only)",
+ elevation = "NASADEM elevation data (resampled to analysis grid, masked to domain)"
+ )
+
+ # Build JSON string directly to avoid complex object serialization
+ json_lines <- c("{", " \"targets\": [")
+
+ for (i in seq_along(descriptions)) {
+ target_name <- names(descriptions)[i]
+ description <- descriptions[i]
+ # Escape quotes in description
+ description <- gsub('"', '\\"', description, fixed = TRUE)
+ comma <- if (i < length(descriptions)) "," else ""
+ json_lines <- c(json_lines, sprintf(' {"name": "%s", "description": "%s"}%s', target_name, description, comma))
+ }
+
+ json_lines <- c(json_lines, " ]", "}")
+ json_manifest <- paste(json_lines, collapse = "\n")
+
+ # Write to file
+ out_file <- "data/target_outputs/TARGET_MANIFEST.json"
+ writeLines(json_manifest, con = out_file)
+
+ out_file # Return the file path for targets
+}
diff --git a/R/get_climate_chelsa.R b/R/get_climate_chelsa.R
new file mode 100644
index 00000000..868f6087
--- /dev/null
+++ b/R/get_climate_chelsa.R
@@ -0,0 +1,154 @@
+#' @title Download and process CHELSA climate data
+#' @description Downloads CHELSA bioclimatic variables, clips to domain, and writes as NetCDF files
+#' @author Brian Maitner & Adam Wilson
+#' @param domain domain (sf polygon) used for masking
+#' @param temp_directory Temporary working directory for downloads (default: "data/temp/raw_data/climate_chelsa/")
+#' @param out_dir Output directory for NetCDF files (default: "data/target_outputs/")
+#' @param cleanup Logical. If TRUE (default, for GitHub Actions), clean temp directory. If FALSE (local development), preserve cached files.
+#' @param verbose Logical for progress messages
+#' @return Character vector of output NetCDF file paths
+#' @import terra
+#' @import sf
+#' @import ncdf4
+
+get_climate_chelsa <- function(
+ domain,
+ temp_directory = "data/temp/raw_data/climate_chelsa/",
+ out_dir = "data/target_outputs/",
+ cleanup = TRUE,
+ verbose = TRUE
+) {
+
+ # Ensure temp directory exists, clean only if cleanup mode enabled
+ if (cleanup && dir.exists(temp_directory)) {
+ unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
+ }
+ dir.create(temp_directory, recursive = TRUE, showWarnings = FALSE)
+ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
+
+ # Adjust download timeout
+ if (getOption('timeout') < 1000) {
+ options(timeout = 1000)
+ }
+
+ # Transform domain to WGS84
+ domain_tf <- domain %>%
+ st_as_sf() %>%
+ sf::st_transform(crs("+proj=longlat +datum=WGS84 +no_defs +ellps=WGS84 +towgs84=0,0,0"))
+
+ # CF-compliant metadata for CHELSA bioclimatic variables
+ bio_metadata <- tribble(
+ ~bio_name, ~long_name, ~units,
+ "bio01", "Annual Mean Temperature", "Β°C * 10",
+ "bio02", "Mean Diurnal Range", "Β°C * 10",
+ "bio03", "Isothermality", "%",
+ "bio04", "Temperature Seasonality", "Β°C * 10",
+ "bio05", "Max Temperature of Warmest Month", "Β°C * 10",
+ "bio06", "Min Temperature of Coldest Month", "Β°C * 10",
+ "bio07", "Temperature Annual Range", "Β°C * 10",
+ "bio08", "Mean Temperature of Wettest Quarter", "Β°C * 10",
+ "bio09", "Mean Temperature of Driest Quarter", "Β°C * 10",
+ "bio10", "Mean Temperature of Warmest Quarter", "Β°C * 10",
+ "bio11", "Mean Temperature of Coldest Quarter", "Β°C * 10",
+ "bio12", "Annual Precipitation", "mm",
+ "bio13", "Precipitation of Wettest Month", "mm",
+ "bio14", "Precipitation of Driest Month", "mm",
+ "bio15", "Precipitation Seasonality", "%",
+ "bio16", "Precipitation of Wettest Quarter", "mm",
+ "bio17", "Precipitation of Driest Quarter", "mm",
+ "bio18", "Precipitation of Warmest Quarter", "mm",
+ "bio19", "Precipitation of Coldest Quarter", "mm"
+ )
+
+ # Record download date
+ download_date <- Sys.Date()
+ output_files <- character()
+
+ for (idx in 1:nrow(bio_metadata)) {
+ i <- bio_metadata$bio_name[idx]
+
+ if (verbose) message("Processing ", i, " (", idx, "/", nrow(bio_metadata), ")")
+
+ # Construct filename
+ tif_filename <- file.path(temp_directory, paste("CHELSA_bio", sprintf("%02d", idx),
+ "_1981-2010_V.2.1.tif", sep = ""))
+
+ # Skip download if file already exists (when cleanup = FALSE, running locally)
+ if (!cleanup && file.exists(tif_filename)) {
+ if (verbose) message(" File already cached, skipping download: ", basename(tif_filename))
+ } else {
+ # Download the file
+ if (verbose) message(" Downloading...")
+ robust_download_file(
+ url = paste("https://os.unil.cloud.switch.ch/chelsa02/chelsa/global/bioclim/", i,
+ "/1981-2010/CHELSA_bio", sprintf("%02d", idx), "_1981-2010_V.2.1.tif", sep = ""),
+ destfile = tif_filename,
+ max_attempts = 10,
+ sleep_time = 10
+ )
+ }
+
+ # Load, crop, and mask
+ rast_i <- terra::rast(tif_filename)
+ domain_tf2=st_transform(domain_tf, st_crs(rast_i))
+ rast_i <- terra::crop(x = rast_i, y = ext(domain_tf2))
+ rast_i <- terra::mask(rast_i, mask = terra::vect(domain_tf2))
+
+ # Check if raster has data after masking
+
+ # Write as NetCDF with CF-compliant metadata
+ nc_filename <- file.path(out_dir, paste("CHELSA_", i, "_1981-2010_V.2.1.nc", sep = ""))
+
+ terra::writeCDF(x = rast_i,
+ filename = nc_filename,
+ overwrite = TRUE,
+ compression = 9)
+
+ # Add CF-compliant metadata using ncdf4 package
+ nc_file <- ncdf4::nc_open(nc_filename, write = TRUE)
+
+ # Get variable name
+ var_name <- names(rast_i)
+ if (is.null(var_name) || var_name == "") {
+ var_name <- i
+ }
+
+ # Get metadata for this bioclimatic variable
+ long_name <- bio_metadata$long_name[idx]
+ units <- bio_metadata$units[idx]
+
+ # Add global attributes
+ ncdf4::ncatt_put(nc_file, 0, "title",
+ paste("CHELSA Bioclimatic Variable", i, sep = " "))
+ ncdf4::ncatt_put(nc_file, 0, "source", "CHELSA v.2.1 (Climatologies at high resolution for the earth land areas)")
+ ncdf4::ncatt_put(nc_file, 0, "dataset_url", "https://chelsa-climate.org/")
+ ncdf4::ncatt_put(nc_file, 0, "download_date", as.character(download_date))
+ ncdf4::ncatt_put(nc_file, 0, "temporal_range", "1981-2010")
+ ncdf4::ncatt_put(nc_file, 0, "Conventions", "CF-1.8")
+ ncdf4::ncatt_put(nc_file, 0, "history",
+ paste("Downloaded on", as.character(download_date),
+ "and clipped to domain. Processed using terra and ncdf4 R packages."))
+
+ # Add variable attributes
+ ncdf4::ncatt_put(nc_file, var_name, "long_name", long_name)
+ ncdf4::ncatt_put(nc_file, var_name, "units", units)
+ ncdf4::ncatt_put(nc_file, var_name, "standard_name", paste("bioclimatic_variable_", i, sep = ""))
+
+ ncdf4::nc_close(nc_file)
+ output_files <- c(output_files, nc_filename)
+
+ rm(rast_i)
+ }
+
+ # Cleanup temp directory only if cleanup mode enabled
+ if (cleanup) {
+ unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
+ }
+
+ if (verbose) message("CHELSA climate files processed to: ", out_dir)
+
+ # Return output file paths for targets to track
+ output_files
+
+}
+
diff --git a/R/get_country.R b/R/get_country.R
new file mode 100644
index 00000000..fddefaa9
--- /dev/null
+++ b/R/get_country.R
@@ -0,0 +1,34 @@
+# National Boundary
+
+#' @author Adam M. Wilson
+#' @description Download national boundary file from the UN
+#' @source https://data.humdata.org/dataset/cod-ab-zaf
+
+get_country <- function(){
+
+ #Adjust timeout to allow for slow internet
+ if(getOption('timeout') < 1000){
+ options(timeout = 1000)
+ }
+
+ url="https://data.humdata.org/dataset/061d4492-56e8-458c-a3fb-e7950991adf0/resource/37175ff4-41a3-4753-a2c3-ced24142a96c/download/zaf_admin_boundaries.geojson.zip"
+ tmpfile1=tempfile()
+ tmpdir1=tempdir()
+ download.file(url,destfile = tmpfile1)
+ unzip(tmpfile1,exdir=tmpdir1)
+
+ # Read the converted GeoJSON, union, and convert to sf
+ country <- st_read(file.path(tmpdir1, "zaf_admin0.geojson"), quiet = TRUE) |>
+ st_union() |>
+ st_as_sf()
+
+ # Write to GeoParquet
+ out_file <- "data/target_outputs/country.parquet"
+ suppressWarnings(sfarrow::st_write_parquet(country, out_file))
+
+ return(out_file)
+}
+# end function
+
+
+
diff --git a/R/get_elevation.R b/R/get_elevation.R
new file mode 100644
index 00000000..93b4fa2d
--- /dev/null
+++ b/R/get_elevation.R
@@ -0,0 +1,213 @@
+#' @title Submit NASADEM elevation request via AppEEARS
+#' @description Submits an AppEEARS area request for NASADEM elevation data
+#' over the provided domain polygon. Returns task ID for polling.
+#' @author EMMA Team
+#' @param domain_vector A SpatVector or sf polygon defining the domain boundary
+#' @param verbose Logical for progress messages
+#' @return Character string with AppEEARS task ID
+
+submit_elevation_task <- function(
+ domain_vector,
+ verbose = TRUE
+) {
+
+ # Convert domain vector to sf, fix geometry, simplify, merge, and reproject to WGS84 (required by AppEEARS)
+ domain_sf <- st_as_sf(domain_vector) %>%
+ st_simplify(dTolerance = 100, preserveTopology = TRUE) %>%
+ st_buffer(0) %>%
+ st_make_valid() %>%
+ st_transform(crs = 4326) %>%
+ geojsonsf::sf_geojson(simplify = FALSE) %>%
+ jsonlite::fromJSON()
+
+ # Build AppEEARS request with proper structure
+ req <- list(
+ task_type = "area",
+ task_name = paste0("NASADEM_", format(Sys.time(), "%Y%m%d%H%M%S")),
+ params = list(
+ dates = list(list(
+ startDate = "02-11-2000",
+ endDate = "02-11-2000"
+ )),
+ layers = list(list(
+ product = "SRTMGL3_NC.003",
+ layer = "SRTMGL3_DEM"
+ )),
+ output = list(
+ format = list(type = "netcdf4"),
+ projection = "native"
+ ),
+ geo = domain_sf
+ )
+ )
+
+ # Submit task
+ if (verbose) message("Submitting AppEEARS elevation task...")
+ task <- appeears::rs_request(
+ request = req,
+ user = Sys.getenv("EARTHDATA_USER"),
+ transfer = FALSE,
+ verbose = verbose
+ )
+
+ task_id <- task$get_task_id()
+ if (verbose) message("Task submitted with ID: ", task_id)
+
+ task_id
+}
+
+
+#' @title Download and process NASADEM elevation from AppEEARS
+#' @description Polls for completion of AppEEARS task and downloads results,
+#' then resamples elevation to domain grid and writes to NetCDF.
+#' @author EMMA Team
+#' @param task_id Character string with AppEEARS task ID (from submit_elevation_task)
+#' @param domain_vector A SpatVector or sf polygon defining the domain boundary
+#' @param domain_raster A SpatRaster (domain.tif) defining the output grid and mask
+#' @param out_file Output NetCDF file path
+#' @param temp_directory Temporary working directory for downloads
+#' @param verbose Logical for progress messages
+#' @return Character path to output NetCDF file
+
+download_elevation_results <- function(
+ task_id,
+ domain_vector,
+ domain_raster,
+ out_file = "data/target_outputs/elevation_nasadem.nc",
+ temp_directory = "data/temp/raw_data/elevation_nasadem/",
+ verbose = TRUE
+) {
+
+ # Ensure clean temp directory
+ unlink(temp_directory, recursive = TRUE, force = TRUE)
+ dir.create(temp_directory, recursive = TRUE, showWarnings = FALSE)
+
+ # Clean terra temp
+ terra_tmp <- file.path(getwd(), "data/temp/terra")
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ dir.create(terra_tmp, recursive = TRUE, showWarnings = FALSE)
+ terraOptions(tempdir = terra_tmp, memfrac = 0.8)
+
+ # Reconnect to task and poll for completion
+ if (verbose) message("Polling task ", task_id, " for completion...")
+
+ # Poll for task completion using rs_list_task
+ max_retries <- 120 # 2 hours at 60s intervals
+ retry_count <- 0
+ task_status <- "pending"
+
+ repeat {
+ retry_count <- retry_count + 1
+
+ # Check task status
+ task_info <- appeears::rs_list_task(task_id = task_id, user = Sys.getenv("EARTHDATA_USER"))
+ task_status <- task_info$status
+
+ if (task_status == "done") {
+ if (verbose) message("Task completed successfully")
+ break
+ }
+
+ if (task_status %in% c("failed", "error")) {
+ stop("AppEEARS task ", task_id, " failed with status: ", task_status)
+ }
+
+ if (retry_count >= max_retries) {
+ stop("Task ", task_id, " polling timed out after ", max_retries, " minutes")
+ }
+
+ if (verbose && retry_count %% 10 == 0) {
+ message("Task status: ", task_status, " (", retry_count, "/", max_retries, ")")
+ }
+
+ Sys.sleep(60)
+ }
+
+ # Download results using rs_transfer
+ if (verbose) message("Downloading files for task: ", task_id)
+ appeears::rs_transfer(
+ task_id = task_id,
+ user = Sys.getenv("EARTHDATA_USER"),
+ path = temp_directory,
+ verbose = verbose
+ )
+
+ # Load the NetCDF file
+ nc_paths <- list.files(temp_directory, pattern = "\\.nc$", full.names = TRUE, recursive = TRUE)
+ if (length(nc_paths) == 0) {
+ stop("No NetCDF files downloaded from AppEEARS")
+ }
+
+ if (verbose) message("Reading elevation data from: ", nc_paths[1])
+ elev_raster <- terra::rast(nc_paths[grepl(nc_paths, pattern = "SRTMGL3_NC.003_90m_aid0001.nc")])
+
+ # Ensure we have a SpatRaster template (accept path or raster)
+ domain_template <- if (is.character(domain_raster)) terra::rast(domain_raster) else domain_raster
+
+ # Project to domain CRS/grid and mask to domain
+ if (verbose) message("Projecting elevation to domain CRS/grid")
+ elev_on_grid <- terra::project(elev_raster, domain_template, method = "average")
+
+ # Mask to domain (NA where domain is NA)
+ mask_layer <- if ("domain" %in% names(domain_template)) domain_template[["domain"]] else domain_template
+ elev_masked <- terra::mask(elev_on_grid, mask_layer)
+
+ # Set metadata
+ names(elev_masked) <- "elevation"
+
+ # Write NetCDF with compression and CF metadata
+ dir.create(dirname(out_file), recursive = TRUE, showWarnings = FALSE)
+ unlink(out_file)
+
+ ext_vals <- ext(elev_masked)
+ dx <- res(elev_masked)[1]
+ dy <- res(elev_masked)[2]
+ x_vals <- seq(ext_vals$xmin + dx/2, ext_vals$xmax - dx/2, by = dx)
+ y_vals <- seq(ext_vals$ymax - dy/2, ext_vals$ymin + dy/2, by = -dy)
+
+ dim_x <- ncdf4::ncdim_def(name = "easting", units = "meter", vals = x_vals, longname = "easting")
+ dim_y <- ncdf4::ncdim_def(name = "northing", units = "meter", vals = y_vals, longname = "northing")
+
+ var_elev <- ncdf4::ncvar_def(
+ name = "elevation",
+ units = "meters",
+ dim = list(dim_x, dim_y),
+ longname = "NASADEM elevation above mean sea level",
+ missval = -3.4e38,
+ prec = "float",
+ compression = 9
+ )
+
+ nc <- ncdf4::nc_create(filename = out_file, vars = list(var_elev), force_v4 = TRUE)
+
+ elev_matrix <- t(as.matrix(elev_masked, wide = TRUE))
+ elev_matrix[is.na(elev_matrix)] <- -3.4e38
+ ncdf4::ncvar_put(nc, var_elev, elev_matrix)
+
+ crs_wkt <- as.character(crs(elev_masked))
+ crs_var <- ncdf4::ncvar_def("crs", "", list(), prec = "integer")
+ nc <- ncdf4::ncvar_add(nc, crs_var)
+ ncdf4::ncatt_put(nc, "crs", "crs_wkt", crs_wkt)
+ ncdf4::ncatt_put(nc, "crs", "spatial_ref", crs_wkt)
+ ncdf4::ncatt_put(nc, "crs", "GeoTransform", paste(ext_vals$xmin, dx, 0, ext_vals$ymax, 0, -dy))
+ ncdf4::ncatt_put(nc, "elevation", "grid_mapping", "crs")
+
+ ncdf4::ncatt_put(nc, 0, "title", "NASADEM elevation resampled to domain")
+ ncdf4::ncatt_put(nc, 0, "source", "NASADEM_HGT.001 via AppEEARS")
+ ncdf4::ncatt_put(nc, 0, "history", paste0("created: ", Sys.time()))
+ ncdf4::ncatt_put(nc, 0, "Conventions", "CF-1.8")
+
+ ncdf4::nc_close(nc)
+
+ # Cleanup
+ unlink(temp_directory, recursive = TRUE, force = TRUE)
+ gc()
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+
+ if (verbose) message("Elevation data saved to: ", out_file)
+ out_file
+}
+
+
+
+
diff --git a/R/get_modis_vi.R b/R/get_modis_vi.R
new file mode 100644
index 00000000..1a2c0445
--- /dev/null
+++ b/R/get_modis_vi.R
@@ -0,0 +1,757 @@
+#' @title Generate monthly sequences for MODIS/VIIRS VI downloads
+#' @description Creates a data frame of all months to download based on a date range.
+#' @param start_date Start date (YYYY-MM-DD), default 2000-02-18 (MODIS Terra start)
+#' @param end_date End date (YYYY-MM-DD), default today
+#' @return Data frame with columns: month_start, month_end, date_str (YYYYMM format)
+#' @export
+generate_monthly_sequence <- function(start_date = "2000-02-18", end_date = NULL) {
+ if (is.null(end_date)) {
+ end_date <- Sys.Date()
+ }
+
+ start_date <- as.Date(start_date)
+ end_date <- as.Date(end_date)
+
+ # Generate all month starts from beginning month
+ month_starts <- seq(as.Date(cut(start_date, "month")), end_date, by = "month")
+ month_ends <- c(month_starts[-1] - 1, end_date)
+
+ # Trim if extends past end_date
+ valid_idx <- month_starts <= end_date
+ month_starts <- month_starts[valid_idx]
+ month_ends <- month_ends[valid_idx]
+
+ data.frame(
+ month_start = month_starts,
+ month_end = pmin(month_ends, end_date),
+ date_str = format(month_starts, "%Y%m"),
+ row.names = NULL
+ )
+}
+
+
+#' @title Identify missing monthly submissions from output directory
+#' @description Compares generated monthly sequence with existing downloaded NetCDF files
+#' @param output_dir Directory containing downloaded raw NetCDF files
+#' @param dataset Name of dataset (e.g., "modis_vi", "viirs_vi") used in filename pattern
+#' @param start_date Start date for sequence (YYYY-MM-DD)
+#' @param end_date End date for sequence (YYYY-MM-DD), default is today
+#' @return Data frame of months that haven't been downloaded yet
+#' @export
+identify_missing_vi <- function(output_dir, dataset = "modis_vi", start_date = "2000-02-18", end_date = NULL) {
+
+ # Create full monthly sequence
+ all_months <- generate_monthly_sequence(start_date, end_date)
+
+ # Check which ones already exist as downloaded NetCDF files
+ dir.create(output_dir, recursive = TRUE, showWarnings = FALSE)
+ pattern <- paste0("^", dataset, "_\\d{6}_monthly\\.nc$")
+ existing_files <- list.files(output_dir, pattern = pattern)
+
+ if (length(existing_files) == 0) {
+ return(all_months)
+ }
+
+ # Extract YYYYMM from existing files and convert to match date_str
+ # Format: __monthly.nc
+ pattern_prefix <- paste0("^", dataset, "_")
+ existing_dates <- existing_files %>%
+ gsub(pattern_prefix, "", .) %>%
+ gsub("_monthly\\.nc$", "", .)
+
+ # Return only missing months
+ missing <- all_months[!all_months$date_str %in% existing_dates, ]
+
+ if (nrow(missing) == 0) {
+ message("All months already downloaded in ", output_dir)
+ return(data.frame())
+ }
+
+ missing
+}
+
+
+
+#' @title Submit monthly MODIS VI request via AppEEARS
+#' @description Submits an AppEEARS area request for MOD13A1.061 and MYD13A1.061
+#' EVI, and QA (500m resolution, 16-day composite) for a monthly period.
+#' @author EMMA Team
+#' @param domain_vector A SpatVector or sf polygon defining the domain boundary
+#' @param month_start Start date for the month (YYYY-MM-DD)
+#' @param month_end End date for the month (YYYY-MM-DD)
+#' @param verbose Logical for progress messages
+#' @return Character string with AppEEARS task ID
+#' @export
+submit_modis_vi <- function(
+ domain_vector,
+ month_start,
+ month_end,
+ verbose = TRUE
+) {
+
+ # Convert domain vector to sf, fix geometry, simplify, merge, and reproject to WGS84
+ domain_sf <- st_as_sf(domain_vector) %>%
+ st_simplify(dTolerance = 100, preserveTopology = TRUE) %>%
+ st_buffer(0) %>%
+ st_make_valid() %>%
+ st_transform(crs = 4326) %>%
+ geojsonsf::sf_geojson(simplify = FALSE) %>%
+ jsonlite::fromJSON()
+
+ # Validate dates
+ month_start <- as.Date(month_start)
+ month_end <- as.Date(month_end)
+
+ if (verbose) {
+ message("AppEEARS MODIS VI monthly request: ", format(month_start, "%Y-%m-%d"),
+ " to ", format(month_end, "%Y-%m-%d"))
+ }
+
+ # Resolve layer names dynamically (same as full-range version)
+ evi_layer <- "_500m_16_days_EVI"
+ qa_layer <- "_500m_16_days_VI_Quality"
+ date_layer <- "_500m_16_days_composite_day_of_the_year"
+
+ try({
+ lyr <- appeears::rs_layers("MOD13A1.061")
+ cand_cols <- intersect(c("Layer", "Name", "layer", "name"), names(lyr))
+ if (length(cand_cols)) {
+ vals <- unlist(lapply(cand_cols, function(cc) lyr[[cc]]))
+ evi_cand <- vals[grepl("EVI", vals, ignore.case = TRUE)][1]
+ qa_cand <- vals[grepl("VI.*Quality|Quality", vals, ignore.case = TRUE)][1]
+ date_cand <- vals[grepl("composite_day_of_the_year", vals, ignore.case = TRUE)][1]
+ if (!is.na(evi_cand)) evi_layer <- evi_cand
+ if (!is.na(qa_cand)) qa_layer <- qa_cand
+ if (!is.na(date_cand)) date_layer <- date_cand
+ }
+ }, silent = TRUE)
+
+ if (verbose) message("Using layers: ", evi_layer, ", ", qa_layer, ", ", date_layer)
+
+ # Build request payload for monthly period
+ req <- list(
+ task_type = "area",
+ task_name = paste0("MODIS_VI_", format(month_start, "%Y%m"), "_", format(Sys.time(), "%H%M%S")),
+ params = list(
+ dates = list(list(
+ startDate = format(month_start, "%m-%d-%Y"),
+ endDate = format(month_end, "%m-%d-%Y")
+ )),
+ layers = list(
+ # MOD13A1.061 (Terra)
+ list(product = "MOD13A1.061", layer = evi_layer),
+ list(product = "MOD13A1.061", layer = qa_layer),
+ list(product = "MOD13A1.061", layer = date_layer),
+ # MYD13A1.061 (Aqua)
+ list(product = "MYD13A1.061", layer = evi_layer),
+ list(product = "MYD13A1.061", layer = qa_layer),
+ list(product = "MYD13A1.061", layer = date_layer)
+ ),
+ output = list(
+ format = list(type = "netcdf4"),
+ projection = "native"
+ ),
+ geo = domain_sf
+ )
+ )
+
+ # Submit task
+ if (verbose) message("Submitting AppEEARS MODIS VI monthly task...")
+ task <- appeears::rs_request(
+ request = req,
+ user = Sys.getenv("EARTHDATA_USER"),
+ transfer = FALSE,
+ verbose = verbose
+ )
+
+ task_id <- task$get_task_id()
+ if (verbose) message("Task submitted with ID: ", task_id)
+
+ task_id
+}
+
+
+#' @title Download MODIS VI NetCDF files from AppEEARS
+#' @description Polls for completion of AppEEARS task and downloads results.
+#' Separates I/O from computation for independent parallelization.
+#' @author EMMA Team
+#' @param task_id Character string with AppEEARS task ID
+#' @param month_start Start date for monthly period (YYYY-MM-DD)
+#' @param temp_directory Temporary working directory for downloads
+#' @param cleanup Logical to delete temporary files after processing. Defaults to TRUE on GitHub Actions (GITHUB_ACTIONS env var), FALSE on local execution.
+#' @param verbose Logical for progress messages
+#' @return Character path to temporary directory containing downloaded NetCDF files and metadata
+#' @export
+download_modis_vi_netcdf <- function(
+ task_id,
+ month_start,
+ temp_directory = "data/temp/raw_data/modis_vi_netcdf/",
+ cleanup = Sys.getenv("GITHUB_ACTIONS") == "true",
+ verbose = TRUE
+) {
+
+ month_start <- as.Date(month_start)
+ yyyymm <- format(month_start, "%Y%m")
+
+ # Check if this month was already downloaded (marker file exists)
+ cache_dir <- "data/target_outputs/modis_vi"
+ dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE)
+ cache_file <- file.path(cache_dir, paste0("modis_vi_", yyyymm, "_monthly.nc"))
+
+ if (file.exists(cache_file)) {
+ if (verbose) message("Marker file found for ", yyyymm, " - skipping AppEEARS download")
+ # Create minimal temp directory so netcdf_to_parquet gets valid path
+ dir.create(temp_directory, recursive = TRUE, showWarnings = FALSE)
+ return(temp_directory)
+ }
+
+ # Clean and create temp directory
+ if (!cleanup_mode) unlink(temp_directory, recursive = TRUE, force = TRUE)
+ dir.create(temp_directory, recursive = TRUE, showWarnings = FALSE)
+
+ # Poll for task completion
+ if (verbose) message("Polling task ", task_id, " for completion...")
+
+ max_retries <- 120 # 2 hours at 60s intervals
+ retry_count <- 0
+ task_status <- "pending"
+
+ repeat {
+ retry_count <- retry_count + 1
+
+ # Check task status
+ task_info <- appeears::rs_list_task(task_id = task_id, user = Sys.getenv("EARTHDATA_USER"))
+ task_status <- task_info$status
+
+ if (task_status == "done") {
+ if (verbose) message("Task completed successfully")
+ break
+ }
+
+ if (task_status %in% c("failed", "error")) {
+ stop("AppEEARS task ", task_id, " failed with status: ", task_status)
+ }
+
+ if (retry_count >= max_retries) {
+ stop("Task ", task_id, " polling timed out after ", max_retries, " minutes")
+ }
+
+ if (verbose && retry_count %% 10 == 0) {
+ message("Task status: ", task_status, " (", retry_count, "/", max_retries, ")")
+ }
+
+ Sys.sleep(60)
+ }
+
+ # Download results
+ if (verbose) message("Downloading files for task: ", task_id)
+ appeears::rs_transfer(
+ task_id = task_id,
+ user = Sys.getenv("EARTHDATA_USER"),
+ path = temp_directory,
+ verbose = verbose
+ )
+
+ # Check if NetCDF files were downloaded
+ nc_paths <- list.files(temp_directory, pattern = "\\.nc$", full.names = TRUE, recursive = TRUE)
+ if (length(nc_paths) == 0) {
+ if (verbose) message("No NetCDF files returned from AppEEARS for month ", yyyymm)
+ if (cleanup) {
+ unlink(temp_directory, recursive = TRUE, force = TRUE)
+ }
+ return(NA_character_)
+ }
+
+ if (verbose) message("Downloaded ", length(nc_paths), " NetCDF files to ", temp_directory)
+
+ # Create persistent marker file to avoid re-downloading this month
+ # Check which months have already been downloaded by looking for marker files
+ cache_dir <- "data/target_outputs/modis_vi"
+ dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE)
+ cache_file <- file.path(cache_dir, paste0("modis_vi_", yyyymm, "_monthly.nc"))
+
+ # Write a simple marker file (just needs to exist to be detected by identify_missing_vi)
+ cat("", file = cache_file)
+
+ if (verbose) message("Created download marker: ", cache_file)
+
+ # Return temp directory so netcdf_to_parquet() can access the actual files
+ return(temp_directory)
+}
+
+
+#' @title Extract QA-good values from AppEEARS lookup table
+#' @description Helper function to extract pixel QA flag values that meet quality criteria.
+#' Filters based on: VI quality, no adjacent cloud, no shadow, no snow, over land, low aerosol.
+#' @param qa_lookup_files Character vector of paths to VI Quality lookup CSV files
+#' @return Integer vector of QA flag values that pass all quality filters
+#' @keywords internal
+extract_keep_qa_values <- function(qa_lookup_files) {
+
+ extract_keep <- function(path) {
+ tab <- tryCatch(read.csv(path, stringsAsFactors = FALSE), error = function(e) NULL)
+ if (is.null(tab)) return(NULL)
+
+ # Find required columns (case-insensitive)
+ value_col <- names(tab)[grepl("^value$", tolower(names(tab)))][1]
+ modland_col <- names(tab)[grepl("modland", tolower(names(tab)))][1]
+ adj_cloud_col <- names(tab)[grepl("adjacent.*cloud", tolower(names(tab)))][1]
+ snow_col <- names(tab)[grepl("possible.*snow|snow.*ice", tolower(names(tab)))][1]
+ shadow_col <- names(tab)[grepl("possible.*shadow", tolower(names(tab)))][1]
+ land_col <- names(tab)[grepl("land/water|land.*water", tolower(names(tab)))][1]
+ aerosol_col <- names(tab)[grepl("aerosol", tolower(names(tab)))][1]
+
+ # Check all required columns are present
+ required_cols <- c(value_col, modland_col, adj_cloud_col, snow_col, shadow_col, land_col, aerosol_col)
+ if (any(is.na(required_cols))) {
+ warning("QA lookup missing required columns. Found: value=", !is.na(value_col),
+ ", modland=", !is.na(modland_col), ", adj_cloud=", !is.na(adj_cloud_col),
+ ", snow=", !is.na(snow_col), ", shadow=", !is.na(shadow_col),
+ ", land=", !is.na(land_col), ", aerosol=", !is.na(aerosol_col))
+ return(NULL)
+ }
+
+ # Filter for pixels that meet ALL QA criteria:
+ # 1. VI produced with good quality
+ # 2. No adjacent cloud detected
+ # 3. No cloud shadow
+ # 4. No snow/ice
+ # 5. Over land (not ocean or water)
+ # 6. Not high aerosol loading
+ keep <- (grepl("vi produced", tolower(tab[[modland_col]]))) &
+ (grepl("^no$", tolower(tab[[adj_cloud_col]]))) &
+ (grepl("^no$", tolower(tab[[shadow_col]]))) &
+ (grepl("^no$", tolower(tab[[snow_col]]))) &
+ (grepl("land", tolower(tab[[land_col]]))) &
+ (!grepl("high", tolower(tab[[aerosol_col]])))
+
+ tab[[value_col]][keep & !is.na(tab[[value_col]])]
+ }
+
+ unique(unlist(lapply(qa_lookup_files, extract_keep)))
+}
+
+
+#' @title Extract MODIS VI observations from single NetCDF file
+#' @description Processes one NetCDF file: detects Terra/Aqua product from metadata, applies QA masking,
+#' reprojects to domain grid, and extracts observations in tabular format.
+#' @param nc_path Character path to NetCDF file
+#' @param domain_template SpatRaster defining output grid and CRS
+#' @param keep_values Integer vector of QA flag values that pass quality criteria
+#' @param month_start Start date for monthly period (YYYY-MM-DD), used for date conversion
+#' @param verbose Logical for progress messages
+#' @return Tibble with columns: pid, date, variable, value. Returns NULL if no valid observations found.
+#' @keywords internal
+extract_vi_observations <- function(
+ nc_path,
+ domain_template,
+ keep_values,
+ month_start,
+ verbose = TRUE
+) {
+
+ # Load raster from NetCDF
+ rast_obj <- terra::rast(nc_path)
+
+ # Detect product (Terra=MOD13 vs Aqua=MYD13) from NetCDF global attributes
+ product_name <- "terra"
+ sensor_id <- 1L
+
+ tryCatch({
+ nc <- ncdf4::nc_open(nc_path)
+ global_attrs <- names(ncdf4::ncatt_get(nc, 0))
+
+ for (attr_name in global_attrs) {
+ attr_val <- ncdf4::ncatt_get(nc, 0, attr_name)$value
+ if (!is.null(attr_val) && is.character(attr_val)) {
+ if (grepl("MYD13", attr_val, ignore.case = TRUE)) {
+ product_name <- "aqua"
+ sensor_id <- 2L
+ break
+ } else if (grepl("MOD13", attr_val, ignore.case = TRUE)) {
+ product_name <- "terra"
+ sensor_id <- 1L
+ break
+ }
+ }
+ }
+ ncdf4::nc_close(nc)
+ }, error = function(e) {
+ if (verbose) warning("Could not detect product from NetCDF metadata, defaulting to terra")
+ })
+
+ # Extract and validate required layer indices
+ evi_idx <- which(grepl("EVI", names(rast_obj), ignore.case = TRUE))[1]
+ qa_idx <- which(grepl("Quality", names(rast_obj), ignore.case = TRUE))[1]
+ date_idx <- which(grepl("composite_day_of_the_year", names(rast_obj), ignore.case = TRUE))[1]
+
+ if (is.na(evi_idx) || is.na(qa_idx) || is.na(date_idx)) {
+ if (verbose) {
+ message("Skipping file: missing required layers. EVI: ", !is.na(evi_idx),
+ ", QA: ", !is.na(qa_idx), ", Date: ", !is.na(date_idx))
+ }
+ return(NULL)
+ }
+
+ # Apply QA mask: keep only pixels with good QA values
+ qa_r <- rast_obj[[qa_idx]]
+ keep_mask <- terra::app(qa_r, function(x) x %in% keep_values)
+
+ # Mask, project, and scale EVI
+ evi <- terra::mask(rast_obj[[evi_idx]], keep_mask, maskvalue = FALSE) |>
+ terra::project(domain_template, method = "average") |>
+ terra::app(function(x) x * 100)
+
+ # Mask and project date (composite day of year)
+ date <- terra::mask(rast_obj[[date_idx]], keep_mask, maskvalue = FALSE) |>
+ terra::project(domain_template, method = "mode")
+
+ # Get pid layer from domain
+ if (!"pid" %in% names(domain_template)) {
+ stop("domain_template must include a 'pid' layer")
+ }
+ pid_layer <- domain_template[["pid"]]
+
+ # Convert rasters to matrices and vectorize
+ evi_matrix <- terra::as.matrix(evi, wide = TRUE)
+ date_matrix <- terra::as.matrix(date, wide = TRUE)
+ pid_matrix <- terra::as.matrix(pid_layer, wide = TRUE)
+
+ evi_vec <- as.vector(evi_matrix)
+ date_vec <- as.vector(date_matrix)
+ pid_vec <- as.vector(pid_matrix)
+
+ # Identify valid observations (all three must be non-NA)
+ valid_idx <- !is.na(evi_vec) & !is.na(date_vec) & !is.na(pid_vec)
+
+ if (!any(valid_idx)) {
+ if (verbose) message("No valid observations in ", basename(nc_path))
+ return(NULL)
+ }
+
+ # Convert day-of-year to days-since-epoch
+ ref_year <- as.integer(format(month_start, "%Y"))
+ year_start <- as.Date(paste0(ref_year, "-01-01"))
+
+ doy_to_epoch <- function(doy) {
+ if (is.na(doy) || !is.finite(doy)) return(NA_integer_)
+ date_obj <- year_start + (as.integer(doy) - 1)
+ as.integer(date_obj - as.Date("1970-01-01"))
+ }
+
+ # Build observation tibble with vectorized date conversion
+ obs_df <- tibble::tibble(
+ pid = as.integer(pid_vec[valid_idx]),
+ date = as.integer(sapply(date_vec[valid_idx], doy_to_epoch)),
+ variable = sensor_id,
+ value = as.integer(evi_vec[valid_idx])
+ )
+
+ if (verbose) {
+ message("Extracted ", nrow(obs_df), " observations from ", basename(nc_path), " (", product_name, ")")
+ }
+
+ obs_df
+}
+
+
+#' @title Convert MODIS VI NetCDF files to parquet format
+#' @description Processes downloaded NetCDF files: applies QA masking, reprojects to domain grid,
+#' and converts observations to parquet format with one row per observation.
+#' @author EMMA Team
+#' @param netcdf_directory Path to directory containing downloaded NetCDF files and metadata
+#' @param domain_raster A SpatRaster (domain.tif) defining the output grid and mask
+#' @param month_start Start date for monthly period (YYYY-MM-DD)
+#' @param out_dir Output directory for parquet files
+#' @param cleanup Logical to delete temporary files after processing. Defaults to TRUE on GitHub Actions (GITHUB_ACTIONS env var), FALSE on local execution.
+#' @param verbose Logical for progress messages
+#' @return Character path to output parquet file (format: modis_vi_YYYYMM.gz.parquet)
+#' @details
+#' Parquet schema:
+#' - pid (int32): Pixel ID from domain grid
+#' - date (int32): Days since epoch (1970-01-01)
+#' - variable (int32): Sensor code (1=Terra, 2=Aqua, 3=VIIRS)
+#' - value (int32): EVI value Γ 100
+#'
+#' One row per unique observation (pid, date, sensor combination).
+#' @export
+netcdf_to_parquet <- function(
+ netcdf_directory,
+ domain_raster,
+ month_start,
+ out_dir = "data/processed_data/dynamic_parquet/modis_vi",
+ cleanup = Sys.getenv("GITHUB_ACTIONS") == "true",
+ verbose = TRUE
+) {
+
+ month_start <- as.Date(month_start)
+ yyyymm <- format(month_start, "%Y%m")
+
+ terra_tmp <- file.path(getwd(), "data/temp/terra")
+
+ # Clean terra temp
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ dir.create(terra_tmp, recursive = TRUE, showWarnings = FALSE)
+ terraOptions(tempdir = terra_tmp, memfrac = 0.8)
+
+ # Validate and load NetCDF files
+ nc_paths <- list.files(netcdf_directory, pattern = "\\.nc$", full.names = TRUE, recursive = TRUE)
+ if (length(nc_paths) == 0) {
+ if (verbose) message("No NetCDF files found in ", netcdf_directory, " - writing skip marker")
+
+ # Create a lightweight skip marker file instead of fake data
+ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
+ skip_file <- file.path(out_dir, paste0("modis_vi_", yyyymm, "_monthly.skip"))
+
+ writeLines(
+ c(
+ paste("Month:", yyyymm),
+ paste("Reason: No NetCDF files available for processing"),
+ paste("Note: Possible causes - polar region, cloud cover, instrument malfunction, or outside data availability period"),
+ paste("Timestamp:", Sys.time())
+ ),
+ skip_file
+ )
+
+ if (verbose) message("Created skip marker: ", skip_file)
+
+ if (cleanup) {
+ unlink(netcdf_directory, recursive = TRUE, force = TRUE)
+ gc()
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ }
+
+ return(skip_file)
+ }
+
+ if (verbose) message("Reading ", length(nc_paths), " NetCDF files")
+
+
+# Vargas et al.,15 pixels with any of the following QA flags were removed:
+# not confidently clear, adjacent to cloud, cloud shadow, snow or ice, thin cirrus cloud,
+# high aerosol loading, solar zenith angle >65 deg, and not over land.
+
+ # Apply QA mask to VI layers
+ qa_lookup <- list.files(
+ netcdf_directory,
+ pattern = "(VI-Quality-lookup).*\\.csv$",
+ full.names = TRUE,
+ recursive = TRUE
+ )
+ if (!length(qa_lookup)) {
+ stop("QA lookup table (VI_Quality*.csv) not found in netcdf_directory; cannot mask VI data")
+ }
+
+ keep_values <- extract_keep_qa_values(qa_lookup)
+ if (!length(keep_values)) {
+ stop("No 'good quality' entries found in any QA table; refusing to proceed")
+ }
+
+ if (verbose) message("Using ", length(keep_values), " QA values for masking")
+
+ # Load and validate domain raster
+ domain_template <- if (is.character(domain_raster)) terra::rast(domain_raster) else domain_raster
+
+ if (!"pid" %in% names(domain_template)) {
+ stop("domain_raster must include a 'pid' layer")
+ }
+
+ # Get domain mask and valid pids
+ mask_layer <- if ("domain" %in% names(domain_template)) domain_template[["domain"]] else domain_template
+ pid_raster <- terra::mask(domain_template[["pid"]], mask_layer)
+ valid_pids <- unique(terra::values(pid_raster)[, 1])
+ valid_pids <- valid_pids[!is.na(valid_pids)]
+
+ if (verbose) message("Domain has ", length(valid_pids), " valid pixels")
+
+ # Process each NetCDF file independently (call helper once per file)
+ all_obs <- list()
+
+ for (nc_path in nc_paths) {
+ obs <- extract_vi_observations(
+ nc_path = nc_path,
+ domain_template = domain_template,
+ keep_values = keep_values,
+ month_start = month_start,
+ verbose = verbose
+ )
+
+ if (!is.null(obs)) {
+ # Filter to domain pids
+ obs <- obs[obs$pid %in% valid_pids, ]
+ if (nrow(obs) > 0) {
+ all_obs[[length(all_obs) + 1]] <- obs
+ }
+ }
+ }
+
+ if (length(all_obs) == 0) {
+ if (verbose) message("No valid observations found after QA masking and domain filtering")
+ if (cleanup) {
+ unlink(netcdf_directory, recursive = TRUE, force = TRUE)
+ gc()
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ }
+ return(NA_character_)
+ }
+
+ # Bind all observations into single dataframe and drop any NAs (should be none after filtering, but just in case)
+ df <- dplyr::bind_rows(all_obs)|>
+ dplyr::filter(!is.na(.data$value))
+
+
+ # Write to parquet
+ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
+ parquet_file <- file.path(out_dir, sprintf("dynamic_modis_vi_%s.gz.parquet", yyyymm))
+
+ unlink(parquet_file)
+ if (verbose) message("Writing ", nrow(df), " observations to parquet")
+ arrow::write_parquet(
+ df,
+ sink = parquet_file,
+ compression = "gzip"
+ )
+
+ if (verbose) message("Parquet file saved: ", parquet_file)
+
+ # Cleanup
+ if (cleanup) {
+ unlink(netcdf_directory, recursive = TRUE, force = TRUE)
+ gc()
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ }
+
+ parquet_file
+}
+
+
+#' @title Convert parquet observations back to GeoTIFF
+#' @description Reads MODIS VI parquet file, maps observations back to spatial grid using domain_nc,
+#' and writes a GeoTIFF. Supports filtering by date/sensor and optional aggregation.
+#' @author EMMA Team
+#' @param parquet_file Path to parquet file (e.g., modis_vi_202401.gz.parquet)
+#' @param domain_nc Path to domain.nc or domain raster (SpatRaster)
+#' @param out_file Output GeoTIFF path
+#' @param date Optional integer (days since epoch) or Date object to filter specific date.
+#' If NULL, uses most recent observation per pixel. If length 2, treated as date range.
+#' @param sensor Optional integer: 1=Terra, 2=Aqua, 3=VIIRS. If NULL, uses all available.
+#' @param aggregate_fun Optional function to aggregate multiple observations per pixel
+#' (e.g., mean, median, max). If NULL, uses most recent observation.
+#' @param verbose Logical for progress messages
+#' @return Path to written GeoTIFF file
+#' @export
+parquet_to_geotif <- function(
+ parquet_file,
+ domain_nc,
+ out_file,
+ date = NULL,
+ sensor = NULL,
+ aggregate_fun = NULL,
+ verbose = TRUE
+) {
+
+ # Load parquet
+ if (!file.exists(parquet_file)) {
+ stop("Parquet file not found: ", parquet_file)
+ }
+
+ if (verbose) message("Reading parquet: ", parquet_file)
+ df <- arrow::read_parquet(parquet_file)
+
+ # Filter by sensor if specified
+ if (!is.null(sensor)) {
+ if (!sensor %in% c(1L, 2L, 3L)) stop("sensor must be 1 (Terra), 2 (Aqua), or 3 (VIIRS)")
+ df <- dplyr::filter(df, .data$variable == sensor)
+ if (verbose) message("Filtered to sensor: ", c("Terra", "Aqua", "VIIRS")[sensor])
+ }
+
+ # Filter by date if specified
+ if (!is.null(date)) {
+ # Convert Date to days since epoch if needed
+ if (inherits(date, "Date")) {
+ date <- as.integer(date - as.Date("1970-01-01"))
+ }
+
+ if (length(date) == 1) {
+ # Single date: filter to exact date
+ df <- dplyr::filter(df, .data$date == date)
+ if (verbose) message("Filtered to date: ", as.Date(date, origin = "1970-01-01"))
+ } else if (length(date) == 2) {
+ # Date range: filter to range
+ df <- dplyr::filter(df, .data$date >= min(date) & .data$date <= max(date))
+ if (verbose) message("Filtered to date range: ",
+ as.Date(min(date), origin = "1970-01-01"), " to ",
+ as.Date(max(date), origin = "1970-01-01"))
+ }
+ }
+
+ if (nrow(df) == 0) {
+ stop("No observations match filter criteria")
+ }
+
+ # Load domain raster to get pidβspatial mapping
+ if (verbose) message("Loading domain raster...")
+ if (is.character(domain_nc)) {
+ if (!file.exists(domain_nc)) {
+ stop("domain_nc file not found: ", domain_nc)
+ }
+ domain <- terra::rast(domain_nc)
+ } else {
+ domain <- domain_nc # Assume it's a SpatRaster
+ }
+
+ # Extract pid layer
+ if (!"pid" %in% names(domain)) {
+ stop("domain raster must include a 'pid' layer")
+ }
+ pid_raster <- domain[["pid"]]
+
+ # Aggregate if multiple observations per pixel and aggregate_fun provided
+ if (!is.null(aggregate_fun)) {
+ if (verbose) message("Aggregating multiple observations per pixel...")
+ df <- df %>%
+ dplyr::group_by(.data$pid) %>%
+ dplyr::summarise(value = aggregate_fun(.data$value), .groups = "drop")
+ } else {
+ # Use most recent observation per pixel (if multiple)
+ if (verbose && nrow(df) > nrow(dplyr::distinct(df, .data$pid))) {
+ message("Multiple observations per pixel - using most recent")
+ }
+ df <- df %>%
+ dplyr::arrange(dplyr::desc(.data$date), dplyr::desc(.data$variable)) %>%
+ dplyr::distinct(.data$pid, .keep_all = TRUE)
+ }
+
+ # Create output raster
+ if (verbose) message("Creating raster from ", nrow(df), " observations...")
+ output_raster <- pid_raster * NA # Start with same structure but all NA
+
+ # Vectorized mapping: create lookup vector indexed by pid
+ pid_vals <- terra::values(pid_raster)[, 1] # Get all pid values
+ max_pid <- max(c(df$pid, pid_vals), na.rm = TRUE)
+
+ # Create lookup vector with NA for missing pids, then use vectorized indexing
+ lookup <- rep(NA_integer_, max_pid)
+ lookup[df$pid] <- df$value
+
+ # Map all pid values to evi values in one vectorized operation
+ output_values <- lookup[pid_vals]
+ terra::values(output_raster) <- output_values
+
+ # Write GeoTIFF
+ if (verbose) message("Writing GeoTIFF: ", out_file)
+ dir.create(dirname(out_file), recursive = TRUE, showWarnings = FALSE)
+ terra::writeRaster(
+ output_raster,
+ filename = out_file,
+ filetype = "GTiff",
+ datatype = "INT2U",
+ overwrite = TRUE,
+ names = "EVI"
+ )
+
+ if (verbose) message("GeoTIFF saved: ", out_file)
+ return(out_file)
+}
+
+# parquet_to_geotif("data/target_outputs/modis_vi/modis_vi_202601.gz.parquet", "data/raw/domain.nc", "output.tif")
\ No newline at end of file
diff --git a/R/get_release_alos.R b/R/get_release_alos.R
deleted file mode 100644
index d3e88137..00000000
--- a/R/get_release_alos.R
+++ /dev/null
@@ -1,167 +0,0 @@
-#ALOS
-
-#' @author Brian Maitner
-
-#make a function to reduce code duplication
-
-#' @param image_text is the text string used by gee to refer to an image, e.g. "CSP/ERGo/1_0/Global/ALOS_mTPI"
-#' @param dir directory to save data in
-#' @param domain domain (sf polygon) used for masking
-#' @note This code is only designed to work with a handful of images by CSP/ERGo
-get_alos_data <- function(image_text, dir, domain,
- json_token){
-
- #Load the image
-
- focal_image <- ee$Image(image_text)
-
- focal_name <- focal_image$getInfo()$properties$visualization_0_name
-
- focal_name <- tolower(focal_name)
-
- focal_name <-gsub(pattern = " ", replacement = "_", x = focal_name)
-
- #Format the domain
- domain <- sf_as_ee(x = domain)
- domain <- domain$geometry()
-
- #get CRS
- crs <- focal_image$getInfo()$bands[[1]]$crs
-
- #Download the raster
- ee_as_raster(image = focal_image,
- region = domain,
- #scale = 100, #used to adjust the scale. commenting out uses the default
- dsn = file.path(dir,focal_name),
- maxPixels = 10000000000,
- drive_cred_path = json_token)
-
-
-}# end function
-
-
-
-#' @description This function makes use of the previous helper function to download data
-#' @param domain domain (sf polygon) used for masking
-#' @param temp_directory Where to save the files, defaults to "data/raw_data/alos/"
-#' @param tag tag for the release you want the data stored in
-get_release_alos <- function(temp_directory = "data/temp/raw_data/alos/",
- tag = "raw_static",
- domain,
- json_token){
-
- #make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
- dir.create(temp_directory, recursive = TRUE)
- }
-
-
- #Make sure there is a release by attempting to create one. If it already exists, this will fail
-
- tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
- tag = tag),
- error = function(e){message("Previous release found")})
-
-
- #Initialize earth engine (for targets works better if called here)
-
- #ee_Initialize()
-
- # Get files that have been downloaded
-
- alos_files <- list.files(temp_directory, pattern = ".tif$")
-
- #Download files that have not previously been downloaded
-
- # mTPI
- if(!length(grep(pattern = "mtpi",x = alos_files)) > 0){
-
- get_alos_data(image_text = "CSP/ERGo/1_0/Global/ALOS_mTPI",
- dir = temp_directory,
- domain = domain,
- json_token = json_token)
-
- }
-
- # release
- pb_upload(repo = "AdamWilsonLab/emma_envdata",
- file = file.path(temp_directory,"alos_mtpi.tif"),
- tag = tag,
- name = "alos_mtpi.tif")
-
- # delete
- file.remove(file.path(temp_directory,"alos_mtpi.tif"))
-
-
- # CHILI
- if(!length(grep(pattern = "chili",x = alos_files)) > 0){
-
- get_alos_data(image_text = "CSP/ERGo/1_0/Global/ALOS_CHILI",
- dir = temp_directory,
- domain = domain,
- json_token = json_token)
-
- }
-
- # release
- pb_upload(repo = "AdamWilsonLab/emma_envdata",
- file = file.path(temp_directory,"alos_chili.tif"),
- tag = tag,
- name = "alos_chili.tif")
-
- # delete
- file.remove(file.path(temp_directory,"alos_chili.tif"))
-
-
- # landforms
- if(!length(grep(pattern = "landforms",x = alos_files)) > 0){
- get_alos_data(image_text = 'CSP/ERGo/1_0/Global/ALOS_landforms',
- dir = temp_directory,
- domain = domain,
- json_token = json_token)
- }
-
- # release
- pb_upload(repo = "AdamWilsonLab/emma_envdata",
- file = file.path(temp_directory,"landforms.tif"),
- tag = tag,
- name = "alos_landforms.tif")
-
- # delete
- file.remove(file.path(temp_directory,"landforms.tif"))
-
-
- # topo diversity
- if(!length(grep(pattern = "topographic",x = alos_files)) > 0){
- get_alos_data(image_text = 'CSP/ERGo/1_0/Global/ALOS_topoDiversity',
- dir = temp_directory,
- domain = domain,
- json_token = json_token)
- }
-
- # release
- pb_upload(repo = "AdamWilsonLab/emma_envdata",
- file = file.path(temp_directory,"alos_topographic_diversity.tif"),
- tag = tag,
- name = "alos_topodiversity.tif")
-
- # delete
- file.remove(file.path(temp_directory,"alos_topographic_diversity.tif"))
-
-
- # Clean up
- unlink(x = file.path(temp_directory), recursive = TRUE)
-
-
- message("Finished downloading ALOS layers")
-
-
- return(tag)
-
-}
-
-
-
-##################################
-
diff --git a/R/get_release_climate_chelsa.R b/R/get_release_climate_chelsa.R
deleted file mode 100644
index 23c8d1eb..00000000
--- a/R/get_release_climate_chelsa.R
+++ /dev/null
@@ -1,129 +0,0 @@
-#R script to download climate data (CHELSA)
-
-library(terra)
-
-#' @author Brian Maitner
-#' @description This function will download CHELSA climate data if it isn't present, and (invisibly) return a NULL if it is present
-#' @param temp_directory Where to save the files, defaults to "data/raw_data/climate_chelsa/"
-#' @param domain domain (sf polygon) used for masking
-#' @param tag Tag for the release
-#' @import terra
-get_release_climate_chelsa <- function(temp_directory = "data/temp/raw_data/climate_chelsa/",
- tag = "raw_static",
- domain){
-
- #ensure temp directory is empty
-
- if(dir.exists(temp_directory)){
- unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
- #make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
- dir.create(temp_directory,recursive = TRUE)
- }
-
-
- #Make sure there is a release by attempting to create one. If it already exists, this will fail
-
- tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
- tag = tag),
- error = function(e){message("Previous release found")})
-
- #Adjust the download timeout duration (this needs to be large enough to allow the download to complete)
-
- if(getOption('timeout') < 1000){
- options(timeout = 1000)
- }
-
-
- #Transform domain to wgs84 to get the coordinates
-
- # domain_extent <-
- # domain %>%
- # st_transform(crs("+proj=longlat +datum=WGS84 +no_defs +ellps=WGS84 +towgs84=0,0,0")@projargs)%>%
- # extent()
-
- domain_tf <-
- domain %>%
- sf::st_transform(crs("+proj=longlat +datum=WGS84 +no_defs +ellps=WGS84 +towgs84=0,0,0"))
-
- # Download the data
- # Note that it would be useful to clip these to a polygon to save space
- # It would also be useful if only the relevant data could be downloaded (rather than downloading and THEN pruning)
-
- bio_vec <-
- c("1","2","3","4","5","6","7","8","9",
- "10","11","12","13","14","15","16","17","18","19")
-
- for(i in bio_vec){
-
- # download files
- # download.file(url = paste("https://os.zhdk.cloud.switch.ch/envicloud/chelsa/chelsa_V1/climatologies/bio/CHELSA_bio10_",i,".tif",sep = ""),
- # destfile = file.path(temp_directory,paste("CHELSA_bio10_",i,"_V1.2.tif",sep = ""))
- # )
-
- # https://os.zhdk.cloud.switch.ch/chelsav2/GLOBAL/climatologies/1981-2010/bio/CHELSA_bio1_1981-2010_V.2.1.tif
- robust_download_file(url = paste("https://os.zhdk.cloud.switch.ch/chelsav2/GLOBAL/climatologies/1981-2010/bio/CHELSA_bio",i,"_1981-2010_V.2.1.tif",sep = ""),
- destfile = file.path(temp_directory,paste("CHELSA_bio",i,"_1981-2010_V.2.1.tif",sep = "")),
- max_attempts = 10,
- sleep_time = 10
- )
-
- # load
- rast_i <- terra::rast(file.path(temp_directory,paste("CHELSA_bio",i,"_1981-2010_V.2.1.tif",sep = "")))
-
- # crop
-
- rast_i <- terra::crop(x = rast_i,
- y = ext(domain_tf))
-
- # mask
- rast_i <-
- terra::mask(rast_i,
- mask = terra::vect(domain_tf))
-
- # save raster
- terra::writeRaster(x = rast_i,
- filename = file.path(temp_directory,paste("CHELSA_bio",i,"_1981-2010_V.2.1.tif",sep = "")),
- overwrite = TRUE)
-
- # plot
- # plot(rast_i)
- # plot(domain_tf,add=TRUE,col=NA)
-
- rm(rast_i)
-
- }
-
- rm(i,bio_vec)
-
-
- # release
- to_release <-
- list.files(path = file.path(temp_directory),
- recursive = TRUE,
- full.names = TRUE)
-
-
- to_release <-
- to_release[grepl(pattern = "CHELSA",
- ignore.case = TRUE,
- x = basename(to_release))]
-
- pb_upload(repo = "AdamWilsonLab/emma_envdata",
- file = to_release,
- tag = tag)
-
- # delete directory and contents
- unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
-
-
-
- message("CHELSA climate files downloaded")
- return(Sys.Date())
-
-
-} # end fx
-
diff --git a/R/get_release_clouds_wilson.R b/R/get_release_clouds_wilson.R
index b538a5c4..47a4cfca 100644
--- a/R/get_release_clouds_wilson.R
+++ b/R/get_release_clouds_wilson.R
@@ -96,15 +96,15 @@ get_release_clouds_wilson <- function(temp_directory = "data/temp/raw_data/cloud
# Push release
- pb_upload(file = file.path(temp_directory, filename),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- name = filename,
- overwrite = TRUE)
+ # pb_upload(file = file.path(temp_directory, filename),
+ # repo = "AdamWilsonLab/emma_envdata",
+ # tag = tag,
+ # name = filename,
+ # overwrite = TRUE)
# Delete file
- file.remove(file.path(temp_directory, filename))
+ # file.remove(file.path(temp_directory, filename))
# pause to keep github happy
Sys.sleep(sleep_time)
diff --git a/R/get_release_kndvi_modis.R b/R/get_release_kndvi_modis.R
deleted file mode 100644
index 8dc6d606..00000000
--- a/R/get_release_kndvi_modis.R
+++ /dev/null
@@ -1,393 +0,0 @@
-
-library(rgee)
-library(tidyverse)
-
-#' @description This function will download kndvi layers (derived from MODIS 16 day products), skipping any that have been downloaded already.
-#' @author Brian Maitner, but built from code by Qinwen, Adam, and the KNDVI ms authors
-#' @param temp_directory The directory the fire layers should be saved to prior to releasing, defaults to "data/raw_data/kndvi_modis/"
-#' @param tag tag to be used for the Github release
-#' @param domain domain (sf polygon) used for masking
-#' @param max_layers the maximum number of layers to download at once. Set to NULL to ignore. Default is 50
-#' @param sleep_time Amount of time to wait between attempts. Needed to keep github happy
-#' @import rgee
-get_release_kndvi_modis <- function(temp_directory = "data/temp/raw_data/kndvi_modis/",
- tag = "raw_kndvi_modis",
- domain,
- max_layers = 50,
- sleep_time = 1,
- json_token,
- verbose = TRUE) {
-
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
-
- if(verbose){message("Clearing directory")}
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- }
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
-
- if(verbose){message("Creating directory")}
- dir.create(temp_directory, recursive = TRUE)
-
- }
-
- #Make sure there is a release by attempting to create one. If it already exists, this will fail
-
-
- # get list releases
-
- if(verbose){message("Getting metadata for releases")}
-
- released_files <- pb_list(repo = "AdamWilsonLab/emma_envdata")
-
- #Make sure there is a release by attempting to create one. If it already exists, this will fail
-
- if(!tag %in% released_files$tag){
-
- if(verbose){message("Creating a new release")}
-
-
- tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
- tag = tag),
- error = function(e){message("Previous release found")})
-
- }
-
- #Initialize earth engine (for targets works better if called here)
-
- #ee_Initialize() #commenting this out since it wipes out manually added credentials
-
- # Load the image collection
-
- if(verbose){message("Loading image collection")}
- modis_ndvi <- ee$ImageCollection("MODIS/061/MOD13A1") #500 m
- # modis_ndvi <- ee$ImageCollection("MODIS/006/MOD13A2") #1 km
-
-
- #Format the domain
-
- if(verbose){message("Formatting the domain")}
- domain <- sf_as_ee(x = domain)
- domain <- domain$geometry()
-
- # Add a kndvi band
-
- get_kndvi <- function(img){
-
- red <- img$select("sur_refl_b01")
- nir <- img$select("sur_refl_b02")
-
- #Commented out code below is the original ee code provided by https://doi-org.gate.lib.buffalo.edu/10.1126/sciadv.abc7447
-
- #// Compute D2 a rename it to d2
- #var D2 = nir.subtract(red).pow(2).select([0],['d2'])
-
- D2 <- nir$subtract(red)$pow(2)$select(0)$rename("d2") #note that this rename should be do-able within the select, but it seems to cause issues when rgee tries to rename a single band using select
-
- # // Gamma, defined as 1/sigmaΛ2
- # var gamma = ee.Number(4e6).multiply(-2.0);
-
- gamma <- ee$Number(4e6)$multiply(-2.0)
-
- # // Compute kernel (k) and KNDVI
- # var k = D2.divide(gamma).exp();
-
- k <- D2$divide(gamma)$exp()
-
- # var kndvi = ee.Image.constant(1)
- # .subtract(k).divide(
- # ee.Image.constant(1).add(k))
- # .select([0],['knd']);
-
- kndvi <- ee$Image$constant(1)$
- subtract(k)$divide(
- ee$Image$constant(1)$add(k))$
- select(0)$rename("KNDVI")$
- set('system:time_start',img$get('system:time_start'))$ #these last lines just copy over metadata I thought might be useful
- set('system:time_end',img$get('system:time_end'))
-
- img$addBands(kndvi)
-
-
- }
-
- if(verbose){message("Generating the KNDVI data")}
-
- modis_kndvi <- modis_ndvi$map(get_kndvi)
-
- #Map$addLayer(modis_kndvi$first()$select("KNDVI"),visParams = ndviviz)
-
-
- #MODIS makes it simple to filter out poor quality pixels thanks to a quality control bits band (DetailedQA).
- #The following function helps us to distinct between good data (bit == β¦00) and marginal data (bit != β¦00).
-
- getQABits <- function(image, qa) {
- # Convert binary (character) to decimal (little endian)
- qa <- sum(2^(which(rev(unlist(strsplit(as.character(qa), "")) == 1))-1))
- # Return a mask band image, giving the qa value.
- image$bitwiseAnd(qa)$lt(1)
- }
-
- #Using getQABits we construct a single-argument function (mod13A2_clean)
- #that is used to map over all the images of the collection (modis_ndvi).
-
- mod13A1_clean <- function(img) {
-
- # Extract the NDVI band
- kndvi_values <- img$select("KNDVI")
-
- # Extract the quality band
- ndvi_qa <- img$select("SummaryQA")
-
- # Select pixels to mask
- quality_mask <- getQABits(ndvi_qa, "11")
-
- # Mask pixels with value zero.
- kndvi_values$updateMask(quality_mask)
-
-
- }
-
-
- # Clean the dataset
-
- if(verbose){message("Cleaning the data")}
-
- kndvi_clean <- modis_kndvi$map(mod13A1_clean)
-
- #Get a list of files already released
-
- if(verbose){message("Identifying which files have been released")}
-
- kndvi_tag <- tag
-
- released_files <-
- released_files %>%
- filter(tag == kndvi_tag)
-
- released_files$date <- gsub(pattern = ".tif",
- replacement = "",
- x = released_files$file_name)
-
- released_files <-
- released_files %>%
- dplyr::filter(file_name != "") %>%
- dplyr::filter(file_name != "log.csv")
-
-
- #check to see if any images have been downloaded already
-
- if(nrow(released_files) == 0){
-
- newest <- lubridate::as_date(-1) #if nothing is downloaded, start in 1970
-
- }else{
-
- newest <- max(lubridate::as_date(released_files$date)) #if there are images, start with the most recent
-
- }
-
- #Filter the data to exclude anything you've already downloaded (or older)
-
- if(verbose){message("Filtering by date")}
-
- kndvi_clean_and_new <- kndvi_clean$filterDate(start = paste(as.Date(newest+1),sep = ""),
- opt_end = paste(format(Sys.time(), "%Y-%m-%d"),sep = "") ) #I THINK I can just pull the most recent date, and then use this to download everything since then
-
-
-
- # Function to optionally limit the number of layers downloaded at once
- ## Note that this code is placed before the gain and offset adjustment, which removes the metadata needed in the date filtering
-
-
- if(verbose){message("Filtering to max layers (if needed)")}
-
- if(!is.null(max_layers)){
-
- info <- kndvi_clean_and_new$getInfo()
- to_download <- unlist(lapply(X = info$features, FUN = function(x){x$properties$`system:index`}))
- to_download <- gsub(pattern = "_", replacement = "-", x = to_download)
-
- if(length(to_download) > max_layers){
- kndvi_clean_and_new <- kndvi_clean_and_new$filterDate(start = to_download[1],
- opt_end = to_download[max_layers+1])
-
- }
-
-
- }# end if maxlayers is not null
-
-
- # This section causes errors in later layer (since early 2022). Despite months of an open ticket on earth engine, the issue persists so I'll do it with R instead
- # #Adjust gain and offset
- # adjust_gain_and_offset <- function(img){
- # img$add(1)$multiply(100)$round()
- #
- # }
-#
-#
-# kndvi_clean_and_new <- kndvi_clean_and_new$map(adjust_gain_and_offset)
-
-
- # Check if anything to download
-
- if(length(kndvi_clean_and_new$getInfo()$features) == 0 ){
-
- message("Releases are already up to date.")
- return(max(gsub(pattern = "_",replacement = "-",x = released_files$date))) #return the last date that had been done
-
- }
-
-
- #Download layers
-
- if(length(kndvi_clean_and_new$getInfo()$features) == 1 ){
-
- if(verbose){message("Downloading a single layer")}
-
- # assign name
-
- file_name <- kndvi_clean_and_new$getInfo()$features[[1]]$properties$`system:index`
-
- if(!is.Date(as_date(file_name))){stop("Error in filename")}
-
- # convert to image
-
- kndvi_clean_and_new_image <- kndvi_clean_and_new %>%
- ee$ImageCollection$toList(count = 1, offset = 0) %>%
- ee$List$get(0) %>%
- ee$Image()
-
- # download single image
-
- tryCatch(expr =
- ee_as_stars(image = kndvi_clean_and_new_image,
- region = domain,
- dsn = file.path(temp_directory,file_name),
- formatOptions = c(cloudOptimized = true),
- drive_cred_path = json_token
- ),
- error = function(e){message("Captured an error in rgee/earth engine processing of NDVI.")}
- )#trycatch
-
- }else{
-
- tryCatch(expr =
- ee_imagecollection_to_local(ic = kndvi_clean_and_new,
- region = domain,
- dsn = temp_directory,
- formatOptions = c(cloudOptimized = true),
- drive_cred_path = json_token
- #,scale = 463.3127
- ),
- error = function(e){message("Captured an error in rgee/earth engine processing of NDVI.")}
- )
-
- }#else
-
-
-
- #Push files to release
-
- # Get a lost of the local files
-
- local_files <- data.frame(local_filename = list.files(path = temp_directory,
- recursive = TRUE,
- full.names = TRUE))
-
- # end things if nothing was downloaded
-
- if(nrow(local_files) == 0){
- message("Nothing downloaded")
- return(max(gsub(pattern = "_",replacement = "-",x = released_files$date))) #return the last date that had been done
- }
-
-
- # Get a list of the local files
- local_files <- data.frame(local_filename = list.files(path = temp_directory,
- recursive = TRUE,
- full.names = TRUE))
-
-
-
- # loop through and release everything
-
- for( i in 1:nrow(local_files)){
-
- # adjusting gain and offset
- # Note: this section could be omitted if earth engine fixes their modis import
-
- # check the filename
-
-
- file_name_i <-
- local_files$local_filename[i] %>%
- basename()
-
- file_name_i <- gsub(pattern = ".tif",replacement = "",x = file_name_i)
-
- if(!is.Date(as_date(file_name_i))){stop("Error in filename")}
-
-
-
- # load the file
-
- rast_i <- terra::rast(local_files$local_filename[i])
-
- # reformat
- rast_i <- ((rast_i + 1)*100) %>%
- round()
-
- # save
-
- terra::writeRaster(x = rast_i,
- filename = local_files$local_filename[i],
- overwrite=TRUE)
-
- #cleanup
- rm(rast_i)
-
- # End gain and offset bit
-
- Sys.sleep(sleep_time) #We need to limit our rate in order to keep Github happy
-
- pb_upload(file = local_files$local_filename[i],
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag)
-
- } # end i loop
-
-
- # Delete temp files
- unlink(x = gsub(pattern = "/$", replacement = "", x = temp_directory), #sub used to delete any trailing slashes, which interfere with unlink
- recursive = TRUE)
-
- # End
-
- message("Finished Downloading KNDVI layers")
-
- local_files %>%
- filter(grepl(pattern = ".tif$",x = local_filename)) %>%
- mutate(date_format = basename(local_filename)) %>%
- mutate(date_format = gsub(pattern = ".tif",replacement = "",x = date_format)) %>%
- mutate(date_format = gsub(pattern = "_",replacement = "-",x = date_format)) %>%
- mutate(date_format = lubridate::as_date(date_format))%>%
- dplyr::select(date_format) -> local_files
-
- return(as.character(max(local_files$date_format))) # return the date of the latest file that was updated
-
-
-}# End get_kndvi fx
-
-
-
-
-
-
-
diff --git a/R/national_boundary.R b/R/national_boundary.R
deleted file mode 100644
index fd86f380..00000000
--- a/R/national_boundary.R
+++ /dev/null
@@ -1,30 +0,0 @@
-# National Boundary
-
-#' @author Adam M. Wilson
-#' @description Download national boundary file from the UN
-#' @source https://data.humdata.org/dataset/south-africa-admin-level-1-boundaries
-
-national_boundary <- function(file="data/south_africa.gpkg"){
-
- #Adjust timeout to allow for slow internet
- if(getOption('timeout') < 1000){
- options(timeout = 1000)
- }
-
-
- url="https://data.humdata.org/dataset/061d4492-56e8-458c-a3fb-e7950991adf0/resource/f5b08257-8d03-48dc-92c8-aaa4fb7285f0/download/zaf_adm_sadb_ocha_20201109_shp.zip"
- tmpfile1=tempfile()
- tmpdir1=tempdir()
- download.file(url,destfile = tmpfile1)
- unzip(tmpfile1,exdir=tmpdir1)
-
- country=st_read(file.path(tmpdir1,"zaf_adm_sadb_ocha_20201109_SHP/zaf_admbnda_adm0_sadb_ocha_20201109.shp"))
-# st_write(country,dsn=file,append=F)
-# return(file)
-
- return(country)
-}
-# end function
-
-
-
diff --git a/R/network_drive_explore.R b/R/network_drive_explore.R
deleted file mode 100644
index 48a9a49c..00000000
--- a/R/network_drive_explore.R
+++ /dev/null
@@ -1,62 +0,0 @@
-# # box code not working
-# library(boxr)
-# library(tidyverse)
-#
-# # library(fs)
-# # dir_create("~/.boxr-auth")
-#
-# # from https://github.com/r-box/boxr/issues/166
-# # options(boxr.retry.times = 10)
-#
-# #box_auth()
-#
-# # sometimes this fails for no apparent reason - must just wait for it to work again!
-# # https://github.com/r-box/boxr/issues/166
-# box_auth_service()
-# #box_auth_service(token_text = unlist(read_lines('~/.boxr-auth/token.json')))
-#
-# # root directory
-#
-#
-# box_setwd()
-#
-#
-# # create new directory to hold results
-# dir_name="emmabox"
-# box_dir_create(dir_name = dir_name)
-#
-# eid <- box_ls() %>% as.data.frame() %>% filter(name==dir_name)
-#
-# # Share folder with
-# uid=271686873 # adamw@buffalo.edu
-#
-#
-# box_collab_create(
-# dir_id = eid$id,
-# user_id = uid,
-# role = "co-owner",
-# can_view_path = TRUE
-# )
-#
-# # set box working directory
-# box_setwd(eid$id)
-# box_setwd("..")
-#
-# boxr_options()
-# box_ls()
-#
-# # test writing to folder
-# box_write(
-# iris,
-# "iris.csv")
-#
-# files <- box_ls() %>% as.data.frame
-#
-# files %>%
-# filter(name=="iris.csv") %>%
-# select(id) %>% unlist() %>%
-# box_delete_file()
-#
-#
-#
-#
diff --git a/R/process_dynamic_data.R b/R/process_dynamic_data.R
new file mode 100644
index 00000000..26f8e33b
--- /dev/null
+++ b/R/process_dynamic_data.R
@@ -0,0 +1,101 @@
+
+library(arrow)
+library(tidyverse)
+
+
+#############################################################
+#' @author Brian Maitner
+#' @description this function takes in tif file from the input, converts them to tidy format, and saves as .gz.parquet
+#' @param input_dir Directory containing input files.
+#' @param output_dir Directory to stick output files in
+#' @param variable_name This is included in the tidy file output
+#' @param ... Does nothing. Used for targets.
+#' @note Output dataframes have three columns: CellID, date, variable, value. ALso note that cells with NA values are omitted.
+process_dynamic_data_to_parquet <- function(input_dir = "data/raw_data/ndvi_modis/",
+ output_dir = "data/processed_data/dynamic_parquet/ndvi/",
+ variable_name = "ndvi",
+ ...){
+
+
+ # make a directory if one doesn't exist yet
+
+ if(!dir.exists(output_dir)){
+
+ dir.create(output_dir, recursive = TRUE)
+
+ }
+
+ # get files
+
+ all_files <- list.files(path = input_dir,pattern = ".tif$",full.names = TRUE)
+
+ all_files_int <-
+ all_files %>%
+ gsub(pattern = input_dir, replacement = "") %>%
+ gsub(pattern = "/", replacement = "") %>%
+ gsub(pattern = ".tif", replacement = "") %>%
+ lubridate::as_date()|>
+ as.numeric()
+
+
+ # figure out which files have been processed
+
+ output_files <-
+ list.files(path = output_dir, pattern = ".gz.parquet", full.names = TRUE) %>%
+ gsub(pattern = output_dir, replacement = "") %>%
+ gsub(pattern = "/", replacement = "") %>%
+ gsub(pattern = ".gz.parquet", replacement = "")
+
+ #Don't worry about files that have been processed already
+
+ all_files <- all_files[which(!all_files_int %in% output_files)]
+
+ rm(output_files, all_files_int)
+
+ #end if things are already done
+ if(length(all_files) == 0){
+
+ message(paste("Finished converting ",
+ variable_name,
+ " files to parquet", sep = ""))
+
+ return(output_dir)
+
+ }
+
+
+ # process the files that haven't been done yet
+
+
+ for(i in 1:length(all_files)){
+
+ # Get the date in integer format (will append to the data)
+ all_files[i] |>
+ gsub(pattern = input_dir, replacement = "")|>
+ gsub(pattern = "/", replacement = "")|>
+ gsub(pattern = ".tif", replacement = "")|>
+ lubridate::as_date()|>
+ as.numeric()-> integer_date_i
+
+
+ # Process ith file
+
+ all_files[i] |>
+ stars::read_stars() |>
+ as.data.frame() %>%
+ mutate(cellID = row_number(),
+ date = integer_date_i,
+ variable = variable_name) %>%
+ rename( value := 3) %>%
+ dplyr::select(cellID, date, variable, value ) %>%
+ drop_na() %>%
+ write_parquet(sink = paste(output_dir, integer_date_i, ".gz.parquet", sep = ""),
+ compression = "gzip")
+
+ } #end i loop
+
+ #End fx
+ message(paste("Finished converting ",variable_name, " files to parquet",sep = ""))
+ return(output_dir)
+
+}#end fx
diff --git a/R/process_fix_modis_NDVI_release_extent.R b/R/process_fix_modis_NDVI_release_extent.R
deleted file mode 100644
index 828f0593..00000000
--- a/R/process_fix_modis_NDVI_release_extent.R
+++ /dev/null
@@ -1,317 +0,0 @@
-temp_directory = "data/temp/raw_data/ndvi_modis_extent/"
-tag = "raw_ndvi_modis"
-max_layers = NULL
-sleep_time = 1
-verbose=TRUE
-
-
-# code to fix raster extent issues
-
-# MODIS 6.1 fire rasters from 2000 to 2019 have one set of extents, 2020 another, and 2021 - present another.
-#' @description to check the Extent of MODIS products downloaded from rgee
-#' @author Brian Maitner
-#' @param temp_directory The directory the layers should be temporarily saved in
-#' @param tag tag associated with the Github release
-#' @param max_layers the maximum number of layers to correct at once. Default (NULL) is to use all.
-#' @param sleep_time amount of time to pause after using pb_upload/download. Used to keep Git happy
-process_fix_modis_NDVI_release_extent <- function(temp_directory,
- tag,
- max_layers = NULL,
- sleep_time = 0.1,
- verbose=FALSE,
- ...){
-
- # get a list of released files
-
- if(verbose){message("Downloading list of releases")}
-
- released_files <- pb_list(repo = "AdamWilsonLab/emma_envdata",
- tag = tag)
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
-
- if(verbose){message("Emptying directory")}
-
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
-
- if(verbose){message("Creating directory")}
-
- dir.create(temp_directory, recursive = TRUE)
-
- }
-
- #set up a change log if needed
-
- if("extent_log.csv" %in% released_files$file_name){
-
- if(verbose){message("Downloading log")}
-
-
- robust_pb_download(file = "extent_log.csv",
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
-
- }else{
-
- if(verbose){message("Creating log")}
-
- suppressWarnings(expr =
- cbind("file","original_extent","final_extent") %>%
- write.table(x = .,
- file = file.path(temp_directory,"extent_log.csv"),
- append = FALSE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
- )
-
-
- }
-
-
- # Get a list of raster that haven't been fixed by comparison with the log
-
- if(verbose){message("Identifying rasters in need of processing")}
-
- rasters <- released_files$file_name[grep(x = released_files$file_name, pattern = ".tif")]
-
- log <- read.csv(file.path(temp_directory, "extent_log.csv"))
-
- rasters <- rasters[which(!rasters %in% log$file)]
-
- if(!is.null(max_layers)){
-
- if(max_layers < length(rasters)){
-
- rasters <- rasters[1:max_layers]
-
- }
-
-
-
- }
-
-
- # check whether there is anything left to fix
-
- if(length(rasters) == 0){
-
- message(paste("Finished checking ",tag," extents",sep = ""))
-
- return(
- released_files %>%
- filter(tag == tag) %>%
- dplyr::select(file_name) %>%
- filter(file_name != "") %>%
- filter(grepl(pattern = ".tif$", x = file_name)) %>%
- mutate(date_format = gsub(pattern = ".tif",
- replacement = "",
- x = file_name))%>%
- mutate(date_format = gsub(pattern = "_", replacement = "-",
- x = date_format)) %>%
- dplyr::pull(date_format) %>%
- max()
- )
-
-
- }
-
- # IF extent checking is required, load the template used for comparison
-
- if(verbose){message("Using first MODIS layer as template")}
-
-
-
- robust_pb_download(file = released_files$file_name[1],
- dest = file.path(temp_directory),
- repo = paste(released_files$owner[1],
- released_files$repo[1],sep = "/"),
- tag = released_files$tag[1],
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
-
- template <- rast(file.path(temp_directory,released_files$file_name[1]))
-
- template_extent <- ext(template) |> as.character()
-
- # iterate and fix
-
- for(i in 1:length(rasters)){
-
- if(verbose){message("Checking raster ", i, " of ", length(rasters))}
-
- # download ith raster
-
- robust_pb_download(file = rasters[i],
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- Sys.sleep(sleep_time)
-
- # load ith raster
-
- rast_i <- terra::rast(x = file.path(temp_directory,rasters[i]))
-
- # get the extent
-
- original_extent <- ext(rast_i) |> as.character()
-
- # check whether the raster matches the correct extent
-
- if(!identical(template_extent, original_extent)){
-
- message("Detected error in MODIS extent, correcting and logging the change")
-
- rast_i <- terra::resample(rast_i,y = template,method="near")
-
- # write a new raster with a different name
-
- terra::writeRaster(x = rast_i,
- filename = file.path(temp_directory,gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = rasters[i])),
- filetype="GTiff",
- overwrite = TRUE)
-
- # delete old raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
-
- # update new name
-
- file.rename(from = file.path(temp_directory,gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = rasters[i])),
- to = file.path(temp_directory, rasters[i]))
-
- #log the change
-
-
- data.frame(file = rasters[i],
- original_extent = original_extent,
- final_extent = template_extent) %>%
-
- write.table(x = .,
- file = file.path(temp_directory,"extent_log.csv"),
- append = TRUE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
-
- # push the updated raster
-
- # pb_upload(file = file.path(temp_directory,rasters[i]),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag,
- # name = rasters[i], overwrite = TRUE)
-
- robust_pb_upload(files = file.path(temp_directory,rasters[i]),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- name = rasters[i],
- overwrite = TRUE,
- sleep_time = sleep_time)
-
-
- #Sys.sleep(sleep_time)
-
- # push the updated log
-
- # pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag)
-
- robust_pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- sleep_time = sleep_time)
-
- #Sys.sleep(sleep_time)
-
- # Delete the new raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
- }else{
-
- #if the projection is correct, log it
-
- data.frame(file = rasters[i],
- original_extent = original_extent,
- final_extent = template_extent) %>%
-
- write.table(x = .,
- file = file.path(temp_directory,"extent_log.csv"),
- append = TRUE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
-
- # pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag)
-
- # robust_pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag)
-
- robust_pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- sleep_time = sleep_time)
-
-
- # Sys.sleep(sleep_time)
-
- unlink(file.path(temp_directory,rasters[i]))
-
-
- }
-
- } #for i rasters loop
-
- # Cleanup and end
-
- if(verbose){message("Cleaning up")}
-
-
- # Delete temp files
- unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- # Finish up
-
- message(paste("Finished checking ",tag," extents",sep = ""))
-
- return(
- rasters |>
- gsub(pattern = ".tif", replacement = "") |>
- gsub(pattern = "_",replacement = "-") |>
- max()
- )
-
-
-
-
-
-
-} #end fx
diff --git a/R/process_fix_modis_release_extent.R b/R/process_fix_modis_release_extent.R
deleted file mode 100644
index 52d32b35..00000000
--- a/R/process_fix_modis_release_extent.R
+++ /dev/null
@@ -1,285 +0,0 @@
-
-# code to fix raster extent issues
-
-# MODIS 6.1 fire rasters from 2000 to 2019 have one set of extents, 2020 another, and 2021 - present another.
-#' @description to check the Extebt of MODIS products downloaded from rgee
-#' @author Brian Maitner
-#' @param temp_directory The directory the layers should be temporarily saved in
-#' @param tag tag associated with the Github release
-#' @param template_release Information on where to find a template to use
-#' @param max_layers the maximum number of layers to correct at once. Default (NULL) is to use all.
-#' @param sleep_time amount of time to pause after using pb_upload/download. Used to keep Git happy
-process_fix_modis_release_extent <- function(temp_directory,
- tag,
- template_release = template_release,
- max_layers = NULL,
- sleep_time = 0.1,
- verbose=FALSE,
- ...){
-
- # get a list of released files
-
- if(verbose){message("Downloading list of releases")}
-
- released_files <- pb_list(repo = "AdamWilsonLab/emma_envdata",
- tag = tag)
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
-
- if(verbose){message("Emptying directory")}
-
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
-
- if(verbose){message("Creating directory")}
-
- dir.create(temp_directory, recursive = TRUE)
-
- }
-
- #set up a change log if needed
-
- if("extent_log.csv" %in% released_files$file_name){
-
- if(verbose){message("Downloading log")}
-
-
- robust_pb_download(file = "extent_log.csv",
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
-
- }else{
-
- if(verbose){message("Creating log")}
-
- suppressWarnings(expr =
- cbind("file","original_extent","final_extent") %>%
- write.table(x = .,
- file = file.path(temp_directory,"extent_log.csv"),
- append = FALSE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
- )
-
-
- }
-
-
- # Get a list of raster that haven't been fixed by comparison with the log
-
- if(verbose){message("Identifying rasters in need of processing")}
-
- rasters <- released_files$file_name[grep(x = released_files$file_name, pattern = ".tif")]
-
- log <- read.csv(file.path(temp_directory, "extent_log.csv"))
-
- rasters <- rasters[which(!rasters %in% log$file)]
-
- if(!is.null(max_layers)){
-
- if(max_layers < length(rasters)){
-
- rasters <- rasters[1:max_layers]
-
- }
-
-
-
- }
-
-
- # check whether there is anything left to fix
-
- if(length(rasters) == 0){
-
- message(paste("Finished checking ",tag," extents",sep = ""))
-
- return(
- released_files %>%
- filter(tag == tag) %>%
- dplyr::select(file_name) %>%
- filter(file_name != "") %>%
- filter(grepl(pattern = ".tif$", x = file_name)) %>%
- mutate(date_format = gsub(pattern = ".tif",
- replacement = "",
- x = file_name))%>%
- mutate(date_format = gsub(pattern = "_", replacement = "-",
- x = date_format)) %>%
- dplyr::pull(date_format) %>%
- max()
- )
-
-
- }
-
- # IF extent checking is required, load the template used for comparison
-
- if(verbose){message("Downloading template")}
-
- robust_pb_download(file = template_release$file,
- dest = file.path(temp_directory),
- repo = template_release$repo,
- tag = template_release$tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = 10)
-
- template <- rast(file.path(temp_directory,template_release$file))
-
- template_extent <- ext(template) |> as.character()
-
- # iterate and fix
-
- for(i in 1:length(rasters)){
-
- if(verbose){message("Checking raster ", i, " of ", length(rasters))}
-
- # download ith raster
-
- robust_pb_download(file = rasters[i],
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- Sys.sleep(sleep_time)
-
- # load ith raster
-
- rast_i <- terra::rast(x = file.path(temp_directory,rasters[i]))
-
- # get the extent
-
- original_extent <- ext(rast_i) |> as.character()
-
- # check whether the raster matches the correct extent
-
- if(!identical(template_extent, original_extent)){
-
- message("Detected error in MODIS extent, correcting and logging the change")
-
- rast_i <- terra::resample(rast_i,y = template,method="near")
-
- # write a new raster with a different name
-
- terra::writeRaster(x = rast_i,
- filename = file.path(temp_directory,gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = rasters[i])),
- filetype="GTiff",
- overwrite = TRUE)
-
- # delete old raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
-
- # update new name
-
- file.rename(from = file.path(temp_directory,gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = rasters[i])),
- to = file.path(temp_directory, rasters[i]))
-
- #log the change
-
-
- data.frame(file = rasters[i],
- original_extent = original_extent,
- final_extent = template_extent) %>%
-
- write.table(x = .,
- file = file.path(temp_directory,"extent_log.csv"),
- append = TRUE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
-
- # push the updated raster
-
- robust_pb_upload(file = file.path(temp_directory,rasters[i]),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- name = rasters[i],
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- # push the updated log
-
- robust_pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- max_attempts = 10,
- sleep_time = sleep_time,
- overwrite = TRUE)
-
- # Delete the new raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
- }else{
-
- #if the projection is correct, log it
-
- data.frame(file = rasters[i],
- original_extent = original_extent,
- final_extent = template_extent) %>%
-
- write.table(x = .,
- file = file.path(temp_directory,"extent_log.csv"),
- append = TRUE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
-
- robust_pb_upload(file = file.path(temp_directory,"extent_log.csv"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- max_attempts = 10,
- sleep_time = sleep_time,
- overwrite = TRUE)
-
- Sys.sleep(sleep_time)
-
- unlink(file.path(temp_directory,rasters[i]))
-
- }
-
- } #for i rasters loop
-
- # Cleanup and end
-
- if(verbose){message("Cleaning up")}
-
-
- # Delete temp files
- unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- # Finish up
-
- message(paste("Finished checking ",tag," extents",sep = ""))
-
- return(
- rasters |>
- gsub(pattern = ".tif", replacement = "") |>
- gsub(pattern = "_",replacement = "-") |>
- max()
- )
-
-} #end fx
diff --git a/R/process_fix_modis_release_projection.R b/R/process_fix_modis_release_projection.R
deleted file mode 100644
index 60743be3..00000000
--- a/R/process_fix_modis_release_projection.R
+++ /dev/null
@@ -1,278 +0,0 @@
-
-#' @description to check the projection of MODIS products downloaded from rgee
-#' @author Brian Maitner
-#' @param temp_directory The directory the layers should be temporarily saved in
-#' @param tag tag associated with the Github release
-#' @param max_layers the maximum number of layers to correct at once. Default (NULL) is to use all.
-#' @param sleep_time amount of time to pause after using pb_upload/download. Used to keep Git happy
-#' @param verbose. More messages are shown
-process_fix_modis_release_projection <-
- function(temp_directory,
- tag,
- max_layers = NULL,
- sleep_time = 0.1,
- verbose = TRUE,
- ...){
-
- # specify the correct projection
- nasa_proj <- "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +R=6371007.181 +units=m +no_defs"
-
-
- # get a list of released files
- released_files <- pb_list(repo = "AdamWilsonLab/emma_envdata",
- tag = tag)
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
- dir.create(temp_directory, recursive = TRUE)
- }
-
- #set up a change log if needed
-
- if("log.csv" %in% released_files$file_name){
-
- robust_pb_download(file = "log.csv",
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
-
- }else{
-
- suppressWarnings(expr =
- cbind("file","original_proj","assigned_proj") %>%
- write.table(x = .,
- file = file.path(temp_directory,"log.csv"),
- append = FALSE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
- )
-
-
- }
-
- #Get a list of raster that haven't been fixed by comparison with the log
-
- rasters <- released_files$file_name[grep(x = released_files$file_name, pattern = ".tif")]
-
- log <- read.csv(paste(temp_directory, "log.csv", sep = ""))
-
- rasters <- rasters[which(!rasters %in% log$file)]
-
- if(!is.null(max_layers)){
-
- if(max_layers < length(rasters)){
-
- rasters <- rasters[1:max_layers]
-
- }
-
-
-
- }
-
-
- # check whether there is anything left to fix
-
- if(length(rasters) == 0){
-
- message(paste("Finished updating ",tag," projections",sep = ""))
-
- return(
- released_files %>%
- filter(tag == tag) %>%
- dplyr::select(file_name) %>%
- filter(file_name != "") %>%
- filter(grepl(pattern = ".tif$", x = file_name)) %>%
- mutate(date_format = gsub(pattern = ".tif",
- replacement = "",
- x = file_name))%>%
- mutate(date_format = gsub(pattern = "_", replacement = "-",
- x = date_format)) %>%
- dplyr::pull(date_format) %>%
- max()
- )
-
-
- }
-
-
-
- #iterate and fix
- for(i in 1:length(rasters)){
-
- if(verbose){message("Checking raster ", i, " of ", length(rasters))}
-
- # download ith raster
-
- robust_pb_download(file = rasters[i],
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- Sys.sleep(sleep_time)
-
- # load ith raster
-
- rast_i <- terra::rast(x = file.path(temp_directory,rasters[i]))
-
- # get the projection
-
- original_proj <- crs(rast_i, proj = TRUE)
-
- # check whether the raster matches the correct projection
- if(!identical(nasa_proj, original_proj)){
-
- message("Detected error in MODIS projection for raster ",rasters[i],
- " correcting and logging the change")
-
- crs(rast_i) <- nasa_proj
-
- # write a new raster with a different name
-
- terra::writeRaster(x = rast_i,
- filename = gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = file.path(temp_directory,rasters[i])),
- filetype="GTiff",
- overwrite = TRUE)
-
- # delete old raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
-
- # update new name
-
- file.rename(from = gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = file.path(temp_directory,rasters[i])),
- to = file.path(temp_directory, rasters[i]))
-
- #log the change
-
-
- data.frame(file = rasters[i],
- original_proj = original_proj,
- assigned_proj = nasa_proj) %>%
-
- write.table(x = .,
- file = file.path(temp_directory,"log.csv"),
- append = TRUE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
-
- # push the updated raster
-
- # pb_upload(file = file.path(temp_directory,rasters[i]),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag,
- # name = rasters[i], overwrite = TRUE)
-
- robust_pb_upload(file = file.path(temp_directory,rasters[i]),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- name = rasters[i],
- overwrite = TRUE,
- sleep_time = sleep_time)
-
- #Sys.sleep(sleep_time)
-
-
-
- # push the updated log
-
- # pb_upload(file = file.path(temp_directory,"log.csv"),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag)
- #
- # Sys.sleep(sleep_time)
-
- robust_pb_upload(file = file.path(temp_directory,"log.csv"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- sleep_time = sleep_time)
-
-
- # Delete the new raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
- }else{
-
- #if the projection is correct, log it
-
- if(verbose){message("Projection for raster ", rasters[i], " looks correct")}
-
- data.frame(file = rasters[i],
- original_proj = original_proj,
- assigned_proj = nasa_proj) %>%
-
- write.table(x = .,
- file = file.path(temp_directory,"log.csv"),
- append = TRUE,
- col.names = FALSE,
- row.names=FALSE,
- sep = ",")
-
- # pb_upload(file = file.path(temp_directory,"log.csv"),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag)
- #
- # Sys.sleep(sleep_time)
-
- robust_pb_upload(file = file.path(temp_directory,"log.csv"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- name = NULL,
- sleep_time = sleep_time)
-
-
- unlink(file.path(temp_directory,rasters[i]))
-
-
- }
-
- } #for i rasters loop
-
- # Cleanup and end
-
- # Delete temp files
- unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- # Finish up
-
- message(paste("Finished updating ",tag," projections",sep = ""))
-
- return(
- rasters |>
- gsub(pattern = ".tif", replacement = "") |>
- gsub(pattern = "_",replacement = "-") |>
- max()
- )
-
-
-
-
-
-} # end function
-
-###########################
-
-
diff --git a/R/process_fix_modis_release_projection_and_extent.R b/R/process_fix_modis_release_projection_and_extent.R
deleted file mode 100644
index 914d349f..00000000
--- a/R/process_fix_modis_release_projection_and_extent.R
+++ /dev/null
@@ -1,288 +0,0 @@
-#' @description to correct the projection and extent of MODIS products downloaded from rgee
-#' @author Brian Maitner
-#' @param temp_directory The directory the layers should be temporarily saved in
-#' @param input_tag tag associated with the Github release
-#' @param output_tag
-#' @param max_layers the maximum number of layers to correct at once. Default (NULL) is to use all.
-#' @param sleep_time amount of time to pause after using pb_upload/download. Used to keep Git happy
-#' @param verbose. More messages are shown
-process_fix_modis_release_projection_and_extent <-
- function(temp_directory,
- input_tag,
- output_tag,
- max_layers = NULL,
- sleep_time = 0.1,
- verbose = TRUE,
- ...){
-
-
- # specify the correct projection
- nasa_proj <- "+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +R=6371007.181 +units=m +no_defs"
-
-
- # get a list of released files
- released_files <- pb_list(repo = "AdamWilsonLab/emma_envdata",
- tag = c(input_tag,output_tag,"raw_ndvi_modis"))
-
- # filter to only tifs
-
- released_files %>%
- filter(grepl(x = file_name, pattern = ".tif")) -> released_files
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
- dir.create(temp_directory, recursive = TRUE)
- }
-
- # make releases if needed
-
- #Make sure there is an input release
-
- if(!input_tag %in% released_files$tag){
-
- if(verbose){message("Creating a new release")}
-
- tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
- tag = input_tag),
- error = function(e){message("Previous release found")})
-
-
- }
-
- #Make sure there is an output release
-
- if(!output_tag %in% released_files$tag){
-
- if(verbose){message("Creating a new release")}
-
- tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
- tag = output_tag),
- error = function(e){message("Previous release found")})
-
-
- }
-
- #Get a list of raster that haven't been fixed by comparing input and output
-
- input_rasters <- released_files %>% filter(tag == input_tag)
-
- output_rasters <- released_files %>% filter(tag == output_tag)
-
- rasters <- input_rasters %>%
- filter(!file_name %in% output_rasters$file_name) %>%
- pull(file_name)
-
- # Only do the first n layers if max_layers specified, otherwise do all of them
-
- if(!is.null(max_layers)){
-
- if(max_layers < length(rasters)){
-
- rasters <- rasters[1:max_layers]
-
- }
- }
-
- # check whether there is anything left to fix
-
- if(length(rasters) == 0){
-
- message(paste("Finished updating ",tag," projections",sep = ""))
-
- return(
- released_files %>%
- filter(tag == tag) %>%
- dplyr::select(file_name) %>%
- filter(file_name != "") %>%
- filter(grepl(pattern = ".tif$", x = file_name)) %>%
- mutate(date_format = gsub(pattern = ".tif",
- replacement = "",
- x = file_name))%>%
- mutate(date_format = gsub(pattern = "_", replacement = "-",
- x = date_format)) %>%
- dplyr::pull(date_format) %>%
- max()
- )
- }
-
- # get template raster to align extent to
-
- # IF extent checking is required, load the template used for comparison
-
- if(verbose){message("Using first MODIS layer as template")}
-
- template <-
- released_files %>%
- filter(tag == "raw_ndvi_modis") %>%
- arrange(file_name) %>%
- slice_head(n=1)
-
- robust_pb_download(file = template$file_name[1],
- dest = file.path(temp_directory),
- repo = paste(template$owner[1],
- template$repo[1],sep = "/"),
- tag = template$tag[1],
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- #rename the file being used as template. This is done for rare instances where it might confict with other names
- file.rename(from = file.path(temp_directory,template$file_name[1]),
- to = file.path(temp_directory,"template.tif"))
-
- template <- rast(file.path(temp_directory,"template.tif"))
-
- template_extent <- ext(template) |> as.character()
-
-
- # iterate and fix
-
- for(i in 1:length(rasters)){
-
- if(verbose){message("Checking raster ", i, " of ", length(rasters))}
-
- # download ith raster
-
- robust_pb_download(file = rasters[i],
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = input_tag,
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- # load ith raster
-
- rast_i <- terra::rast(x = file.path(temp_directory,rasters[i]))
-
- # get the projection
-
- original_proj <- crs(rast_i, proj = TRUE)
-
- # check whether the raster matches the correct projection
-
- if(!identical(nasa_proj, original_proj)){
-
- message("Detected error in MODIS projection for raster ",rasters[i],
- " correcting")
-
- crs(rast_i) <- nasa_proj
-
- }else{
-
- if(verbose){message("Raster ", rasters[i], " projection looks good")}
-
- }
-
-
- # check get the extent
-
- original_extent <- ext(rast_i) |> as.character()
-
- # check whether the raster matches the correct extent
-
- if(!identical(template_extent, original_extent)){
-
- message("Detected error in MODIS extent, correcting")
-
- rast_i <- terra::resample(rast_i,y = template,method="near")
-
- }else{
-
- if(verbose){message("Raster ", rasters[i], " extent looks good")}
-
- }
-
-
- # write a new raster with a different name
-
- terra::writeRaster(x = rast_i,
- filename = gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = file.path(temp_directory,rasters[i])),
- filetype="GTiff",
- overwrite = TRUE)
-
- # delete old raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
- # update new name
-
- file.rename(from = gsub(pattern = ".tif$",
- replacement =".temp.tif",
- x = file.path(temp_directory,rasters[i])),
- to = file.path(temp_directory, rasters[i]))
-
- # check that updates worked
-
- updated_raster <- terra::rast(x = file.path(temp_directory,rasters[i]))
-
-
- if(!identical(nasa_proj, crs(updated_raster, proj=TRUE))){
- stop("Error in fixing CRS")
-
- }
-
- if(!identical(template_extent,
- ext(updated_raster) |> as.character())){
-
- message("Error in fixing extent")
-
- }
-
- # push the updated raster
-
- pb_upload(file = file.path(temp_directory,rasters[i]),
- repo = "AdamWilsonLab/emma_envdata",
- tag = output_tag,
- name = rasters[i],
- overwrite = TRUE)
-
- Sys.sleep(sleep_time)
-
- # Using the regular pb_upload because it uses fewer queries
-
- # robust_pb_upload(file = file.path(temp_directory,rasters[i]),
- # repo = "AdamWilsonLab/emma_envdata",
- # tag = tag,
- # name = rasters[i],
- # overwrite = TRUE,
- # sleep_time = sleep_time)
-
-
- # Delete the new raster
-
- unlink(file.path(temp_directory,rasters[i]))
-
- } #for i rasters loop
-
- # Cleanup and end
-
- if(verbose){message("Cleaning up")}
-
- # Delete temp files
-
- unlink(x = file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- # Finish up
-
- message(paste("Finished checking ",input_tag," extents and projections",sep = ""))
-
- return(
- rasters |>
- gsub(pattern = ".tif", replacement = "") |>
- gsub(pattern = "_",replacement = "-") |>
- max()
- )
-
-
- } #end fx
diff --git a/R/process_release_alos.R b/R/process_release_alos.R
deleted file mode 100644
index 1c488feb..00000000
--- a/R/process_release_alos.R
+++ /dev/null
@@ -1,160 +0,0 @@
-#' @author Brian Maitner
-#' @param input_dir directory where the input files live
-#' @param output_dir directory for the output files
-#' @param template_release path to raster file to use as a template for reprojection
-#' @param ... Does nothing, but is used in making connections between files in the targets framework
-#' @note This function uses bilinear for continuous variables and nearest neighbor for categorical
-process_release_alos <- function(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/alos/",
- template_release,
- sleep_time = 30,
- ...){
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
- dir.create(temp_directory, recursive = TRUE)
- }
-
-
- # get template raster
-
- robust_pb_download(file = template_release$file,
- dest = temp_directory,
- repo = template_release$repo,
- tag = template_release$tag,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- template <- terra::rast(file.path(temp_directory, template_release$file))
-
- # get input rasters
-
- raster_list <- pb_list(repo = "AdamWilsonLab/emma_envdata",
- tag = input_tag) %>%
- filter(grepl(pattern = "alos_",
- x = file_name))
-
- robust_pb_download(file = raster_list$file_name,
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = input_tag,
- max_attempts = 10,
- sleep_time = sleep_time)
-
-
- # reformat and save each
-
- for(i in 1:nrow(raster_list)){
-
- raster_i <- terra::rast(file.path(temp_directory, raster_list$file_name[i]))
-
-
- #Use bilinear for everything except landforms
-
- if(length(grep(pattern = "landforms", x = raster_list$file_name[i])) > 0){
-
- method <- "near" # uncomment for terra
-
- }else{
-
- method <- "bilinear"
-
- }
-
- # terra doesn't overwrite, so I have to delete and rename
-
- terra::project(x = raster_i,
- y = template,
- method = method,
- filename = file.path(temp_directory,
- gsub(pattern = ".tif$",
- replacement = ".temp.tif",
- x = raster_list$file_name[i])),
- overwrite=TRUE)
-
- # check the projection
-
- if(terra::crs(rast(file.path(temp_directory,
- gsub(pattern = ".tif$",
- replacement = ".temp.tif",
- x = raster_list$file_name[i]))),
- proj=TRUE) != terra::crs(template, proj=TRUE)){
- stop("Issue with reprojection")}
-
- # delete the original
-
- file.remove(file.path(temp_directory, raster_list$file_name[i]))
-
- file.rename(from = file.path(temp_directory,
- gsub(pattern = ".tif$",
- replacement = ".temp.tif",
- x = raster_list$file_name[i])),
- to = file.path(temp_directory, raster_list$file_name[i]))
-
- # check the new projection
-
- if(terra::crs(rast(file.path(temp_directory, raster_list$file_name[i])),
- proj=TRUE) != terra::crs(template, proj=TRUE)){
- stop("Issue with reprojection")}
-
- # upload the new file
-
- pb_upload(file = file.path(temp_directory, raster_list$file_name[i]),
- repo = "AdamWilsonLab/emma_envdata",
- tag = output_tag,
- name = raster_list$file_name[i])
-
- rm(raster_i)
-
- file.remove(file.path(temp_directory, raster_list$file_name[i]))
-
- Sys.sleep(sleep_time)
-
-
- } #i loop
-
-
- #Clear out the folder
-
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- #End functions
-
- message("Finished processing ALOS layers")
- return(invisible(NULL))
-
-
-
-} #end fx
-
-
-
-#################################
-
-
-# CSP/ERGo/1_0/Global/ALOS_mTPI
-#continuous measure of hills vs valleys -> bilinear
-#270
-
-
-# "CSP/ERGo/1_0/Global/ALOS_CHILI"
-#continuous -> bilinear
-#90
-
-# 'CSP/ERGo/1_0/Global/ALOS_landforms'
-# categorical land classes -> near
-#90 meter
-
-# 'CSP/ERGo/1_0/Global/ALOS_topoDiversity'
-#continuous -> bilinear
-#270 m
-
-
diff --git a/R/process_release_biome_raster.R b/R/process_release_biome_raster.R
deleted file mode 100644
index 95d2875b..00000000
--- a/R/process_release_biome_raster.R
+++ /dev/null
@@ -1,107 +0,0 @@
-# Make Domain
-
-#' @author Brian S. Maitner
-
-# Process 2018 Vegetation dataset into a raster with MODIS specs
-
-
-#' @param vegmap_shp path to the 2018 national vegetation map shapefile
-#' @param template_release path information to template release.
-#' @param temp_directory temporary directory. will be deleted.
-#' @param sleep_time amount of time (in seconds) to pause after a Github query. Defaults to 10.
-
-process_release_biome_raster <- function(template_release,
- vegmap_shp,
- domain,
- temp_directory = "data/temp/raw_data/vegmap_raster/",
- sleep_time = 10){
-
- # Ensure directory is empty if it exists
-
- if(dir.exists(file.path(temp_directory))){
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(file.path(temp_directory))){
- dir.create(file.path(temp_directory), recursive = TRUE)
- }
-
- # get template raster
-
- robust_pb_download(file = template_release$file,
- dest = temp_directory,
- repo = template_release$repo,
- tag = template_release$tag,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- # load template
-
- template <- terra::rast(file.path(temp_directory, template_release$file))
-
- # load vegmap
-
- vegmap_za <- st_read(vegmap_shp) %>%
- janitor::clean_names() %>%
- st_make_valid() %>%
- st_transform(crs = crs(template))
-
- #transform domain
- domain %>% st_transform(crs = crs(template)) -> domain
-
- #crop vegmap to save size?
-
- vegmap_za %>%
- st_intersection(y = domain) -> vegmap_za
-
- # rasterize vegmap
-
- # Note: the Github version of exactextractr could do this more simply using exactextractr::coverage_fraction()
-
- n <- 10 #number of subcells to use for aggregation
-
- template <- disagg(rast(template), n) #break raster into smaller one
-
- #r <- disagg(template, n) #break raster into smaller one: this is more memory-intense
-
- r <- rasterize(x = vect(vegmap_za),
- y = template,
- field = "biome_18") #rasterize at fine resolution
-
- out_rast <- aggregate(r, n, "modal") #re-aggregate using modal biome
-
- # save output version
-
- writeRaster(x = out_rast,
- filename = file.path(temp_directory,"biome_raster_modis_proj.tif"),
- overwrite=TRUE)
-
- # upload transformed version
-
- pb_upload(file = file.path(temp_directory,"biome_raster_modis_proj.tif"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = "processed_static",
- name = "biome_raster_modis_proj.tif")
-
- pb_upload(file = file.path(temp_directory,"biome_raster_modis_proj.tif.aux.xml"),
- repo = "AdamWilsonLab/emma_envdata",
- tag = "processed_static",
- name = "biome_raster_modis_proj.tif.aux.xml")
-
- # cleanup
-
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
-
- # End functions
-
- message("Finished rasterizing vegmap")
- return(as.character(Sys.Date()))
-
-
-}
-
-
-
-
diff --git a/R/process_release_dynamic_data_to_parquet.R b/R/process_release_dynamic_data_to_parquet.R
deleted file mode 100644
index a9b67bb9..00000000
--- a/R/process_release_dynamic_data_to_parquet.R
+++ /dev/null
@@ -1,189 +0,0 @@
-library(piggyback)
-library(arrow)
-library(tidyverse)
-
-
-#############################################################
-#' @author Brian Maitner
-#' @description this function takes in tif file from the input, converts them to tidy format, and saves as .gz.parquet
-#' @param input_dir Directory containing input files.
-#' @param output_dir Directory to stick output files in
-#' @param variable_name This is included in the tidy file output
-#' @param ... Does nothing. Used for targets.
-#' @note Output dataframes have three columns: CellID, date, variable, value. ALso note that cells with NA values are omitted.
-process_release_dynamic_data_to_parquet <- function(temp_directory = "data/temp/raw_data/ndvi_modis/",
- input_tag = "raw_ndvi_modis",
- output_tag = "current",
- variable_name = "ndvi",
- sleep_time = 30,
- ...){
-
-
- # #Ensure directory is empty if it exists
-
- if(dir.exists(temp_directory)){
- unlink(file.path(temp_directory), recursive = TRUE, force = TRUE)
- }
-
- # make a directory if one doesn't exist yet
-
- if(!dir.exists(temp_directory)){
- dir.create(temp_directory, recursive = TRUE)
- }
-
- #get release assets
- release_assetts <- pb_list(repo = "AdamWilsonLab/emma_envdata")
-
- # load files
-
- raster_list <-
- release_assetts %>%
- filter(tag == input_tag) %>%
- filter(file_name != "log.csv") %>%
- mutate(parquet_name = lubridate::as_date(x = gsub(pattern = ".tif",replacement = "",x = file_name,)))%>%
- mutate(parquet_name = as.numeric(parquet_name))%>%
- mutate(parquet_name = paste("-dynamic_parquet-",variable_name,"-",parquet_name,".gz.parquet",sep = ""))
-
- #-dynamic_parquet-ndvi-11005.gz.parquet
-
- # get files
-
-
- # figure out which files have been processed
-
- processed_list <-
- release_assetts %>%
- filter(tag == output_tag)%>%
- filter(grepl(pattern = paste("-",variable_name,"-",sep = ""),
- x = file_name))
-
- #Don't worry about files that have been processed already
-
- raster_list <- raster_list[which(!raster_list$parquet_name %in% processed_list$file_name),]
-
- #end if things are already done
-
- if(nrow(raster_list) == 0){
-
- message(paste("Finished converting ",
- variable_name,
- " files to parquet", sep = ""))
-
- return(
- release_assetts %>%
- filter(tag == input_tag) %>%
- dplyr::select(file_name) %>%
- filter(file_name != "") %>%
- filter(grepl(pattern = ".tif$", x = file_name)) %>%
- mutate(date_format = gsub(pattern = ".tif",
- replacement = "",
- x = file_name))%>%
- mutate(date_format = gsub(pattern = "_", replacement = "-", x = date_format)) %>%
- dplyr::pull(date_format) %>%
- max()
- )
-
-
-
- }
-
-
- # process the files that haven't been done yet
-
-
- for(i in 1:nrow(raster_list)){
-
- #Download the ith raster
- robust_pb_download(file = raster_list$file_name[i],
- dest = temp_directory,
- repo = "AdamWilsonLab/emma_envdata",
- tag = raster_list$tag[i],
- overwrite = TRUE,
- max_attempts = 10,
- sleep_time = sleep_time)
-
- #Pause to keep Github happy
-
- Sys.sleep(sleep_time)
-
- # Get the date in integer format (will append to the data)
-
- raster_list$file_name[i] |>
- gsub(pattern = "/", replacement = "")|>
- gsub(pattern = ".tif", replacement = "")|>
- lubridate::as_date()|>
- as.numeric() -> integer_date_i
-
- # Process ith file
-
- file.path(temp_directory,raster_list$file_name[i]) |>
- stars::read_stars() |>
- as.data.frame() %>%
- mutate(cellID = row_number(),
- date = integer_date_i,
- variable = variable_name) %>%
- rename( value := 3) %>%
- dplyr::select(cellID, date, variable, value ) %>%
- drop_na() %>%
- write_parquet(sink = file.path(temp_directory,
- paste("-dynamic_parquet-",
- variable_name,"-",
- integer_date_i,
- ".gz.parquet", sep = "")),
- compression = "gzip")
-
- # Upload ith file
-
- pb_upload(file = file.path(temp_directory,
- paste("-dynamic_parquet-",
- variable_name,"-",
- integer_date_i,
- ".gz.parquet", sep = "")),
- repo = "AdamWilsonLab/emma_envdata",
- tag = output_tag,
- overwrite = TRUE
- )
-
- #clean up
-
- unlink(file.path(temp_directory,raster_list$file_name[i]))
-
- unlink( file.path(temp_directory,
- paste("-dynamic_parquet-",
- variable_name,"-",
- integer_date_i,
- ".gz.parquet", sep = "")) )
-
- rm(integer_date_i)
-
- #Pause to keep Github happy
-
- Sys.sleep(sleep_time)
-
- } #end i loop
-
- #Clean up
-
- unlink(file.path(temp_directory),
- recursive = TRUE,
- force = TRUE)
-
- #End fx
-
- message(paste("Finished converting ",variable_name, " files to parquet",sep = ""))
- return(
- raster_list %>%
- filter(tag == input_tag) %>%
- dplyr::select(file_name) %>%
- filter(file_name != "") %>%
- filter(grepl(pattern = ".tif$", x = file_name)) %>%
- mutate(date_format = gsub(pattern = ".tif",
- replacement = "",
- x = file_name))%>%
- mutate(date_format = gsub(pattern = "_", replacement = "-", x = date_format)) %>%
- dplyr::pull(date_format) %>%
- max()
- )
-
-
-}#end fx
diff --git a/R/process_release_protected_area_distance.R b/R/process_release_protected_area_distance.R
index a7ab1b6b..62555d4c 100644
--- a/R/process_release_protected_area_distance.R
+++ b/R/process_release_protected_area_distance.R
@@ -9,7 +9,7 @@ library(terra)
#' @author Adam Wilson & Brian Maitner
#' @description This code produces a raster containing distances to protected areas
#' @param template_release path to raster file to use as a template for reprojection
-process_release_protected_area_distance <- function(template_release,
+data_protected_areas <- function(domain_raster,
out_file="protected_area_distance.tif",
temp_directory = "data/temp/protected_area",
out_tag = "processed_static"
diff --git a/R/process_stable_data.R b/R/process_stable_data.R
new file mode 100644
index 00000000..ba742f6f
--- /dev/null
+++ b/R/process_stable_data.R
@@ -0,0 +1,65 @@
+library(arrow)
+
+#' @param output_dir directory (no file name) in which to save the csv that is returned
+#' @param precip_dir directory containing the precipitation layers
+#' @param landcover_dir directory containing the landcover layers
+#' @param elevation_dir directory containing the elevation layer
+#' @param cloud_dir directory containing the could layers
+#' @param climate_dir directory containing the climate layers
+#' @param alos_dir directory containing the alos layers
+#' @param ... Does nothing, used to ensure upstream changes impact things
+process_stable_data <- function(output_dir = "data/processed_data/model_data/",
+ precip_dir = "data/processed_data/precipitation_chelsa/",
+ landcover_dir = "data/processed_data/landcover_za/",
+ elevation_dir = "data/processed_data/elevation_nasadem/",
+ cloud_dir = "data/processed_data/clouds_wilson/",
+ climate_dir = "data/processed_data/climate_chelsa/",
+ alos_dir = "data/processed_data/alos/",
+ ...) {
+
+
+ # make a directory if one doesn't exist yet
+
+ if(!dir.exists(output_dir)){
+ dir.create(output_dir, recursive = TRUE)
+ }
+
+ # process data
+
+ c(precip_dir,
+ landcover_dir,
+ elevation_dir,
+ cloud_dir,
+ climate_dir,
+ alos_dir) |>
+
+ lapply(FUN = function(x){
+ list.files(path = x,
+ pattern = ".tif$",
+ full.names = T,
+ recursive = T)}) |>
+ unlist() |>
+ stars::read_stars() |>
+ as.data.frame() |>
+ mutate(cellID = row_number()) %>%
+ mutate(count_na = apply(., 1,FUN = function(x){sum(is.na(x))} )) %>%
+ filter(count_na < 20) %>%
+ write_parquet(sink = paste(output_dir,"stable_data.gz.parquet",sep = ""),
+ compression = "gzip")
+
+ #The following line of code can be used to break things down by a grouping variable
+ # write_dataset(path = output_dir,
+ # format = "parquet",
+ # basename_template = "stable_data{i}.parquet.gz",
+ # compression = "gzip",
+ # existing_data_behavior = "delete_matching")
+
+ #cleanup
+ gc()
+
+ # Return filename
+
+ message("Finished processing stable model data")
+ return(paste(output_dir,"stable_data.gz.parquet",sep = ""))
+
+}
diff --git a/R/release_data.R b/R/release_data.R
deleted file mode 100644
index 11ea589f..00000000
--- a/R/release_data.R
+++ /dev/null
@@ -1,95 +0,0 @@
-library(piggyback)
-
-#' @author Brian Maitner
-#' @param data_directory Directory storing the data you want to serve via Github releases
-#' @param tag tag for the release
-#' @param ... Does nothing but helps with targets connections
-#' @note Releases doesn't handle directories, so any directory structure is converted to part of the file name
-release_data <- function(data_directory = "data/processed_data/model_data/", tag = "current", ...){
-
-
-
- #Make sure there is a release by attempting to create one. If it already exists, this will fail
- tryCatch(expr = pb_new_release(repo = "AdamWilsonLab/emma_envdata",
- tag = tag),
- error = function(e){message("Previous release found")})
-
- #Get a list of files already released
- released_files <- pb_list(repo = "AdamWilsonLab/emma_envdata",
- tag = tag)
-
-
- # Get a lost of the local files
- local_files <- data.frame(local_filename = list.files(path = data_directory,
- recursive = TRUE,
- full.names = TRUE))
-
- # Convert local filenames to be releases compatible
- local_files$file_name <-
- sapply(X = local_files$local_filename,
- FUN = function(x){
-
- name_i <- gsub(pattern = data_directory,
- replacement = "",
- x = x)
-
- name_i <- gsub(pattern = "/",
- replacement = "-",
- x = name_i)
- return(name_i)
-
- })
-
- # Get timestamps on local files
- local_files$last_modified <-
- Reduce(c, lapply(X = local_files$local_filename,
- FUN = function(x) {
- file.info(x)$mtime})
- )
-
-
- # Figure out which files DON'T need to be released
- merged_info <- merge(x = released_files,
- y = local_files,
- all = TRUE)
-
- merged_info$diff_hrs <- difftime(time2 = merged_info$timestamp,
- time1 = merged_info$last_modified,
- units = "hours")
-
-
- # We only want time differences of greater than zero (meaning that the local file is more recent) or NA
- merged_info <- merged_info[which(!merged_info$diff_hrs < 0 | is.na(merged_info$diff_hrs)),]
-
- #Quit if there are no new/updated files to release
- if(nrow(merged_info) == 0){
-
- message("Releases are already up to date.")
- return(invisible(NULL))
-
-
- }
-
-
- # loop through and release everything
- for( i in 1:nrow(merged_info)){
-
- Sys.sleep(0.1) #We need to limit our rate in order to keep Github happy
-
-
- pb_upload(file = merged_info$local_filename[i],
- repo = "AdamWilsonLab/emma_envdata",
- tag = tag,
- name = merged_info$file_name[i])
-
-
-
-
- } # end i loop
-
- # End
- message("Finished releasing data")
- return(invisible(NULL))
-
-
-}#end function
diff --git a/R/stac_functions.R b/R/stac_functions.R
new file mode 100644
index 00000000..165c3af4
--- /dev/null
+++ b/R/stac_functions.R
@@ -0,0 +1,298 @@
+#' @title Generate STAC Collection for MODIS VI dataset
+#' @description Creates a STAC Collection and individual Item files for monthly MODIS VI parquet data.
+#' Items are configured to point to GitHub release URLs.
+#' This is a dataset-specific collection that will be linked from a parent STAC Catalog.
+#' @author EMMA Team
+#' @param parquet_files Character vector of processed parquet file paths (from targets branching; used to establish dependency)
+#' @param parquet_dir Directory containing monthly MODIS VI parquet files
+#' @param stac_dir Output directory for this collection's STAC JSON files
+#' @param parent_catalog_path Path to parent catalog (for generating relative links)
+#' @param gh_repo GitHub repository in format "owner/repo"
+#' @param gh_release_tag GitHub release tag where files will be hosted
+#' @param verbose Logical for progress messages
+#' @return Character path to collection.json for this dataset
+#' @keywords internal
+generate_modis_vi_stac <- function(
+ parquet_files = NULL, # Dependency on branched target, may be unused
+ parquet_dir = "data/processed_data/dynamic_parquet/modis_vi",
+ stac_dir = "data/stac/modis_vi",
+ parent_catalog_path = "data/stac",
+ gh_repo = "AdamWilsonLab/emma_envdata",
+ gh_release_tag = "data_modis_vi_current",
+ verbose = TRUE
+) {
+
+ # Create output directory
+ dir.create(stac_dir, recursive = TRUE, showWarnings = FALSE)
+
+ # Find all monthly parquet files
+ parquet_files <- list.files(
+ parquet_dir,
+ pattern = "^dynamic_modis_vi_\\d{6}\\.gz\\.parquet$",
+ full.names = FALSE
+ )
+
+ if (length(parquet_files) == 0) {
+ if (verbose) warning("No MODIS VI parquet files found in ", parquet_dir)
+ return(NA_character_)
+ }
+
+ # Extract year-month from filenames
+ dates <- as.Date(paste0(gsub(".*_(\\d{6})\\..*", "\\1", parquet_files), "01"), "%Y%m%d")
+
+ if (verbose) message("Generating STAC Collection for MODIS VI with ", length(parquet_files), " monthly files")
+
+ # Create STAC Collection (part of parent catalog)
+ collection <- list(
+ stac_version = "1.0.0",
+ stac_extensions = c(
+ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
+ ),
+ type = "Collection",
+ id = "modis_vi",
+ description = "MODIS Enhanced Vegetation Index (EVI) observations from Terra and Aqua satellites. 500m resolution, 16-day composites. Data processed from AppEEARS.",
+ license = "CC-BY-4.0",
+ keywords = c("MODIS", "EVI", "vegetation", "Terra", "Aqua", "500m", "16-day"),
+ extent = list(
+ spatial = list(
+ bbox = list(c(-180, -90, 180, 90))
+ ),
+ temporal = list(
+ interval = list(c(
+ paste0(format(min(dates), "%Y-%m-%d"), "T00:00:00Z"),
+ paste0(format(max(dates), "%Y-%m-%d"), "T23:59:59Z")
+ ))
+ )
+ ),
+ links = list(
+ list(
+ rel = "root",
+ href = "../catalog.json",
+ type = "application/json"
+ ),
+ list(
+ rel = "parent",
+ href = "../catalog.json",
+ type = "application/json"
+ ),
+ list(
+ rel = "license",
+ href = "https://creativecommons.org/licenses/by/4.0/",
+ type = "text/html"
+ ),
+ list(
+ rel = "about",
+ href = "https://lpdaac.usgs.gov/products/mod13a1v061/",
+ title = "MOD13A1.061 Product Information",
+ type = "text/html"
+ )
+ ),
+ providers = list(
+ list(
+ name = "USGS LP DAAC",
+ description = "Data source for MOD13A1 and MYD13A1",
+ roles = c("producer", "licensor"),
+ url = "https://lpdaac.usgs.gov/"
+ ),
+ list(
+ name = "NASA AppEEARS",
+ description = "Data access and subsetting service",
+ roles = c("processor"),
+ url = "https://appeears.org/"
+ ),
+ list(
+ name = "EMMA Lab",
+ description = "Data processing and aggregation",
+ roles = c("processor"),
+ url = "https://adamwilsonlab.github.io/"
+ )
+ ),
+ summaries = list(
+ sci_doi = "10.5067/MODIS/MOD13A1.061|10.5067/MODIS/MYD13A1.061",
+ platforms = c("Terra", "Aqua"),
+ instruments = c("MODIS"),
+ gsd = list(500),
+ bands = list(
+ list(
+ name = "EVI",
+ description = "Enhanced Vegetation Index",
+ data_type = "int32",
+ scale = 0.01,
+ offset = 0,
+ nodata = -9999
+ )
+ )
+ )
+ )
+
+ # Write collection.json
+ collection_file <- file.path(stac_dir, "collection.json")
+ jsonlite::write_json(collection, collection_file, pretty = TRUE, auto_unbox = TRUE)
+
+ if (verbose) message("Created STAC Collection: ", collection_file)
+
+ # Create individual Item files
+ for (i in seq_along(parquet_files)) {
+ pq_file <- parquet_files[i]
+ pq_date <- dates[i]
+ year_month <- format(pq_date, "%Y%m")
+
+ # GitHub release URL
+ gh_raw_url <- paste0(
+ "https://github.com/", gh_repo, "/releases/download/", gh_release_tag, "/",
+ pq_file
+ )
+
+ item <- list(
+ stac_version = "1.0.0",
+ stac_extensions = c(
+ "https://stac-extensions.github.io/scientific/v1.0.0/schema.json"
+ ),
+ type = "Feature",
+ id = paste0("modis_vi_", year_month),
+ description = paste("MODIS EVI observations for", format(pq_date, "%B %Y")),
+ geometry = list(
+ type = "Polygon",
+ coordinates = list(list(
+ c(-180, -90), c(180, -90), c(180, 90), c(-180, 90), c(-180, -90)
+ ))
+ ),
+ properties = list(
+ `datetime` = paste0(format(pq_date, "%Y-%m-%d"), "T00:00:00Z"),
+ start_datetime = paste0(format(pq_date, "%Y-%m-01"), "T00:00:00Z"),
+ end_datetime = paste0(format(as.Date(paste0(format(pq_date + 31, "%Y-%m"), "-01")) - 1, "%Y-%m-%d"), "T23:59:59Z"),
+ platforms = c("Terra", "Aqua"),
+ instruments = c("MODIS"),
+ gsd = 500,
+ dataset = "modis_vi"
+ ),
+ links = list(
+ list(
+ rel = "collection",
+ href = "collection.json",
+ type = "application/json"
+ ),
+ list(
+ rel = "root",
+ href = "../catalog.json",
+ type = "application/json"
+ ),
+ list(
+ rel = "parent",
+ href = "collection.json",
+ type = "application/json"
+ )
+ ),
+ assets = list(
+ data = list(
+ href = gh_raw_url,
+ title = paste0("MODIS VI Parquet - ", year_month),
+ description = "Enhanced Vegetation Index observations in parquet format",
+ type = "application/octet-stream",
+ roles = c("data")
+ )
+ )
+ )
+
+ item_file <- file.path(stac_dir, paste0("modis_vi_", year_month, ".json"))
+ jsonlite::write_json(item, item_file, pretty = TRUE, auto_unbox = TRUE)
+
+ # Add item link to collection
+ collection$links[[length(collection$links) + 1]] <- list(
+ rel = "item",
+ href = paste0("modis_vi_", year_month, ".json"),
+ type = "application/json",
+ title = paste("MODIS VI", year_month)
+ )
+ }
+
+ # Update collection.json with all item links
+ jsonlite::write_json(collection, collection_file, pretty = TRUE, auto_unbox = TRUE)
+
+ if (verbose) message("Generated ", length(parquet_files), " STAC Item files")
+
+ collection_file
+}
+
+
+#' @title Generate parent STAC Catalog for EMMA environmental datasets
+#' @description Creates a parent STAC Catalog that organizes and links all EMMA datasets
+#' (MODIS VI, VIIRS VI, burned area, age, etc.).
+#' @author EMMA Team
+#' @param stac_base_dir Base directory for STAC output (datasets will be in subdirectories)
+#' @param dataset_collections List of dataset collection paths (e.g., list(modis_vi = "data/stac/modis_vi"))
+#' @param gh_repo GitHub repository in format "owner/repo"
+#' @param verbose Logical for progress messages
+#' @return Character path to parent catalog.json
+#' @keywords internal
+generate_emma_stac_catalog <- function(
+ stac_base_dir = "data/stac",
+ dataset_collections = list(
+ modis_vi = "data/stac/modis_vi"
+ ),
+ gh_repo = "AdamWilsonLab/emma_envdata",
+ verbose = TRUE
+) {
+
+ dir.create(stac_base_dir, recursive = TRUE, showWarnings = FALSE)
+
+ # Create parent STAC Catalog
+ catalog <- list(
+ stac_version = "1.0.0",
+ type = "Catalog",
+ id = "emma",
+ description = "EMMA Environmental Data Catalog - A curated collection of environmental datasets for the Eastern Mediterranean and Maghreb region.",
+ links = list(
+ list(
+ rel = "root",
+ href = "catalog.json",
+ type = "application/json",
+ title = "EMMA Catalog"
+ ),
+ list(
+ rel = "license",
+ href = "https://creativecommons.org/licenses/by/4.0/",
+ type = "text/html",
+ title = "Creative Commons Attribution 4.0"
+ ),
+ list(
+ rel = "about",
+ href = paste0("https://github.com/", gh_repo),
+ type = "text/html",
+ title = "EMMA Project Repository"
+ )
+ )
+ )
+
+ if (verbose) message("Creating parent STAC Catalog with ", length(dataset_collections), " dataset(s)")
+
+ # Add links to each dataset collection
+ for (dataset_name in names(dataset_collections)) {
+ collection_path <- dataset_collections[[dataset_name]]
+ collection_file <- file.path(collection_path, "collection.json")
+
+ if (file.exists(collection_file)) {
+ # Relative path from catalog to collection
+ rel_path <- paste0(dataset_name, "/collection.json")
+
+ catalog$links[[length(catalog$links) + 1]] <- list(
+ rel = "child",
+ href = rel_path,
+ type = "application/json",
+ title = paste0(dataset_name, " - Dynamic VI observations")
+ )
+
+ if (verbose) message(" Linked collection: ", dataset_name)
+ } else {
+ if (verbose) warning("Collection not found: ", collection_file)
+ }
+ }
+
+ # Write parent catalog
+ catalog_file <- file.path(stac_base_dir, "catalog.json")
+ jsonlite::write_json(catalog, catalog_file, pretty = TRUE, auto_unbox = TRUE)
+
+ if (verbose) message("Created parent STAC Catalog: ", catalog_file)
+
+ catalog_file
+}
diff --git a/R/tar_release_storage.R b/R/tar_release_storage.R
new file mode 100644
index 00000000..903158b1
--- /dev/null
+++ b/R/tar_release_storage.R
@@ -0,0 +1,392 @@
+#' Download targets from GitHub Release
+#' @description Download locally stored targets from GitHub releases (useful for GitHub Actions)
+#' @param repo Repository in "owner/repo" format (default from environment or "AdamWilsonLab/emma_envdata")
+#' @param tag Release tag to store objects (default from environment or "objects_current")
+#' @param cache_dir Cache directory (default: "data/target_outputs/.tar_cache")
+#' @param which_targets Optional vector of specific target names to download
+#' @param verbose Logical for progress messages
+#' @details Call this at the start of tar_make() in update mode to download targets
+#' @export
+tar_download_github_release <- function(
+ repo = NULL,
+ tag = NULL,
+ cache_dir = "data/target_outputs/.tar_cache",
+ which_targets = NULL,
+ verbose = TRUE
+) {
+ # Use environment variables as fallback, but allow explicit parameters
+ repo <- repo %||% Sys.getenv("TAR_GH_RELEASE_REPO") %||% "AdamWilsonLab/emma_envdata"
+ tag <- tag %||% Sys.getenv("TAR_GH_RELEASE_TAG") %||% "objects_current"
+ cache_dir <- cache_dir %||% Sys.getenv("TAR_GH_RELEASE_CACHE_DIR") %||% "data/target_outputs/.tar_cache"
+ objects_dir <- "_targets/objects"
+
+ if (!nzchar(repo) || !nzchar(tag)) {
+ stop("GitHub release configuration not set. Provide repo and tag parameters or set environment variables.")
+ }
+
+ dir.create(cache_dir, recursive = TRUE, showWarnings = FALSE)
+ dir.create(objects_dir, recursive = TRUE, showWarnings = FALSE)
+
+ # Get list of assets on GitHub release
+ tryCatch({
+ assets <- piggyback::pb_list(repo = repo, tag = tag)
+ if (verbose) message("[tar_github_release] Found ", nrow(assets), " assets on GitHub release")
+ }, error = function(e) {
+ stop("[tar_github_release] Could not access GitHub release: ", conditionMessage(e))
+ })
+
+ # Filter assets if specific targets requested
+ if (!is.null(which_targets)) {
+ assets <- assets[assets$file_name %in% which_targets | startsWith(assets$file_name, which_targets), ]
+ }
+
+ if (nrow(assets) == 0) {
+ if (verbose) message("[tar_github_release] No assets to download")
+ return(invisible(NULL))
+ }
+
+ # Download each asset
+ for (i in seq_len(nrow(assets))) {
+ asset_name <- assets$file_name[i]
+
+ # Check if this is a file-format target (has extension)
+ # File-format targets are stored as "target_name.extension" (e.g., "country.parquet")
+ # Regular objects are stored as "target_name" (e.g., "elevation_task_id")
+ is_file_format <- grepl("\\.[^.]+$", asset_name)
+
+ if (is_file_format) {
+ # Extract target name by removing extension
+ target_name <- sub("\\.[^.]+$", "", asset_name)
+ file_ext <- sub(".*\\.", "", asset_name)
+ } else {
+ target_name <- asset_name
+ file_ext <- NULL
+ }
+
+ local_path <- file.path(objects_dir, target_name)
+ cached_path <- file.path(cache_dir, asset_name)
+
+ # Download to cache if not already there
+ if (!file.exists(cached_path)) {
+ if (verbose) message("[tar_github_release] Downloading: ", asset_name)
+ max_attempts <- 3
+ for (attempt in 1:max_attempts) {
+ tryCatch({
+ piggyback::pb_download(
+ file = asset_name,
+ repo = repo,
+ tag = tag,
+ dest = cache_dir,
+ overwrite = TRUE
+ )
+ if (verbose) message("[tar_github_release] Downloaded: ", asset_name)
+ break
+ }, error = function(e) {
+ if (attempt < max_attempts) {
+ if (verbose) message("[tar_github_release] Download attempt ", attempt, " failed: ", conditionMessage(e))
+ Sys.sleep(2)
+ } else {
+ warning("[tar_github_release] Failed to download after ", max_attempts, " attempts: ", conditionMessage(e))
+ }
+ })
+ }
+ } else {
+ if (verbose) message("[tar_github_release] Already cached: ", asset_name)
+ }
+
+ # Copy from cache to appropriate target location
+ if (file.exists(cached_path)) {
+ if (is_file_format) {
+ # For file-format targets:
+ # 1. Copy actual file to _targets/workspaces/ (where targets expects it)
+ # 2. Copy to data/target_outputs/ (for user access)
+ # 3. Create RDS wrapper in _targets/objects/ pointing to workspaces location
+
+ ws_dir <- "_targets/workspaces"
+ dir.create(ws_dir, recursive = TRUE, showWarnings = FALSE)
+ ws_path <- file.path(ws_dir, asset_name)
+ file.copy(cached_path, ws_path, overwrite = TRUE)
+
+ # Also copy to data/target_outputs/ for user access
+ out_dir <- "data/target_outputs"
+ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
+ out_path <- file.path(out_dir, asset_name)
+ file.copy(cached_path, out_path, overwrite = TRUE)
+
+ # Create RDS wrapper in _targets/objects/ that points to the workspaces file path
+ obj_dir <- "_targets/objects"
+ dir.create(obj_dir, recursive = TRUE, showWarnings = FALSE)
+ obj_path <- file.path(obj_dir, target_name)
+ saveRDS(ws_path, obj_path)
+ if (verbose) message("[tar_github_release] Restored file-format target: ", target_name)
+ } else {
+ # Regular object file: copy to _targets/objects/
+ obj_dir <- "_targets/objects"
+ dir.create(obj_dir, recursive = TRUE, showWarnings = FALSE)
+ obj_path <- file.path(obj_dir, target_name)
+ file.copy(cached_path, obj_path, overwrite = TRUE)
+ if (verbose) message("[tar_github_release] Restored: ", target_name)
+ }
+ }
+ }
+
+ if (verbose) message("[tar_github_release] Download complete")
+ invisible(NULL)
+}
+
+#' Upload targets to GitHub Release after tar_make() completes
+#' @description Upload locally stored targets to GitHub releases
+#' @param repo Repository in "owner/repo" format (default from environment or "AdamWilsonLab/emma_envdata")
+#' @param tag Release tag to store objects (default from environment or "objects_current")
+#' @param format Serialization format: "qs", "rds", or "parquet" (default: "qs")
+#' @param cache_dir Cache directory (default: "data/target_outputs/.tar_cache")
+#' @param which_targets Optional vector of specific target names to upload
+#' @param verbose Logical for progress messages
+#' @details Call this after tar_make() to upload all targets
+#' @export
+tar_upload_github_release <- function(
+ repo = NULL,
+ tag = NULL,
+ format = "qs",
+ cache_dir = "data/target_outputs/.tar_cache",
+ which_targets = NULL,
+ verbose = TRUE
+) {
+ # Use environment variables as fallback, but allow explicit parameters
+ repo <- repo %||% Sys.getenv("TAR_GH_RELEASE_REPO") %||% "AdamWilsonLab/emma_envdata"
+ tag <- tag %||% Sys.getenv("TAR_GH_RELEASE_TAG") %||% "objects_current"
+ cache_dir <- cache_dir %||% Sys.getenv("TAR_GH_RELEASE_CACHE_DIR") %||% "data/target_outputs/.tar_cache"
+
+ if (!nzchar(repo) || !nzchar(tag)) {
+ stop("GitHub release configuration not set. Provide repo and tag parameters or set environment variables.")
+ }
+
+ # Ensure release exists
+ tryCatch({
+ piggyback::pb_list(repo = repo, tag = tag)
+ }, error = function(e) {
+ if (verbose) message("[tar_github_release] Creating release: ", tag)
+ piggyback::pb_new_release(repo = repo, tag = tag)
+ })
+
+ # Get metadata to find file-format target paths
+ meta_df <- tryCatch({
+ tar_meta()
+ }, error = function(e) {
+ if (verbose) message("[tar_github_release] Could not read targets metadata")
+ data.frame(name = character(0), format = character(0), path = list())
+ })
+
+ # Get current assets on GitHub to check what exists
+ remote_assets <- tryCatch({
+ piggyback::pb_list(repo = repo, tag = tag)
+ }, error = function(e) {
+ data.frame(file_name = character(0))
+ })
+
+ # Get list of local target files
+ if (is.null(which_targets)) {
+ # Get all targets from _targets/objects/ (regular objects)
+ regular_files <- list.files("_targets/objects", full.names = TRUE, recursive = FALSE)
+
+ # Also get file-format targets from _targets/workspaces/
+ file_format_targets <- character(0)
+ if (dir.exists("_targets/workspaces")) {
+ ws_files <- list.files("_targets/workspaces", full.names = FALSE, recursive = FALSE)
+ # Filter to only include those that are file-format targets (have metadata)
+ for (ws_file in ws_files) {
+ target_meta <- meta_df[meta_df$name == ws_file, ]
+ if (nrow(target_meta) > 0 && target_meta$format[1] == "file") {
+ file_format_targets <- c(file_format_targets, file.path("_targets/workspaces", ws_file))
+ }
+ }
+ }
+
+ # Also get actual data files from data/target_outputs/
+ data_output_files <- character(0)
+ if (dir.exists("data/target_outputs")) {
+ all_output_files <- list.files("data/target_outputs", full.names = TRUE, recursive = FALSE)
+ # Exclude cache directory
+ data_output_files <- all_output_files[!grepl("\\.tar_cache", all_output_files)]
+ }
+
+ local_files <- c(regular_files, file_format_targets, data_output_files)
+ if (verbose) message("[tar_github_release] Found ", length(local_files), " local target files to upload")
+ } else {
+ # Find specific targets - check all locations
+ regular_files <- character(0)
+ file_format_targets <- character(0)
+ data_output_files <- character(0)
+
+ for (target in which_targets) {
+ obj_file <- file.path("_targets/objects", target)
+ ws_file <- file.path("_targets/workspaces", target)
+ data_file <- file.path("data/target_outputs", target)
+
+ if (file.exists(obj_file)) {
+ regular_files <- c(regular_files, obj_file)
+ } else if (file.exists(ws_file)) {
+ target_meta <- meta_df[meta_df$name == target, ]
+ if (nrow(target_meta) > 0 && target_meta$format[1] == "file") {
+ file_format_targets <- c(file_format_targets, ws_file)
+ }
+ } else if (file.exists(data_file)) {
+ data_output_files <- c(data_output_files, data_file)
+ }
+ }
+ local_files <- c(regular_files, file_format_targets, data_output_files)
+ }
+
+ if (length(local_files) == 0) {
+ message("[tar_github_release] No targets to upload")
+ return(invisible(NULL))
+ }
+
+ # Upload each file
+ for (local_file in local_files) {
+ target_name <- basename(local_file)
+
+ # Determine if this is a file-format target based on location
+ # Files in _targets/workspaces/ are file-format targets
+ is_file_target <- grepl("_targets/workspaces", local_file)
+
+ if (is_file_target) {
+ # This is a file-format target in _targets/workspaces/
+ # Use the file from workspaces as the source to upload
+ # Get extension from metadata or from actual file
+ target_meta <- meta_df[meta_df$name == target_name, ]
+
+ # Try to get extension from metadata path
+ ext <- ""
+ if (nrow(target_meta) > 0) {
+ metadata_path <- target_meta$path[[1]]
+ # Handle case where path is a vector with multiple values
+ if (is.character(metadata_path) && length(metadata_path) > 0) {
+ # Take the first one and get the extension from it
+ ext <- tools::file_ext(metadata_path[1])
+ }
+ }
+
+ # If we couldn't get extension from metadata, skip this file
+ if (nchar(ext) == 0) {
+ if (verbose) message("[tar_github_release] Skipping file-format target (no extension found): ", target_name)
+ next
+ }
+
+ # Create upload name - avoid double extensions
+ # If target name already ends with this extension, don't add it again
+ if (grepl(paste0("\\.", ext, "$"), target_name)) {
+ upload_name <- target_name
+ } else {
+ upload_name <- paste0(target_name, ".", ext)
+ }
+
+ if (verbose) message("[tar_github_release] Uploading file-format target: ", upload_name, " from ", local_file)
+
+ # Check if already exists on GitHub
+ exists_on_github <- any(remote_assets$file_name == upload_name)
+ if (exists_on_github) {
+ if (verbose) message("[tar_github_release] File already on GitHub, deleting old version: ", upload_name)
+ tryCatch({
+ # Delete old version
+ old_asset <- remote_assets[remote_assets$file_name == upload_name, ]
+ if (nrow(old_asset) > 0) {
+ piggyback::pb_delete(repo = repo, tag = tag, file = upload_name)
+ Sys.sleep(1)
+ }
+ }, error = function(e) {
+ if (verbose) message("[tar_github_release] Could not delete old asset: ", conditionMessage(e))
+ })
+ }
+
+ max_attempts <- 3
+ for (attempt in 1:max_attempts) {
+ tryCatch({
+ piggyback::pb_upload(
+ file = local_file,
+ repo = repo,
+ tag = tag,
+ name = upload_name,
+ overwrite = FALSE
+ )
+ if (verbose) message("[tar_github_release] Uploaded: ", upload_name)
+ Sys.sleep(1)
+ break
+ }, error = function(e) {
+ if (attempt < max_attempts) {
+ if (verbose) message("[tar_github_release] Upload attempt ", attempt, " failed: ", conditionMessage(e))
+ Sys.sleep(2)
+ } else {
+ warning("[tar_github_release] Failed to upload after ", max_attempts, " attempts: ", conditionMessage(e))
+ }
+ })
+ }
+ } else {
+ # Regular objects or data output files
+ # Determine if it's from data/target_outputs or _targets/objects
+ is_data_output <- grepl("data/target_outputs", local_file)
+
+ if (is_data_output) {
+ # Data output file - upload with its original name
+ upload_name <- basename(local_file)
+ if (verbose) message("[tar_github_release] Uploading data file: ", upload_name)
+ } else {
+ # Serialized object from _targets/objects
+ upload_name <- target_name
+ if (verbose) message("[tar_github_release] Uploading object: ", target_name)
+ }
+
+ # Check if already exists on GitHub
+ exists_on_github <- any(remote_assets$file_name == upload_name)
+ if (exists_on_github) {
+ if (verbose) message("[tar_github_release] File already on GitHub, deleting old version: ", upload_name)
+ tryCatch({
+ # Delete old version
+ piggyback::pb_delete(repo = repo, tag = tag, file = upload_name)
+ Sys.sleep(1)
+ }, error = function(e) {
+ if (verbose) message("[tar_github_release] Could not delete old asset: ", conditionMessage(e))
+ })
+ }
+
+ max_attempts <- 3
+ for (attempt in 1:max_attempts) {
+ tryCatch({
+ piggyback::pb_upload(
+ file = local_file,
+ repo = repo,
+ tag = tag,
+ name = upload_name,
+ overwrite = FALSE
+ )
+ if (verbose) message("[tar_github_release] Uploaded: ", upload_name)
+ Sys.sleep(1)
+ break
+ }, error = function(e) {
+ if (attempt < max_attempts) {
+ if (verbose) message("[tar_github_release] Upload attempt ", attempt, " failed: ", conditionMessage(e))
+ Sys.sleep(2)
+ } else {
+ warning("[tar_github_release] Failed to upload: ", conditionMessage(e))
+ }
+ })
+ }
+ }
+ }
+
+ if (verbose) message("[tar_github_release] Upload complete")
+ invisible(NULL)
+}
+
+#' Create GitHub Releases repository for targets
+#' @description DEPRECATED - Use tar_upload_github_release() instead
+#' @export
+tar_github_release_repo <- function(
+ repo,
+ tag,
+ format = "qs",
+ cache_dir = "data/.tar_cache"
+) {
+ stop("tar_github_release_repo() is deprecated. Use tar_upload_github_release() after tar_make() instead.")
+}
+
diff --git a/R/upload_releases.R b/R/upload_releases.R
new file mode 100644
index 00000000..fdef2283
--- /dev/null
+++ b/R/upload_releases.R
@@ -0,0 +1,160 @@
+#' @title Upload files to GitHub release
+#' @description Creates/updates a GitHub release with specified files.
+#' Skips files that already exist in the release.
+#' @param files Character vector of file paths to upload
+#' @param repo Repository in format "owner/repo"
+#' @param release_tag Release tag (e.g., "static_current", "data_modis_vi_current")
+#' @param release_name Human-readable release name (e.g., "Static Data - Current")
+#' @param verbose Logical for messages
+#' @return Character vector of uploaded file paths (invisibly)
+#' @details
+#' Requires piggyback package and GITHUB_TOKEN environment variable.
+#' Uses piggyback::pb_upload() which requires gh CLI or authentication.
+#' @export
+upload_to_github_release <- function(
+ files,
+ repo,
+ release_tag,
+ release_name = release_tag,
+ verbose = TRUE,
+ ... #include to allow for targets dependencies without affecting function behavior
+) {
+
+
+ # Handle NA or empty inputs
+ if (length(files) == 0 || all(is.na(files))) {
+ if (verbose) message("No files to upload")
+ return(invisible(character(0)))
+ }
+
+ # Remove NA values
+ files <- files[!is.na(files)]
+
+
+ if (length(files) == 0) {
+ if (verbose) message("No files to upload (all were NA)")
+ return(invisible(character(0)))
+ }
+
+ # Check for GITHUB_TOKEN
+ token <- Sys.getenv("GITHUB_TOKEN")
+ if (token == "") {
+ message("GITHUB_TOKEN environment variable not set. Upload may fail.")
+ }
+
+
+ # Filter to files that exist
+ files <- files[file.exists(files)]
+ if (length(files) == 0) {
+ message("No files found to upload")
+ return(invisible(character(0)))
+ }
+
+
+ # Ensure release exists (create it if needed)
+ tryCatch({
+ piggyback::pb_new_release(
+ repo = repo,
+ tag = release_tag,
+ .token = token
+ )
+ if (verbose) message("β Created release '", release_tag, "'")
+ }, error = function(e) {
+ # Release likely already exists, which is fine
+ if (verbose) message("Using existing release '", release_tag, "'")
+ })
+
+
+
+
+ # List existing files in release
+ # pb_list returns NULL if empty (not an error) or a dataframe if files exist
+ existing <- tryCatch({
+ piggyback::pb_list(
+ repo = repo,
+ tag = release_tag,
+ .token = token
+ )
+ }, error = function(e) {
+ if (verbose) warning("Unable to list files in release: ", e$message)
+ data.frame()
+ })
+ existing_names <- ifelse(nrow(existing) > 0, existing$file_name, character(0))
+
+ # Filter to files that don't already exist
+ files_to_upload <- files[basename(files) %not_in% existing_names]
+
+ if (length(files_to_upload) == 0) {
+ if (verbose) message("All files already in release '", release_tag, "'")
+ return(invisible(character(0)))
+ }
+
+ if (verbose) {
+ message("Uploading ", length(files_to_upload), " files to release '", release_tag, "'")
+ }
+
+ # Upload each file
+ uploaded <- character(0)
+ for (file in files_to_upload) {
+ if (!file.exists(file)) {
+ warning("File not found: ", file)
+ next
+ }
+
+ if (verbose) message(" Uploading: ", basename(file))
+
+ tryCatch({
+ piggyback::pb_upload(
+ file = file,
+ repo = repo,
+ tag = release_tag,
+ .token = token,
+ show_progress = verbose
+ )
+
+ uploaded <- c(uploaded, file)
+ if (verbose) message(" β Uploaded")
+ }, error = function(e) {
+ warning("Failed to upload: ", basename(file), " - ", e$message)
+ })
+ }
+
+
+ # Verify uploaded files are now in the release
+ if (length(uploaded) > 0) {
+ if (verbose) message("Verifying uploaded files in release...")
+
+ Sys.sleep(1) # Brief pause to ensure files are indexed
+
+ tryCatch({
+ release_files <- piggyback::pb_list(
+ repo = repo,
+ tag = release_tag,
+ .token = token
+ )
+
+ uploaded_names <- basename(uploaded)
+ if (!is.null(release_files) && nrow(release_files) > 0) {
+ verified <- uploaded_names %in% release_files$file_name
+
+ if (all(verified)) {
+ if (verbose) message("β Verified: All ", length(uploaded), " files confirmed in release")
+ } else {
+ not_found <- uploaded_names[!verified]
+ stop("Verification failed: ", length(not_found), " file(s) not found in release: ",
+ paste(not_found, collapse = ", "))
+ }
+ } else {
+ stop("Verification failed: Could not list files in release after upload")
+ }
+ }, error = function(e) {
+ stop("Could not verify uploads: ", e$message)
+ })
+ }
+
+ invisible(uploaded)
+}
+
+#' Helper operator
+#' @keywords internal
+`%not_in%` <- Negate(`%in%`)
diff --git a/R/vegetation_map.R b/R/vegetation_map.R
index df37576e..315ee82a 100644
--- a/R/vegetation_map.R
+++ b/R/vegetation_map.R
@@ -8,7 +8,7 @@
#' @biomes list of biomes to keep
get_vegmap <- function(vegmap_shp, biomes = c("Fynbos","Succulent Karoo","Albany Thicket")){
-
+
# Must manually download the following and put in the raw_data folder
# 2018 National Vegetation Map
# http://bgis.sanbi.org/SpatialDataset/Detail/1674
@@ -19,9 +19,8 @@ get_vegmap <- function(vegmap_shp, biomes = c("Fynbos","Succulent Karoo","Alban
vegmap <- vegmap_za %>%
filter(biome_18 %in% biomes ) %>% #filter to list above
- st_make_valid() #some polygons had errors - this fixes them
-
-# st_write(vegmap,dsn = "data/vegmap.gpkg",append=F)
+ st_make_valid() |> #some polygons had errors - this fixes them
+ vect()
return(vegmap)
diff --git a/README.md b/README.md
deleted file mode 100644
index ce63a50b..00000000
--- a/README.md
+++ /dev/null
@@ -1,130 +0,0 @@
-
-
-
-# Ecological Monitoring and Management Application (EMMA)
-
-This is the core repository for environmental data processing in the
-Ecological Monitoring and Management Application [EMMA.io](EMMA.io).
-
-
-# EMMA workflow overview
-
-The EMMA workflow consists of four modules, each with a separate Github repo:
-1) The Environmental Data module (https://github.com/AdamWilsonLab/emma_envdata)
-2) The Modelling and Change Detection module (https://github.com/AdamWilsonLab/emma_model)
-3) The Change Classification module (https://github.com/AdamWilsonLab/emma_change_classification)
-4) The Reporting module (https://github.com/AdamWilsonLab/emma_report)
-
-
-## File structure
-
-The most important files are:
-
-``` r
-βββ _targets.R (data processing workflow and dependency management)
-βββ R/
-βββββ [data_processing_functions]
-βββ data/
-βββββ manual_download (files behind firewalls that must be manually downloaded)
-βββββ raw_data (raw data files downloaded by the workflow)
-βββββ processed_data (data processed and stored by the workflow)
-βββ Readme.Rmd (this file)
-```
-
-Files generated by the workflow are stored in the targets-runs branch.
-The final output of the workflow is a set of parquet files stored as
-Github releases with the tag βcurrentβ.
-
-# Workflow structure
-
-
-
-# Workflow Notes
-
-## Runtime and frequency
-
-Github places some constraints on actions, including memory limits and
-run time limits. To prevent this workflow from taking too long to run
-(and thereby losing all progress), there are a few key parameters that
-can be changed. In the \_targets.R file, the argument βmax_layersβ
-controls the maximum number of layers that rgee will attempt to download
-in one action run. When initially setting up the repo, it may be
-necessary to lower this value and increase the frequency that the
-targets workflow is run (by adjusting the cron parameters in
-targets.yaml). Github also limits the rates of requests, and so the file
-release_data.R includes a call to Sys.sleep that can be adjusted to slow
-down/speed up the process of pushing data to a Github release.
-
-# Data notes
-
- * MODIS NDVI values have been transformed to save space. To restore them to the original values (between -1 and 1), divide by 100 and subtract 1.
- * Untransformed NDVI = (transformed NDVI / 100) - 1
- * Raw MODIS fire dates (tag:raw_fire_modis): values are either 0 (no fire) or the day of the year a fire was observed (1 through 366).
- * Processed MODIS fire dates (tag: processed_fire_dates: values are either 0 (no fire) or the UNIX date (days since 1 Jan. 1970) a fire was observed.
-
-## Data layers
-
-- Continuous Heat-Insolation Load Index (CHILI; ALOS)
-- Multi-Scale Topographic Position Index (MTPI; compares elevation to
- surroundings; ALOS)
-- Topographic Diversity (represents the variety of temperature,
- moisture conditions; ALOS )
-- Mean annual air temperature (CHELSA Bio1)
-- Mean diurnal air temperature range (CHELSA Bio2)
-- Isothermality (ratio of diurnal variation to annual variation in
- temperatures; CHELSA Bio3)
-- Temperature seasonality(std. deviation of the monthly mean
- temperatures; CHELSA Bio4)
-- Mean daily maximum air temperature of the warmest month (CHELSA
- Bio5)
-- Mean daily minimum air temperature of the coldest month (CHELSA
- Bio6)
-- Annual range of air temperature (CHELSA Bio7)
-- Mean daily mean air temperatures of the wettest quarter (CHELSA
- Bio8)
-- Mean daily mean air temperatures of the driest quarter (CHELSA Bio9)
-- Mean daily mean air temperatures of the warmest quarter (CHELSA
- Bio10)
-- Mean daily mean air temperatures of the coldest quarter (CHELSA
- Bio11)
-- Annual precipitation amount (CHELSA Bio12)
-- Precipitation amount of the wettest month (CHELSA Bio13)
-- Precipitation amount of the driest month (CHELSA Bio14)
-- Precipitation seasonality (CV of the monthly precipitation
- estimates; CHELSA Bio15)
-- Mean monthly precipitation amount of the wettest quarter (CHELSA
- Bio16)
-- Mean monthly precipitation amount of the driest quarter (CHELSA
- Bio17)
-- Mean monthly precipitation amount of the warmest quarter (CHELSA
- Bio18)
-- Mean monthly precipitation amount of the coldest quarter (CHELSA
- Bio19)
-- January (mid dry season) precipitation (CHELSA)
-- July (mid wet season) precipitation (CHELSA)
-- Interannual variability in cloud frequency (MODCF)
-- Intraannual variability in cloud frequency (MODCF)
-- Mean annual cloud frequency (MODCF)
-- Cloud frequency seasonality concentration (sum(monthly concentration
- vectors); MODCF)
-- Elevation (NASA DEM)
-- Soil electrical conductivity (soil_EC_mS_m, Cramer et al.Β 2019)
-- Soil extractable K (soil_Ext_K\_cmol_kg, Cramer et al.Β 2019)
-- Soil extractable NA (soil_Ext_Na_cmol_kg, Cramer et al.Β 2019)
-- Soil extractable P(soil_Ext_P\_mg_kg, Cramer et al.Β 2019)
-- Soil pH (Cramer et al.Β 2019)
-- Total soil C (Cramer et al.Β 2019)
-- Total soil N (Cramer et al.Β 2019)
-- Time since fire (generated from MODIS active fire products and
- CapeNature fire polygons)
-
-## Setting up the repo
-
- * This repo requires github credentials. To store those securely...
- * Credentials are decrypted with the function decryp_secret.sh
-
-# Extras
-
- * Call `targets::tar_renv(extras = character(0))` to write a `_packages.R` file to expose hidden dependencies.
- * Call `renv::init()` to initialize the `renv` lockfile `renv.lock` or `renv::snapshot()` to update it.
- * Commit `renv.lock` to your Git repository.
diff --git a/README.Rmd b/README.qmd
similarity index 63%
rename from README.Rmd
rename to README.qmd
index 1fe282d3..a49c5fa9 100644
--- a/README.Rmd
+++ b/README.qmd
@@ -10,6 +10,7 @@ knitr::opts_chunk$set(
comment = "#>"
)
library(tidyverse)
+library(rstac)
```
@@ -44,6 +45,109 @@ targets::tar_glimpse() %>%
webshot::webshot("img/network.html", "img/network.png")
```
+# Data Archive Summary
+
+```{r stac-summary, message=FALSE, warning=FALSE}
+
+# Connect to STAC catalog
+stac_base_url <- "https://github.com/AdamWilsonLab/emma_envdata/releases/download/stac"
+cat_url <- paste0(stac_base_url, "/catalog.json")
+
+tryCatch({
+ # Load root catalog
+ cat <- stac(cat_url)
+
+ # Get MODIS VI collection
+ modis_link <- cat$doc$links %>%
+ Filter(function(x) x$rel == "child" && grepl("MODIS.*VI|modis.*vi", x$title, ignore.case = TRUE), .)
+
+ if (length(modis_link) > 0) {
+ collection_url <- paste0(stac_base_url, "/modis_vi/collection.json")
+ collection <- stac(collection_url)
+
+ # Get all items
+ items <- stac_search(collection) %>% get_request()
+
+ if (!is.null(items$features) && length(items$features) > 0) {
+ # Extract item metadata
+ items_table <- data.frame(
+ Month = sapply(items$features, function(x) x$id),
+ Start = sapply(items$features, function(x) x$properties$start_datetime),
+ End = sapply(items$features, function(x) x$properties$end_datetime),
+ stringsAsFactors = FALSE
+ ) %>%
+ mutate(
+ Start = as.Date(substr(Start, 1, 10)),
+ End = as.Date(substr(End, 1, 10))
+ ) %>%
+ arrange(Start)
+
+ # Display summary statistics
+ cat("## MODIS Enhanced Vegetation Index (EVI)\n\n")
+
+ if (!is.null(collection$doc$description)) {
+ cat("**Collection Description:** ", collection$doc$description, "\n\n")
+ }
+
+ cat("**Temporal Coverage:** ", min(items_table$Start), " to ", max(items_table$End), "\n\n")
+ cat("**Total Months Available:** ", nrow(items_table), "\n\n")
+ cat("**Last Updated:** ", max(items_table$End), "\n\n")
+ } else {
+ cat("No items found in MODIS VI collection\n")
+ }
+ } else {
+ cat("MODIS VI collection not found in STAC catalog\n")
+ }
+}, error = function(e) {
+ cat("Could not access STAC catalog: ", e$message, "\n")
+})
+```
+
+### Available Data Files
+
+```{r stac-table, echo=FALSE}
+if (exists("items_table")) {
+ knitr::kable(
+ items_table,
+ format = "markdown",
+ caption = "MODIS VI monthly data files in archive"
+ )
+}
+```
+
+### Accessing Data with rstac
+
+Users can programmatically access this data using the `rstac` R package:
+
+```{r, eval=FALSE}
+library(rstac)
+library(arrow)
+
+# Connect to STAC collection
+stac_base_url <- "https://github.com/AdamWilsonLab/emma_envdata/releases/download/stac"
+cat_url <- paste0(stac_base_url, "/catalog.json")
+cat <- stac(cat_url)
+
+collection_url <- paste0(stac_base_url, "/modis_vi/collection.json")
+collection <- stac(collection_url)
+
+# Search for items in a date range
+results <- stac_search(
+ q = collection,
+ datetime = "2026-01-01/2026-12-31"
+) %>%
+ get_request()
+
+# Download and read parquet files
+if (!is.null(results$features)) {
+ for (item in results$features) {
+ url <- item$assets$data$href
+ df <- arrow::read_parquet(url)
+ # Process data...
+ }
+}
+```
+
# Workflow Notes
## Runtime and frequency
@@ -104,9 +208,11 @@ Github places some constrains on actions, including memory limits and run time l
# Extras
- * Call `targets::tar_renv(extras = character(0))` to write a `_packages.R` file to expose hidden dependencies.
- * Call `renv::init()` to initialize the `renv` lockfile `renv.lock` or `renv::snapshot()` to update it.
- * Commit `renv.lock` to your Git repository.
+Package dependencies are specified in `DESCRIPTION`. To install all required packages:
+
+```r
+remotes::install_deps()
+```
# DOI
[](https://zenodo.org/badge/latestdoi/421127852)
diff --git a/README_files/figure-gfm/unnamed-chunk-3-1.png b/README_files/figure-gfm/unnamed-chunk-3-1.png
index 4dbfc023..ab241357 100644
Binary files a/README_files/figure-gfm/unnamed-chunk-3-1.png and b/README_files/figure-gfm/unnamed-chunk-3-1.png differ
diff --git a/_targets.R b/_targets.R
index d8714617..2b044cb8 100644
--- a/_targets.R
+++ b/_targets.R
@@ -1,304 +1,224 @@
-message("Starting tar_make()")
-print("Starting tar_make() - print")
-
-library(targets)
-library(tarchetypes)
-library(visNetwork)
-library(future) #not sure why this is needed, but we get an error in some of the files without it
-library(googledrive)
-
-#If running this locally, make sure to set up github credentials using gitcreds::gitcreds_set()
+# ============================================================================
+# EMMA Environmental Data Pipeline
+# ============================================================================
+# This pipeline assembles environmental datasets for the EMMA project using
+# targets for workflow orchestration.
-#devtools::install_github(repo = "bmaitner/rgee",
-# ref = "noninteractive_auth")
+message("Starting tar_make()")
-# Ensure things are clean
- unlink(file.path("data/temp/"), recursive = TRUE, force = TRUE)
- unlink(file.path("data/raw_data/", recursive = TRUE, force = TRUE))
- message(paste("Objects:",ls(),collapse = "\n"))
+devtools::load_all() # load all functions in R
+description_packages <- load_description_packages(verbose=TRUE) # Load all packages from DESCRIPTION and get list
-# source all files in R folder
- lapply(list.files("R",pattern="[.]R",full.names = T), source)
- message(paste("Objects:",ls(),collapse = "\n")) # To make sure all packages are loaded
+# check what system we are on
+ sys_info <- Sys.info(); message(paste("System info:",paste(names(sys_info), sys_info, sep="=", collapse = "; ")))
+ # if nodename includes "ccr.buffalo.edu", set working directory to /gscratch/scrubbed/...
+ if (grepl("ccr.buffalo.edu", sys_info[["nodename"]])) {
+ setwd("~/project/projects/emma/emma_envdata")
+ message(paste("Set working directory to:", getwd()))
+ }
+#If running this locally, make sure to set up github credentials using gitcreds::gitcreds_set()
options(tidyverse.quiet = TRUE)
- #options(clustermq.scheduler = "multicore")
-
- tar_option_set(packages = c("cmdstanr", "posterior", "bayesplot", "tidyverse",
- "stringr","knitr","sf","stars","units",
- "cubelyr","rgee", "reticulate"))
-
-#set JSON token location (should be authorized for drive and earth engine)
- json_token <- "secrets/ee-wilsonlab-emma-ef416058504a.json"
-
-# ee authentication
- if(T) {
- message("loading rgee")
-# rgee::ee_install_set_pyenv('/usr/bin/python3','r-reticulate', confirm = F)
- library(rgee)
- #Initializing with service account key
- service_account <- jsonlite::read_json(json_token)$client_email
- credentials <- ee$ServiceAccountCredentials(service_account, json_token)
- ee$Initialize(credentials = credentials)
+ # Ensure output directories exist early (before terra options)
+ dir.create("data/raw", recursive = TRUE, showWarnings = FALSE)
+ dir.create("data/temp", recursive = TRUE, showWarnings = FALSE)
+ dir.create("data/temp/terra", recursive = TRUE, showWarnings = FALSE)
+ dir.create("data/releases", recursive = TRUE, showWarnings = FALSE)
+ dir.create("data/target_outputs", recursive = TRUE, showWarnings = FALSE)
+
+ # GitHub release repository configuration - releases are used to store target objects and publish final data
+ gh_repo_config <- list(
+ repo = "AdamWilsonLab/emma_envdata",
+ tag = "objects_current",
+ cache_dir = "data/target_outputs/.tar_cache" #this is local cache for speed
+ )
- #Setting up needed objects for rgee
+ # Store config as environment variables for upload function to use
+ Sys.setenv(
+ TAR_GH_RELEASE_REPO = gh_repo_config$repo,
+ TAR_GH_RELEASE_TAG = gh_repo_config$tag,
+ TAR_GH_RELEASE_FORMAT = gh_repo_config$format,
+ TAR_GH_RELEASE_CACHE_DIR = gh_repo_config$cache_dir
+ )
- message("Initializing rgee")
+ tar_option_set(
+ memory="transient",
+ garbage_collection = TRUE, #run gc() after each target to free memory
+ packages = description_packages, # Use all packages from DESCRIPTION file
+ repository = "local", # Store locally; manual upload after tar_make() completes
+ cue = tar_cue(mode = "thorough") # Recompute if any inputs change
+ )
- ee_Initialize(drive = TRUE,
- gcs = FALSE,
- use_oob = FALSE,
- drive_cred_path = json_token,
- gcs_cred_path = json_token,
- ee_cred_path = json_token)
+ terraOptions(tempdir = "data/temp/terra", memfrac = 0.6)
+ # Set cleanup behavior based on execution environment
+ # In GitHub Actions, we want to clean up temp files to avoid filling up disk space. Locally, we may want to keep them for debugging or inspection.
+ cleanup_mode <- Sys.getenv("GITHUB_ACTIONS") == "true"
+ if (interactive()) {
+ message("Cleanup mode: ", if (cleanup_mode) "ENABLED (GitHub Actions)" else "DISABLED (Local server)")
}
-# # Sys.setenv(GOOGLE_APPLICATION_CREDENTIALS = "secrets/ee-wilsonlab-emma-ef416058504a.json")
-# message("Starting tar_make()")
-# print("Starting tar_make() - print")
-
-# library(targets)
-# library(tarchetypes)
-# library(visNetwork)
-# library(future) #not sure why this is needed, but we get an error in some of the files without it
-# options(gargle_verbosity = "debug")
-# library(googledrive)
-# library(jsonlite)
-
-# library(jsonlite)
-# # tok <- fromJSON("secrets/ee-wilsonlab-emma-ef416058504a.json")
-# # print(tok$scopes) # or tok$scopes
-
-# library(reticulate)
-# # message("------ reticulate::py_discover_config() ------")
-# # print(py_discover_config())
-
-# # message("------ checking ee module availability ------")
-# # print(py_module_available("ee"))
-
-# # message("------ py_config() output ------")
-# # print(py_config())
-
-# #If running this locally, make sure to set up github credentials using gitcreds::gitcreds_set()
-
-# #devtools::install_github(repo = "bmaitner/rgee",
-# # ref = "noninteractive_auth")
-
-# # Ensure things are clean
-# unlink(file.path("data/temp/"), recursive = TRUE, force = TRUE)
-# unlink(file.path("data/raw_data/"), recursive = TRUE, force = TRUE)
-# message(paste("Objects:",ls(),collapse = "\n"))
-
-# # source all files in R folder
-# lapply(list.files("R",pattern="[.]R",full.names = T), source)
-# message(paste("Objects:",ls(),collapse = "\n")) # To make sure all packages are loaded
-
-
-# options(tidyverse.quiet = TRUE)
-# #options(clustermq.scheduler = "multicore")
-
-# tar_option_set(packages = c("cmdstanr", "posterior", "bayesplot", "tidyverse",
-# "stringr","knitr","sf","stars","units",
-# "cubelyr","rgee", "reticulate"))
-
-# #set JSON token location (should be authorized for drive and earth engine)
-# json_token <- "secrets/ee-wilsonlab-emma-ef416058504a.json"
-
-# # drive_auth(path = json_token)
-
-# # ee authentication
-# if(T) {
-# message("loading rgee")
-# py_run_string("import ee")
-# py_run_string("print(ee.__version__)")
-# # rgee::ee_install_set_pyenv('/usr/bin/python3','r-reticulate', confirm = F)
-# library(rgee)
-# print(packageVersion("rgee"))
-# options(rgee.verbose = TRUE)
-# options(gargle_verbosity = "debug")
-# #Initializing with service account key
-
-
-# # unlink("~/.config/earthengine", recursive = TRUE, force = TRUE)
-# #ee$Authenticate(auth_mode='appdefault', quiet=TRUE)
-# message("Authentication is completed")
-# # rgee::ee_clean_credentials()
-# service_account <- jsonlite::read_json(json_token)$client_email
-# credentials <- ee$ServiceAccountCredentials(service_account, json_token)
-# ee$Initialize(credentials=credentials)
-# message("Initialization is completed")
-
-# # point to your service-account JSON
-# # Sys.setenv(GOOGLE_APPLICATION_CREDENTIALS = json_token)
-
-# # preload Drive & GCS creds headlessly
-# #googledrive::drive_auth(path = json_token, cache = FALSE)
-# #googleCloudStorageR::gcs_auth(json_file = json_token)
-# #dir.create("~/.config/earthengine", recursive = TRUE, showWarnings = FALSE)
-# message("Before ee_Initialize")
-
-# # App-Default auth for rgee (no browser)
-# # drive_auth(path = json_token, cache = FALSE)
-# # gargle::gargle_oauth_cache()
-# # token <- gargle::credentials_service_account(
-# # path = json_token,
-# # scopes = NULL
-
-# # )
-# # googledrive::drive_auth(token = token)
-# ee_Authenticate(auth_mode='appdefault', quiet=TRUE) # , scopes='https://www.googleapis.com/auth/cloud-platform',
-# # ee_Initialize(
-# # # user= "20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner",
-# # # # user = "emma-envdata@ee-wilsonlab-emma.iam.gserviceaccount.com",
-# # # credentials = "secrets/ee-wilsonlab-emma-ef416058504a.json",
-# # credentials = "/github/home/.config/earthengine/",
-# # # # drive = TRUE,
-# # # # gcs = FALSE,
-# # # project = "ee-wilsonlab-emma",
-# # # # auth_mode = 'service_account',
-# # auth_quiet = TRUE,
-# # quiet = TRUE
-# # )
-# #ee_clean_user_credentials()
-# #ee_install_upgrade()
-# # ee_Authenticate(auth_mode='appdefault', quiet=TRUE)
-
-# #ee_Authenticate()
-# ee_Initialize()
-# # #project = "ee-wilsonlab-emma",
-# # #scopes='https://www.googleapis.com/auth/devstorage.full_control',
-# # credentials=credentials,
-# # auth_mode = "gcloud",
-# # quiet = TRUE
-# # ) #auth_mode="appdefault", quiet = TRUEγ
£, credentials=credentials, project = "ee-wilsonlab-emma",
-# reticulate::py_last_error()
-# message("ee_Initialize is completed")
-# # unlink("~/.config/earthengine", recursive = TRUE, force = TRUE)
-# # unlink("~/.rgee", recursive = TRUE, force = TRUE)
-# # dir.create("~/.config/earthengine", recursive = TRUE, showWarnings = FALSE)
-# # file.create("~/.config/earthengine/rgee_sessioninfo.txt")
-# # options(rgee.session.info = FALSE)
-
-# #Setting up needed objects for rgee
-# message("Initializing rgee")
-
-# # ee_Initialize(
-# # service_account = "emma-envdata@ee-wilsonlab-emma.iam.gserviceaccount.com",
-# # credentials = "secrets/ee-wilsonlab-emma-ef416058504a.json",
-# # drive = TRUE,
-# # gcs = TRUE
-# # )
-# message("After ee_Initialize")
-# # # 3) JSONμμ μλΉμ€ κ³μ μ΄λ©μΌ μΆμΆ
-# # key_path <- Sys.getenv("GOOGLE_APPLICATION_CREDENTIALS")
-# # sa_email <- read_json(key_path)$client_email
-
-# # # 4) SaK(Service account Key)λ₯Ό rgee μ격μ¦λͺ
ν΄λλ‘ λ³΅μ¬Β·κ²μ¦
-# # ee_utils_sak_copy(
-# # sakfile = key_path,
-# # users = sa_email
-# # )
-# # ee_utils_sak_validate(
-# # users = sa_email,
-# # quiet = TRUE
-# # )
-
-# # # 5) Earth Engine λΉλνν μ΄κΈ°ν (μλΉμ€ κ³μ λͺ¨λ)
-# # ee_Initialize(
-# # email = sa_email,
-# # project = "ee-wilsonlab-emma",
-# # auth_mode = "service_account",
-# # quiet = TRUE
-# # )
-
-# # # 6) rgee_sessioninfo.txt μμ± λ³΄μ₯
-# # ee_sessioninfo(
-# # email = sa_email,
-# # user = sa_email
-# # )
-
-# # message("Earth Engine non-interactive initialization complete.")
-# }
-
+# Ensure things are clean
+# unlink(file.path("data/temp/"), recursive = TRUE, force = TRUE)
+# unlink(file.path("data/raw_data/", recursive = TRUE, force = TRUE))
+# message(paste("Objects:",ls(),collapse = "\n"))
-list(
+ # Set MODIS date range as variables or targets (customize as needed)
+ modis_start_date <- "2000-02-18" # or tar_target(...)
+ modis_start_date <- "2026-01-01" # or tar_target(...)
+ modis_end_date <- as.character(Sys.Date())
-# #Prep needed files # start
+list(
tar_target(
- vegmap_shp, # 2018 National Vegetation Map http://bgis.sanbi.org/SpatialDataset/Detail/1674
- "data/manual_download/VEGMAP2018_AEA_16082019Final/NVM2018_AEA_V22_7_16082019_final.shp",
+ vegmap_shp,
+ download_vegmap_release(
+ repo = "AdamWilsonLab/emma_envdata",
+ tag = "vegmap2024",
+ file = "NVM2024final_Shapefile.zip",
+ local_dir = "data/manual_download/NVM2024",
+ shapefile_name = "NVM2024Final_IEM5_12_07012025.shp"
+ ),
format = "file"
),
tar_target(
remnants_shp,
"data/manual_download/RLE_2021_Remnants/RLE_Terr_2021_June2021_Remnants_ddw.shp",
- format = "file"
+ format="file"
),
tar_target(
- sanbi_fires_shp,
- st_read("data/manual_download/All_Fires/All_Fires_20_21_gw.shp")
+ capenature_fires_shp,
+ "data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp",
+ format="file"
),
+# Get country boundary
+ tar_target(
+ country.parquet,
+ get_country(),
+ format = "file"
+ ),
- tar_target(
- country,
- national_boundary()
- )
-,
+# Create domain file based on country boundary and vegmap
+ tar_target(
+ domain_boundary.parquet,
+ domain_define(vegmap_shp = vegmap_shp, country = country.parquet),
+ format = "file"
+ ),
+
+# Stable bounding box for downloads (50km buffer around domain)
+ tar_target(
+ domain_bbox.parquet,
+ make_domain_bbox(domain_boundary.parquet, buffer_m = 50000, out_file = "data/target_outputs/domain_bbox.parquet"),
+ format = "file",
+ cue = tar_cue(mode = "never") # Never re-downloads unless manually invalidated, even if analysis domain changes
+ ),
+
+# Domain raster with pixel IDs, remnants, and distance to remnants. This defines the model grid that is used for everything!
+ tar_target(
+ domain_nc,
+ domain_rasterize(
+ domain = sfarrow::st_read_parquet(domain_boundary.parquet),
+ remnants_shp = remnants_shp,
+ out_file = "data/target_outputs/domain.nc"
+ ),
+ format = "file",
+ cue = tar_cue(mode = "never") # Never rerun unless manually invalidated because this will trigger complete reprocessing of rs data
+ # tar_invalidate(domain_nc) # run this to force recompute, which will trigger redownloading all RS data from appeears
- tar_target(
- vegmap,
- get_vegmap(vegmap_shp)
),
+ # Convert domain raster to geoparquet for spatial reference with coordinates and pid
tar_target(
- domain,
- domain_define(vegmap = vegmap, country)
- )
-,
+ domain_geoparquet,
+ domain_to_geoparquet(
+ domain_raster_file = domain_nc,
+ out_file = "data/target_outputs/domain.parquet",
+ verbose = TRUE
+ ),
+ format = "file"
+ ),
+# Rasterize the vegetation map
+ tar_target(
+ vegmap_nc,
+ data_vegmap(domain_raster = domain_nc,
+ vegmap_shp = vegmap_shp,
+ out_file = "data/target_outputs/vegmap.nc"),
+ format = "file"),
+
+
+ # tar_target(
+ # protected_area_distance_release,
+ # process_release_protected_area_distance(template_release,
+ # out_file = "protected_area_distance.tif",
+ # temp_directory = "data/temp/protected_area",
+ # out_tag = "processed_static")
+ # ),
+
+# tar_target(
+# alos_release,
+# get_release_alos(temp_directory = "data/temp/raw_data/alos/",
+# tag = "raw_static",
+# domain = domain,
+# json_token)
+# )
+#,
+
+# Climate CHELSA bioclimatic variables (BIO1-BIO19)
+ tar_target(
+ climate_chelsa,
+ get_climate_chelsa(
+ domain = sfarrow::st_read_parquet(domain_boundary.parquet),
+ cleanup = cleanup_mode,
+ verbose = TRUE),
+ format = "file"
+ ),
-# # # # Infrequent updates via releases
+ # tar_terra_rast(
+ # clouds_wilson_release,
+ # get_release_clouds_wilson(temp_directory = "data/temp/raw_data/clouds_wilson/",
+ # tag = "raw_static",
+ # domain,
+ # sleep_time = 180)
+ # ),
+ ##################### AppEEARS Static Data Processing #########################
+ # Sequential targets for AppEEARS elevation: submit task, then poll for results
+ # Allows independent timeouts and retries for long-running API calls
tar_target(
- alos_release,
- get_release_alos(temp_directory = "data/temp/raw_data/alos/",
- tag = "raw_static",
- domain = domain,
- json_token)
- )
-,
-
- tar_target(
- climate_chelsa_release,
- get_release_climate_chelsa(temp_directory = "data/temp/raw_data/climate_chelsa/",
- tag = "raw_static",
- domain = domain)
- )
-,
+ elevation_task_id,
+ submit_elevation_task(
+ domain_vector = sfarrow::st_read_parquet(domain_boundary.parquet),
+ verbose = TRUE
+ )
+ ),
tar_target(
- clouds_wilson_release,
- get_release_clouds_wilson(temp_directory = "data/temp/raw_data/clouds_wilson/",
- tag = "raw_static",
- domain,
- sleep_time = 180)
+ elevation,
+ download_elevation_results(
+ task_id = elevation_task_id,
+ domain_vector = sfarrow::st_read_parquet(domain_boundary.parquet),
+ domain_raster = domain_nc,
+ out_file = "data/target_outputs/elevation_nasadem.nc",
+ temp_directory = "data/temp/raw_data/elevation_nasadem/",
+ verbose = TRUE
),
+ format = "file"
+ ),
- tar_target(
- elevation_nasadem_release,
- get_release_elevation_nasadem(temp_directory = "data/temp/raw_data/elevation_nasadem/",
- tag = "raw_static",
- domain)
- )
-,
+ # Generate human-readable manifest of all targets for release documentation
+# tar_target(
+# release_manifest,
+# generate_release_manifest(),
+# format = "file"
+# )
+# #,
#Temporarily commented out, seems to be an issue with URL for landcover data at present
# tar_target(
@@ -308,12 +228,12 @@ list(
# domain = domain)
# ),
#
- tar_target(
- precipitation_chelsa_release,
- get_release_precipitation_chelsa(temp_directory = "data/temp/raw_data/precipitation_chelsa/",
- tag = "raw_static",
- domain = domain)
- ),
+ # tar_target(
+ # precipitation_chelsa_release,
+ # get_release_precipitation_chelsa(temp_directory = "data/temp/raw_data/precipitation_chelsa/",
+ # tag = "raw_static",
+ # domain = domain)
+ # )#,
# ## commented out soil_gcfr_release at present due to API/rdryad issues.
# ## Emailed dryad folks on 2024/01/04, it seems the API update broke RDryad
@@ -327,406 +247,390 @@ list(
# # domain)
# # ),
#
-# # # # # Frequent updates via releases
-
- tar_age(
- fire_modis_release,
- get_release_fire_modis(temp_directory = "data/temp/raw_data/fire_modis/",
- tag = "raw_fire_modis",
- domain = domain,
- max_layers = 5,
- sleep_time = 5,
- json_token = json_token,
- verbose = FALSE),
- #age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- age = as.difftime(0, units = "hours")
- ),
-
- tar_age(
- kndvi_modis_release,
- get_release_kndvi_modis(temp_directory = "data/temp/raw_data/kndvi_modis/",
- tag = "raw_kndvi_modis",
- domain = domain,
- max_layers = 5,
- sleep_time = 5,
- json_token = json_token,
- verbose = TRUE),
- age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- #age = as.difftime(0, units = "hours")
- ),
-
- tar_age(
- ndvi_modis_release,
- get_release_ndvi_modis(temp_directory = "data/temp/raw_data/ndvi_modis/",
- tag = "raw_ndvi_modis",
- domain = domain,
- max_layers = 12,
- sleep_time = 5,
- json_token = json_token),
- #age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- age = as.difftime(0, units = "hours")
- ),
-
- tar_age(
- ndvi_viirs_release,
- get_release_ndvi_viirs(temp_directory = "data/temp/raw_data/ndvi_viirs/",
- tag = "raw_ndvi_viirs",
- domain,
- max_layers = 3,
- sleep_time = 30,
- json_token = json_token),
- age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- #age = as.difftime(0, units = "hours")
- ),
-
-
- tar_age(
- ndvi_dates_modis_release,
- get_release_ndvi_dates_modis(temp_directory = "data/temp/raw_data/ndvi_dates_modis/",
- repo_tag = "raw_ndvi_dates_modis",
- domain = domain,
- max_layers = 5,
- sleep_time = 10,
- json_token = json_token),
- #age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- age = as.difftime(0, units = "hours")
- ),
-
- tar_age(
- ndvi_dates_viirs_release,
- get_release_ndvi_dates_viirs(temp_directory = "data/temp/raw_data/ndvi_dates_viirs/",
- tag = "raw_ndvi_dates_viirs",
- domain = domain,
- max_layers = 3,
- sleep_time = 30,
- json_token = json_token),
- age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- #age = as.difftime(0, units = "hours")
- ),
-
-
-
- tar_age(mean_ndvi_release,
- get_release_mean_ndvi_modis(temp_directory = "data/temp/raw_data/mean_ndvi_modis/",
- tag = "current",
- domain = domain,
- sleep_time = 1,
- json_token = json_token),
- #age = as.difftime(7, units = "days")
- #age = as.difftime(1, units = "days")
- age = as.difftime(0, units = "hours")
- ),
-
-# # # # tar_age(
-# # # # ndwi_modis_release,
-# # # # get_release_ndwi_modis(temp_directory = "data/temp/raw_data/NDWI_MODIS/",
-# # # # tag = "current",
-# # # # domain,
-# # # # drive_cred_path = json_token),
-# # # # age = as.difftime(7, units = "days")
-# # # # #age = as.difftime(1, units = "days")
-# # # # #age = as.difftime(0, units = "hours")
-# # # # ),
-# # #
-# # #
-# # #
-# # # # # # Fixing projection via releases
-
-
- tar_target(
- correct_fire_release_proj_and_extent,
- process_fix_modis_release_projection_and_extent(temp_directory = "data/temp/raw_data/fire_modis/",
- input_tag = "raw_fire_modis",
- output_tag = "clean_fire_modis",
- max_layers = NULL,
- sleep_time = 30,
- verbose = TRUE,
- ... = fire_modis_release)
- ),
-
- tar_target(
- correct_ndvi_release_proj_and_extent,
- process_fix_modis_release_projection_and_extent(temp_directory = "data/temp/raw_data/ndvi_modis/",
- input_tag = "raw_ndvi_modis",
- output_tag = "clean_ndvi_modis",
- max_layers = NULL,
- sleep_time = 30,
- verbose = TRUE,
- ... = ndvi_modis_release)
- ),
+##################### AppEEARS Dynamic Data Processing #########################
+
+# tar_age(
+# fire_modis_release,
+# get_release_fire_modis_appeears(temp_directory = "data/temp/raw_data/fire_modis/",
+# tag = "raw_fire_modis_nc",
+# domain = domain,
+# max_layers = 5,
+# sleep_time = 5,
+# verbose = TRUE),
+# age = as.difftime(7, units = "days")
+# #age = as.difftime(1, units = "days")
+# #age = as.difftime(0, units = "hours"),
+# cue = tar_cue(mode = if (run_mode == "update") "always" else "thorough")
+# ),
+
+ # ============================================================================
+ # MODIS VI Download Pipeline (Dynamically Branched)
+ # ============================================================================
+
+ # Identify which monthly periods need to be downloaded
tar_target(
- correct_ndvi_dates_release_proj_and_extent,
- process_fix_modis_release_projection_and_extent(temp_directory = "data/temp/raw_data/ndvi_dates_modis/",
- input_tag = "raw_ndvi_dates_modis",
- output_tag = "clean_ndvi_dates_modis",
- max_layers = NULL,
- sleep_time = 30,
- verbose = TRUE,
- ... = ndvi_dates_modis_release)
+ modis_vi_to_download,
+ {
+ output_dir <- "data/target_outputs/modis_vi"
+
+ # Check which monthly periods have already been downloaded
+ # identify_missing_vi() checks for existing NetCDF files
+ missing <- identify_missing_vi(
+ output_dir = output_dir,
+ dataset = "modis_vi",
+ start_date = modis_start_date,
+ end_date = modis_end_date
+ )
+
+ # Always include the current month to ensure up-to-date data
+ today <- Sys.Date()
+ current_month_start <- as.Date(paste0(format(today, "%Y-%m"), "-01"))
+ current_month_end <- as.Date(paste0(format(today + 31, "%Y-%m"), "-01")) - 1
+ current_month_str <- format(current_month_start, "%Y-%m")
+
+ # Check if current month is already in missing
+ current_in_missing <- any(missing$date_str == current_month_str)
+
+ if (!current_in_missing) {
+ current_row <- data.frame(
+ month_start = current_month_start,
+ month_end = current_month_end,
+ date_str = current_month_str
+ )
+ missing <- rbind(missing, current_row)
+ }
+
+ if (nrow(missing) == 0) {
+ message("All monthly periods from ", modis_start_date, " to ", modis_end_date, " already downloaded")
+ # Return empty data frame with correct structure
+ data.frame(
+ month_start = as.Date(character(0)),
+ month_end = as.Date(character(0)),
+ date_str = character(0)
+ )
+ } else {
+ message("Found ", nrow(missing), " missing monthly periods to download (current month always included)")
+ missing
+ }
+ }
),
-
+ # Dynamically submit monthly AppEEARS tasks
tar_target(
- correct_ndvi_viirs_release_proj_and_extent,
- process_fix_modis_release_projection_and_extent(temp_directory = "data/temp/raw_data/ndvi_viirs/",
- input_tag = "raw_ndvi_viirs",
- output_tag = "clean_ndvi_viirs",
- max_layers = 30,
- sleep_time = 30,
- verbose = TRUE,
- ... = ndvi_viirs_release)
+ modis_vi_task_ids,
+ {
+ # Within this branch, modis_vi_to_download is auto-sliced to one row
+ submit_modis_vi(
+ domain_vector = sfarrow::st_read_parquet(domain_boundary.parquet),
+ month_start = modis_vi_to_download$month_start,
+ month_end = modis_vi_to_download$month_end
+ )
+ },
+ pattern = map(modis_vi_to_download),
),
+ # Download NetCDF files from AppEEARS (I/O only)
+ tar_target(
+ modis_vi_netcdf,
+ {
+ download_modis_vi_netcdf(
+ task_id = modis_vi_task_ids,
+ month_start = modis_vi_to_download$month_start,
+ temp_directory = "data/temp/raw_data/modis_vi_netcdf/",
+ cleanup = cleanup_mode,
+ verbose = TRUE
+ )
+ },
+ pattern = map(modis_vi_task_ids, modis_vi_to_download),
+ format = "file",
+ ),
- tar_target(
- correct_ndvi_dates_viirs_release_proj_and_extent,
- process_fix_modis_release_projection_and_extent(temp_directory = "data/temp/raw_data/ndvi_dates_viirs/",
- input_tag = "raw_ndvi_dates_viirs",
- output_tag = "clean_ndvi_dates_viirs",
- max_layers = 30,
- sleep_time = 30,
- verbose = TRUE,
- ... = ndvi_dates_viirs_release)
- ),
+ # Process NetCDF to parquet format
+ tar_target(
+ modis_vi_parquet,
+ {
+ netcdf_to_parquet(
+ netcdf_directory = modis_vi_netcdf,
+ domain_raster = domain_nc,
+ month_start = modis_vi_to_download$month_start,
+ out_dir = "data/target_outputs/modis_vi/",
+ cleanup = cleanup_mode,
+ verbose = TRUE
+ )
+ },
+ pattern = map(modis_vi_netcdf, modis_vi_to_download),
+ format = "file"
+ ),
- tar_target(
- correct_kndvi_release_proj_and_extent,
- process_fix_modis_release_projection_and_extent(temp_directory = "data/temp/raw_data/kndvi_modis/",
- input_tag = "raw_kndvi_modis",
- output_tag = "clean_kndvi_modis",
- max_layers = 30,
- sleep_time = 45,
- verbose = TRUE,
- ... = kndvi_modis_release)
- ), # second chunk
-
-# # # Processing via release
-
- tar_target(
- fire_doy_to_unix_date_release,
- process_release_fire_doy_to_unix_date(input_tag = "clean_fire_modis",
- output_tag = "processed_fire_dates",
- temp_directory = "data/temp/processed_data/fire_dates/",
- sleep_time = 20,
- template_release = template_release,
- ... = correct_fire_release_proj_and_extent)
- ),
-
- tar_target(
- burn_date_to_last_burned_date_release,
- process_release_burn_date_to_last_burned_date(input_tag = "processed_fire_dates",
- output_tag = "processed_most_recent_burn_dates",
- temp_directory_input = "data/temp/processed_data/fire_dates/",
- temp_directory_output = "data/temp/processed_data/most_recent_burn_dates/",
- sleep_time = 180,
- sanbi_sf = sanbi_fires_shp,
- expiration_date = NULL,
- ... = fire_doy_to_unix_date_release)
+ # Generate STAC Collection for MODIS VI dataset
+ tar_target(
+ modis_vi_stac,
+ generate_modis_vi_stac(
+ parquet_files = modis_vi_parquet, # Automatically aggregated from branched target
+ parquet_dir = "data/target_outputs/modis_vi",
+ stac_dir = "data/stac/modis_vi",
+ parent_catalog_path = "data/stac",
+ gh_repo = "AdamWilsonLab/emma_envdata",
+ gh_release_tag = "data_modis_vi_current",
+ verbose = TRUE
),
+ format = "file"
+ ),
-
- tar_target(
- ndvi_relative_days_since_fire_release,
- process_release_ndvi_relative_days_since_fire(temp_input_ndvi_date_folder = "data/temp/raw_data/ndvi_dates_modis/",
- temp_input_fire_date_folder = "data/temp/processed_data/most_recent_burn_dates/",
- temp_fire_output_folder = "data/temp/processed_data/ndvi_relative_time_since_fire/",
- input_fire_dates_tag = "processed_most_recent_burn_dates",
- input_modis_dates_tag = "clean_ndvi_dates_modis",
- output_tag = "processed_ndvi_relative_days_since_fire",
- sleep_time = 60,
- ... = burn_date_to_last_burned_date_release,
- ... = correct_ndvi_dates_release_proj_and_extent)
- ),
-
- tar_target(
- template_release,
- get_release_template_raster(input_tag = "clean_ndvi_modis",
- output_tag = "raw_static",
- temp_directory = "data/temp/template",
- ... = correct_ndvi_release_proj_and_extent)
- ),
-
- tar_target(
- remnants_release,
- domain_remnants_release(domain = domain,
- remnants_shp = remnants_shp,
- template_release,
- temp_directory = "data/temp/remnants",
- out_file = "remnants.tif",
- out_tag = "processed_static")
- ), # 3-1
-
- tar_target(
- remnant_distance_release,
- domain_distance_release(remnants_release = remnants_release,
- out_file = "remnant_distance.tif",
- temp_directory = "data/temp/remnants",
- out_tag = "processed_static")
- ),
-
- tar_target(
- protected_area_distance_release,
- process_release_protected_area_distance(template_release,
- out_file = "protected_area_distance.tif",
- temp_directory = "data/temp/protected_area",
- out_tag = "processed_static")
- ),
-
- tar_target(
- projected_alos_release,
- process_release_alos(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/alos/",
- template_release = template_release,
- sleep_time = 60,
- ... = alos_release)
- ),
-
- tar_target(
- projected_climate_chelsa_release,
- process_release_climate_chelsa(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/climate_chelsa/",
- template_release = template_release,
- ... = climate_chelsa_release)
+ # Generate parent STAC Catalog linking all datasets (MODIS VI, VIIRS VI, burned area, age, etc.)
+ tar_target(
+ emma_stac_catalog,
+ {
+ generate_emma_stac_catalog(
+ stac_base_dir = "data/stac",
+ dataset_collections = list(
+ modis_vi = "data/stac/modis_vi"
+ # Additional datasets will be added here as they become available:
+ # viirs_vi = "data/stac/viirs_vi",
+ # burned_area = "data/stac/burned_area",
+ # age = "data/stac/age"
),
-
- tar_target(
- projected_clouds_wilson_release,
- process_release_clouds_wilson(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/clouds_wilson/",
- template_release = template_release,
- sleep_time = 180,
- ... = clouds_wilson_release)
- ), # 3-2
-
- tar_target(
- projected_elevation_nasadem_release,
- process_release_elevation_nasadem(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/elevation_nasadem/",
- template_release = template_release,
- sleep_time = 0,
- ... = elevation_nasadem_release)
- ),
-
- tar_target(
- projected_landcover_za_release,
- process_release_landcover_za(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/landcover_za/",
- template_release,
- sleep_time = 60,
- ... = landcover_za_release)
+ gh_repo = "AdamWilsonLab/emma_envdata",
+ verbose = TRUE
)
- ,
-
- tar_target(
- projected_precipitation_chelsa_release,
- process_release_precipitation_chelsa(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/precipitation_chelsa/",
- template_release,
- sleep_time = 60,
- ... = precipitation_chelsa_release)
-
- ),
-
- tar_target(
- projected_soil_gcfr_release,
- process_release_soil_gcfr(input_tag = "raw_static",
- output_tag = "processed_static",
- temp_directory = "data/temp/raw_data/soil_gcfr/",
- template_release,
- sleep_time = 60,
- ... = soil_gcfr_release)
-
- ),
-
- tar_target(
- vegmap_modis_proj,
- process_release_biome_raster(template_release = template_release,
- vegmap_shp = vegmap_shp,
- domain = domain,
- temp_directory = "data/temp/raw_data/vegmap_raster/",
- sleep_time = 10)
-
- ),
-
-
-
-
-# # # # # Prep model data
-
- tar_target(
- stable_data_release,
- process_release_stable_data(temp_directory = "data/temp/processed_data/static/",
- input_tag = "processed_static",
- output_tag = "current",
- sleep_time = 120,
- ... = projected_precipitation_chelsa_release,
- ... = projected_landcover_za_release,
- ... = projected_elevation_nasadem_release,
- ... = projected_clouds_wilson_release,
- ... = projected_climate_chelsa_release,
- ... = projected_alos_release,
- ... = remnant_distance_release,
- ... = protected_area_distance_release,
- ... = projected_soil_gcfr_release)
- ),
-
- tar_target(
- ndvi_to_parquet_release,
- process_release_dynamic_data_to_parquet(temp_directory = "data/temp/raw_data/ndvi_modis/",
- input_tag = "clean_ndvi_modis",
- output_tag = "current",
- variable_name = "ndvi",
- sleep_time = 30,
- ... = correct_ndvi_release_proj_and_extent)
- ),
-
- tar_target(
- fire_dates_to_parquet_release,
- process_release_dynamic_data_to_parquet(temp_directory = "data/temp/processed_data/ndvi_relative_time_since_fire/",
- input_tag = "processed_ndvi_relative_days_since_fire",
- output_tag = "current",
- variable_name = "time_since_fire",
- sleep_time = 30,
- ... = ndvi_relative_days_since_fire_release)
- ),
+ },
+ format = "file",
+ deployment = "main"
+ ),
- tar_target(
- most_recent_fire_dates_to_parquet_release,
- process_release_dynamic_data_to_parquet(temp_directory = "data/temp/processed_data/most_recent_burn_dates/",
- input_tag = "processed_most_recent_burn_dates",
- output_tag = "current",
- variable_name = "most_recent_burn_dates",
- sleep_time = 30,
- ... = burn_date_to_last_burned_date_release)
- ),
-# periodically clean up google drive folder
+ # revise modis for viirs
+
+
+
+# # # # Processing via release
+
+# tar_target(
+# fire_doy_to_unix_date_release,
+# process_release_fire_doy_to_unix_date(input_tag = "clean_fire_modis",
+# output_tag = "processed_fire_dates",
+# temp_directory = "data/temp/processed_data/fire_dates/",
+# sleep_time = 20,
+# template_release = template_release,
+# ... = correct_fire_release_proj_and_extent)
+# ),
+
+# tar_target(
+# burn_date_to_last_burned_date_release,
+# process_release_burn_date_to_last_burned_date(input_tag = "processed_fire_dates",
+# output_tag = "processed_most_recent_burn_dates",
+# temp_directory_input = "data/temp/processed_data/fire_dates/",
+# temp_directory_output = "data/temp/processed_data/most_recent_burn_dates/",
+# sleep_time = 180,
+# sanbi_sf = sanbi_fires_shp,
+# expiration_date = NULL,
+# ... = fire_doy_to_unix_date_release)
+# ),
+
+
+# tar_target(
+# ndvi_relative_days_since_fire_release,
+# process_release_ndvi_relative_days_since_fire(temp_input_ndvi_date_folder = "data/temp/raw_data/ndvi_dates_modis/",
+# temp_input_fire_date_folder = "data/temp/processed_data/most_recent_burn_dates/",
+# temp_fire_output_folder = "data/temp/processed_data/ndvi_relative_time_since_fire/",
+# input_fire_dates_tag = "processed_most_recent_burn_dates",
+# input_modis_dates_tag = "clean_ndvi_dates_modis",
+# output_tag = "processed_ndvi_relative_days_since_fire",
+# sleep_time = 60,
+# ... = burn_date_to_last_burned_date_release,
+# ... = correct_ndvi_dates_release_proj_and_extent)
+# ),
+
+# tar_target(
+# template_release,
+# get_release_template_raster(input_tag = "clean_ndvi_modis",
+# output_tag = "raw_static",
+# temp_directory = "data/temp/template",
+# ... = correct_ndvi_release_proj_and_extent)
+# ),
+
+
+
+# tar_target(
+# projected_alos_release,
+# process_release_alos(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/alos/",
+# template_release = template_release,
+# sleep_time = 60,
+# ... = alos_release)
+# ),
+
+# tar_target(
+# projected_climate_chelsa_release,
+# process_release_climate_chelsa(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/climate_chelsa/",
+# template_release = template_release,
+# ... = climate_chelsa_release)
+# ),
+
+# tar_target(
+# projected_clouds_wilson_release,
+# process_release_clouds_wilson(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/clouds_wilson/",
+# template_release = template_release,
+# sleep_time = 180,
+# ... = clouds_wilson_release)
+# ), # 3-2
+
+# tar_target(
+# projected_elevation_nasadem_release,
+# process_release_elevation_nasadem(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/elevation_nasadem/",
+# template_release = template_release,
+# sleep_time = 0,
+# ... = elevation_nasadem_release)
+# ),
+
+# tar_target(
+# projected_landcover_za_release,
+# process_release_landcover_za(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/landcover_za/",
+# template_release,
+# sleep_time = 60,
+# ... = landcover_za_release)
+# )
+# ,
+
+# tar_target(
+# projected_precipitation_chelsa_release,
+# process_release_precipitation_chelsa(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/precipitation_chelsa/",
+# template_release,
+# sleep_time = 60,
+# ... = precipitation_chelsa_release)
+
+# ),
+
+# tar_target(
+# projected_soil_gcfr_release,
+# process_release_soil_gcfr(input_tag = "raw_static",
+# output_tag = "processed_static",
+# temp_directory = "data/temp/raw_data/soil_gcfr/",
+# template_release,
+# sleep_time = 60,
+# ... = soil_gcfr_release)
+
+# ),
+
+# tar_target(
+# vegmap_modis_proj,
+# process_release_biome_raster(template_release = template_release,
+# vegmap_shp = vegmap_shp,
+# domain = domain,
+# temp_directory = "data/temp/raw_data/vegmap_raster/",
+# sleep_time = 10)
+
+# ),
+
+
+
+
+# # # # # # Prep model data
+
+# tar_target(
+# stable_data_release,
+
+# tar_target(
+# fire_dates_to_parquet_release,
+# process_release_dynamic_data_to_parquet(temp_directory = "data/temp/processed_data/ndvi_relative_time_since_fire/",
+# input_tag = "processed_ndvi_relative_days_since_fire",
+# output_tag = "current",
+# variable_name = "time_since_fire",
+# sleep_time = 30,
+# ... = ndvi_relative_days_since_fire_release)
+# ),
+
+# tar_target(
+# most_recent_fire_dates_to_parquet_release,
+# process_release_dynamic_data_to_parquet(temp_directory = "data/temp/processed_data/most_recent_burn_dates/",
+# input_tag = "processed_most_recent_burn_dates",
+# output_tag = "current",
+# variable_name = "most_recent_burn_dates",
+# sleep_time = 30,
+# ... = burn_date_to_last_burned_date_release)
+# )
+
+ ##################### GitHub Release Uploads #########################
+
+ # Upload static data files (domain, elevation, climate, etc.)
+ # tar_target(
+ # upload_static_data,
+ # {
+ # upload_to_github_release(
+ # files = c(
+ # domain_boundary.parquet,
+ # elevation,
+ # climate_chelsa,
+ # vegmap_nc
+ # ),
+ # repo = "AdamWilsonLab/emma_envdata",
+ # release_tag = "static_current",
+ # release_name = "Static Data - Current",
+ # verbose = TRUE
+ # )
+ # },
+ # deployment = "main"
+ # ),
+
+ # Upload dynamic MODIS VI data files
+ tar_target(
+ upload_modis_vi_data,
+ {
+
+ # Get all parquet files from disk
+ parquet_files <- list.files(
+ "data/target_outputs/modis_vi",
+ pattern = "\\.parquet$",
+ full.names = TRUE
+ )
+
+ upload_to_github_release(
+ files = parquet_files,
+ repo = gh_repo_config$repo,
+ release_tag = "dynamic_modis_vi",
+ release_name = "Dynamic MODIS Vegetation Index",
+ verbose = TRUE,
+ modis_vi_parquet #include to force dependency on the parquet files being created before upload
+ )
+ },
+ deployment = "main"
+ ),
- tar_age(
- remove_ee_backup,
- clean_up(),
- #age = as.difftime(7, units = "days")
- age = as.difftime(0, units = "hours")
+ # Upload STAC metadata catalog
+ tar_target(
+ upload_stac_catalog,
+ {
+ # Ensure STAC targets are complete before proceeding
+ stac_parent <- emma_stac_catalog
+ stac_modis_items <- modis_vi_stac
+
+ stac_files <- c(
+ file.path("data/stac", "catalog.json"),
+ list.files("data/stac/modis_vi", pattern = "\\.json$", full.names = TRUE)
+ )
+
+ upload_to_github_release(
+ files = stac_files,
+ repo = gh_repo_config$repo,
+ release_tag = "stac",
+ release_name = "STAC Catalog - Current",
+ verbose = TRUE
+ )
+ },
+ deployment = "main"
)
-
)
-
-
-
-
-################################################################################
diff --git a/_targets/.gitignore b/_targets/.gitignore
index a8c51cda..23ab7917 100644
--- a/_targets/.gitignore
+++ b/_targets/.gitignore
@@ -1,3 +1,9 @@
+# CAUTION: do not edit this file by hand!
+# _targets/objects/ may have large data files,
+# and _targets/meta/process may have sensitive information.
+# It is good pratice to either commit nothing from _targets/,
+# or if your data is not too sensitive,
+# commit only _targets/meta/meta.
*
!.gitignore
!meta
diff --git a/_targets/meta/meta b/_targets/meta/meta
new file mode 100644
index 00000000..aad44afd
--- /dev/null
+++ b/_targets/meta/meta
@@ -0,0 +1,38 @@
+name|type|data|command|depend|seed|path|time|size|bytes|format|repository|iteration|parent|children|seconds|warnings|error
+modis_vi_to_download|stem|c639e19bb42857e0|92e00a234ba7236a|a96d110c8e3f76f4|775029312||t20511.6129038579s|s178b|178|rds|local|vector||modis_vi_to_download_995445d06381534e|0.008||
+remnants_shp|stem|d03b72197917a3dc|4adc3961bf1f92a3|2c530c1562a7fbd1|977857804|data/manual_download/RLE_2021_Remnants/RLE_Terr_2021_June2021_Remnants_ddw.shp|t20445.6915274758s|s726393404b|726393404|file|local|vector|||0||
+vegmap_shp|stem|9cade294bcf5aff4|66bbc84b2758a527|2c530c1562a7fbd1|-979952299|data/manual_download/NVM2024/shapefile/NVM2024Final_IEM5_12_07012025.shp|t20465.9081331559s|s499322168b|499322168|file|local|vector|||0.006||
+country.parquet|stem|2f020da9d9337cd7|81ddbcfc163d4096|2c530c1562a7fbd1|1129261391|data/target_outputs/country.parquet|t20511.6130437355s|s4488713b|4488713|file|local|vector|||11.321||
+emma_stac_catalog|stem|d1d60756752f5367|b48d3e3359ec0799|2c530c1562a7fbd1|400305184|data/stac/catalog.json|t20511.6158712733s|s868b|868|file|local|vector|||0.008||
+capenature_fires_shp|stem|13d0029b14d0e2de|cb2c5823b318874b|2c530c1562a7fbd1|-1931602475|data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp|t19949.7370138889s|s15173392b|15173392|file|local|vector|||0.001||
+domain.parquet|stem|c46b7f544a0c0629|b81897df57a5dbef|f6a11d3d418e76d8|522092063|data/raw/domain.parquet|t20511.616036936s|s2564266b|2564266|file|local|vector|||14.121|This is an initial implementation of Parquet/Feather file support and geo metadata. This is tracking version 0.1.0 of the metadata (https://github.com/geopandas/geo-arrow-spec). This metadata specification may change and does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.|
+domain_bbox.parquet|stem|25cd0fd1894c1176|8ef5b89424c2d345|9c999260891eaabc|1651755667|data/target_outputs/domain_bbox.parquet|t20511.6254390934s|s15057b|15057|file|local|vector|||0.025|This is an initial implementation of Parquet/Feather file support and geo metadata. This is tracking version 0.1.0 of the metadata (https://github.com/geopandas/geo-arrow-spec). This metadata specification may change and does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.|
+modis_vi_netcdf|pattern|8d29f41e12b3a094|ed27e16bdccf6054||-764661159||||5462392|file|local|vector||modis_vi_netcdf_894e9f5957fbfb12|0||
+modis_vi_parquet|pattern|a605346000427f94|d4bd92bf6265c44c||-266669619||||6800341|file|local|vector||modis_vi_parquet_70688497b90c650e|42.263||
+domain_boundary.parquet|stem|c46b7f544a0c0629|b81897df57a5dbef|f6a11d3d418e76d8|399009199|data/raw/domain_boundary.parquet|t20511.6392432368s|s2564266b|2564266|file|local|vector|||14.106|This is an initial implementation of Parquet/Feather file support and geo metadata. This is tracking version 0.1.0 of the metadata (https://github.com/geopandas/geo-arrow-spec). This metadata specification may change and does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.|
+modis_vi_task_ids_b1c365f5dff1a632|branch|687af2168a0d2e07|43340e81c4f32ef4|e5b43c8dcc4ca607|872828479||t20511.6392802057s|s89b|89|rds|local|vector|modis_vi_task_ids||3.08||
+modis_vi_task_ids|pattern|a672dec49d64ca1c|43340e81c4f32ef4||593130349||||89|rds|local|vector||modis_vi_task_ids_b1c365f5dff1a632|3.08||
+elevation_task_id|stem|bbf44597b9b8b642|f9bd270f9407f46a|e5b43c8dcc4ca607|1108492909||t20511.6393319318s|s87b|87|rds|local|vector|||4.329||
+domain_nc|stem|f4a72cf2ce8f9389|263ac436e41580e5|ac33cc46697840b1|191087344|data/target_outputs/domain.nc|t20511.6411944969s|s2292195b|2292195|file|local|vector|||160.794|attribute variables are assumed to be spatially constant throughout all geometries. NAs introduced by coercion to integer range. NAs introduced by coercion to integer range|
+modis_vi_netcdf_894e9f5957fbfb12|branch|61c838c6b1f9a5d2|ed27e16bdccf6054|f8d95ae6d5ca6228|-725027365|data/temp/raw_data/modis_vi_netcdf/|t20510.8522571188s|s5462392b|5462392|file|local|vector|modis_vi_netcdf||0.001||
+elevation|stem|99c75232029964be|dc3d519915a0e914|8af3afb7ef711469|223776802|data/target_outputs/elevation_nasadem.nc|t20511.6490582214s|s3257558b|3257558|file|local|vector|||25.126||
+vegmap_nc|stem|2528441099f91ef7|ff356bc5e06835c0|347faa175c3afcea|573083685|data/target_outputs/vegmap.nc|t20511.652485635s|s537402b|537402|file|local|vector|||108.018|attribute variables are assumed to be spatially constant throughout all geometries|
+domain_geoparquet|stem|45c57037de90eda1|fc79eb843faac3e9|ca1ef0429e724a28|-1258951668|data/target_outputs/domain.parquet|t20511.653124704s|s9803528b|9803528|file|local|vector|||1.002|This is an initial implementation of Parquet/Feather file support and geo metadata. This is tracking version 0.1.0 of the metadata (https://github.com/geopandas/geo-arrow-spec). This metadata specification may change and does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.|
+release_tag|object|c03465355186ceda|||||||||||||||
+existing|object|08d5f59e833de599|||||||||||||||
+parquet_to_geotif|function|eadecbbe2fb012c7|||||||||||||||
+verbose|object|988c41ba10911dc8|||||||||||||||
+repo|object|8533b5ca41d7ca06|||||||||||||||
+token|object|d5162055e1a864d2|||||||||||||||
+upload_stac_catalog|stem|da959f195c649105|5f46be48ab8be538|a92bbb96ba84e4aa|2024368032||t20511.8058566013s|s104b|104|rds|local|vector|||13.323|Failed to create release: 'stac' already exists!. Matching or more recent version of data/stac/modis_vi/collection.json found on GH, not uploading.. Matching or more recent version of data/stac/modis_vi/modis_vi_202601.json found on GH, not uploading.. Matching or more recent version of data/stac/modis_vi/modis_vi_202602.json found on GH, not uploading.|
+climate_chelsa|stem|cb28ff8939a1bcf0|6c9a1482b661fbcb|fcdcb4c5e838eac7|175391582|data/target_outputs//CHELSA_bio01_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio02_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio03_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio04_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio05_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio06_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio07_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio08_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio09_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio10_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio11_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio12_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio13_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio14_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio15_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio16_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio17_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio18_1981-2010_V.2.1.nc*data/target_outputs//CHELSA_bio19_1981-2010_V.2.1.nc|t20511.8447175439s|s6813490b|6813490|file|local|vector|||588.198||
+modis_vi_parquet_70688497b90c650e|branch|b940f3b0d1e49d80|31a4ebcbe30ed2e5|c7aec36a60ec43ee|2111712832|data/target_outputs/modis_vi//dynamic_modis_vi_202602.gz.parquet|t20511.8451693136s|s6800341b|6800341|file|local|vector|modis_vi_parquet||38.902||
+modis_vi_stac|stem|b7fee518fb5439c2|3b0d676f7989d3ed|f2856d8e2c6e29c9|-118413829|data/stac/modis_vi/collection.json|t20511.8451706268s|s2240b|2240|file|local|vector|||0.01||
+gh_repo_config|object|148485602b17d22f
+sys_info|object|ada53e2071fe586e
+modis_start_date|object|f1911cd6006b2c3d
+.Random.seed|object|80e117231c868546
+description_packages|object|a18d364edea3f253
+cleanup_mode|object|cc6a331fb3c0077c
+modis_end_date|object|ec7a789eac228ba0
+upload_modis_vi_data|stem|44fd56b943ee89f6|69cf2b2ea9f8a205|43a5f4c7de3c6a13|925520158||t20511.84678456s|s46b|46|rds|local|vector|||14.141|Failed to create release: 'dynamic_modis_vi' already exists!|
diff --git a/ccr_startup.sh b/ccr_startup.sh
new file mode 100644
index 00000000..840233c0
--- /dev/null
+++ b/ccr_startup.sh
@@ -0,0 +1,28 @@
+#! /bin/bash
+
+# Connect to CCR and request an interactive job
+
+ssh vortex.ccr.buffalo.edu
+salloc --cluster=faculty --qos=adamw --partition=adamw \
+ --job-name=InteractiveJob --nodes=1 --ntasks=4 \
+ --mem=10G -C INTEL --time=24:00:00
+
+export GROUP="adamw"
+export PROJECT_FOLDER="/projects/academic/"$GROUP
+export APPTAINER_CACHEDIR="/vscratch/grp-adamw/"$USER"/singularity"
+export SIF_PATH=$PROJECT_FOLDER"/users/"$USER"/singularity"
+export SIF_FILE="AdamWilsonLab-emma_docker-latest.sif"
+
+
+apptainer run \
+ --bind $PROJECT_FOLDER:$PROJECT_FOLDER \
+ --bind $APPTAINER_CACHEDIR/tmp:/tmp \
+ --bind $APPTAINER_CACHEDIR/run:/run \
+ $SIF_PATH/$SIF_FILE R
+
+
+
+# Test github actions locally
+act -j targets \
+ --platform ubuntu-latest=adamwilsonlab/emma:latest \
+ --container-architecture linux/amd64
\ No newline at end of file
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.dbf b/data/manual_download/All_Fires/All_Fires_20_21_gw.dbf
deleted file mode 100644
index 712fa7ae..00000000
--- a/data/manual_download/All_Fires/All_Fires_20_21_gw.dbf
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d6b6256fd8c9c3b9943579d919ac4d2605a4a7c5b66ec19e5c590d28017fb6df
-size 4455696
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.lyr b/data/manual_download/All_Fires/All_Fires_20_21_gw.lyr
deleted file mode 100644
index 91c7cbad..00000000
Binary files a/data/manual_download/All_Fires/All_Fires_20_21_gw.lyr and /dev/null differ
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.pdf b/data/manual_download/All_Fires/All_Fires_20_21_gw.pdf
deleted file mode 100644
index ecda38f4..00000000
Binary files a/data/manual_download/All_Fires/All_Fires_20_21_gw.pdf and /dev/null differ
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.prj b/data/manual_download/All_Fires/All_Fires_20_21_gw.prj
deleted file mode 100644
index a30c00a5..00000000
--- a/data/manual_download/All_Fires/All_Fires_20_21_gw.prj
+++ /dev/null
@@ -1 +0,0 @@
-GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]]
\ No newline at end of file
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.qml b/data/manual_download/All_Fires/All_Fires_20_21_gw.qml
deleted file mode 100644
index a784974e..00000000
--- a/data/manual_download/All_Fires/All_Fires_20_21_gw.qml
+++ /dev/null
@@ -1,577 +0,0 @@
-
-
-
- 1
- 1
- 1
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 0
- 0
- 1
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 0
-
-
- 0
- generatedlayout
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- RES_NAME
-
- 2
-
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.qpj b/data/manual_download/All_Fires/All_Fires_20_21_gw.qpj
deleted file mode 100644
index 5fbc831e..00000000
--- a/data/manual_download/All_Fires/All_Fires_20_21_gw.qpj
+++ /dev/null
@@ -1 +0,0 @@
-GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]]
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.sbn b/data/manual_download/All_Fires/All_Fires_20_21_gw.sbn
deleted file mode 100644
index fb296b86..00000000
Binary files a/data/manual_download/All_Fires/All_Fires_20_21_gw.sbn and /dev/null differ
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.sbx b/data/manual_download/All_Fires/All_Fires_20_21_gw.sbx
deleted file mode 100644
index 15303df9..00000000
Binary files a/data/manual_download/All_Fires/All_Fires_20_21_gw.sbx and /dev/null differ
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.shp b/data/manual_download/All_Fires/All_Fires_20_21_gw.shp
deleted file mode 100644
index e729e422..00000000
--- a/data/manual_download/All_Fires/All_Fires_20_21_gw.shp
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:acafbdd28da2f5329a16bfa4bc69e7c77baad88f9cc43dbc1a925672e230b8e5
-size 14277344
diff --git a/data/manual_download/All_Fires/All_Fires_20_21_gw.shx b/data/manual_download/All_Fires/All_Fires_20_21_gw.shx
deleted file mode 100644
index 9d0c67a7..00000000
Binary files a/data/manual_download/All_Fires/All_Fires_20_21_gw.shx and /dev/null differ
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.cpg b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.cpg
new file mode 100644
index 00000000..3ad133c0
--- /dev/null
+++ b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.cpg
@@ -0,0 +1 @@
+UTF-8
\ No newline at end of file
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.dbf b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.dbf
new file mode 100644
index 00000000..4a2ce1e6
--- /dev/null
+++ b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.dbf
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a0a89f91aea8c4c393f11a356ecbb87a7eb0b6c379a395fd4fdb8f9b11e343d
+size 3752962
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.lyr b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.lyr
new file mode 100644
index 00000000..f4ba38cf
Binary files /dev/null and b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.lyr differ
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.pdf b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.pdf
new file mode 100644
index 00000000..2024f82b
Binary files /dev/null and b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.pdf differ
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.prj b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.prj
new file mode 100644
index 00000000..f45cbadf
--- /dev/null
+++ b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.prj
@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]
\ No newline at end of file
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.qml b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.qml
new file mode 100644
index 00000000..8026a13c
--- /dev/null
+++ b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.qml
@@ -0,0 +1,972 @@
+
+
+
+ 1
+ 1
+ 1
+ 0
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0
+ 0
+ 1
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 0
+
+
+ 0
+ generatedlayout
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ "RES_NAME"
+
+ 2
+
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.sbn b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.sbn
new file mode 100644
index 00000000..bc393860
Binary files /dev/null and b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.sbn differ
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.sbx b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.sbx
new file mode 100644
index 00000000..c98d69a1
Binary files /dev/null and b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.sbx differ
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp
new file mode 100644
index 00000000..3c62efb3
--- /dev/null
+++ b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c9d77abce418d99fe169e64914d7779dc3d5db38c221a0396166ab46515c53c
+size 15173392
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp.xml b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp.xml
new file mode 100644
index 00000000..02e803eb
--- /dev/null
+++ b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shp.xml
@@ -0,0 +1,2 @@
+
+20240726134913001.0FALSECalculateField All_Fires_23_24_merge_fields1 AREA [Area_ha] VB #CalculateField All_Fires_23_24_merge_fields Area_ha [AREA] VB #CalculateField All_Fires_23_24_merge_fields RES_CODE Left( [FIRE_CODE],4 ) VB #CalculateField All_Fires_23_24_merge_fields yr_temp Mid( [FIRE_CODE],9,4 ) VB #CalculateField All_Fires_23_24_merge_fields mnth_temp Mid( [FIRE_CODE],6,2 ) VB #CalculateField All_Fires_23_24_merge_fields MONTH [mnth_temp] VB #CalculateField All_Fires_23_24_merge_fields YEAR [yr_temp] VB #CalculateField All_Fires_23_24_merge_fields Yr_mnth "[yr_temp] +"-" + [mnth_temp]" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Witzenberg Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Boland Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Kogelberg Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Garden Route Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Karoo Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Anysberg Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Overberg Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Langeberg Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "De Hoop Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Peninsula Landscape Unit" VB #CalculateField All_Fires_23_24_merge_fields LAND_UNIT "Ceder-Berg Landscape Unit" VB #Sort All_Fires_23_24_merge_fields D:\1_GIS_work\9_00_CNC2024_1161_Fires\1_FINAL\All_fires_23_24_gw.shp "YEAR ASCENDING" URRepairGeometry All_fires_23_24_gw DELETE_NULLRepairGeometry All_fires_23_24_gw DELETE_NULLCalculateField All_fires_23_24_gw FIRE_CODE "DEHP/02/1992/01" VB #CalculateField All_fires_23_24_gw LOCAL_DESC "Witklip Infanta" VB #CalculateField All_fires_23_24_gw LOCAL_DESC "Buffelsfontein" VB #CalculateField All_fires_23_24_gw FIRE_CODE "DEHP/02/1992/02" VB #All_fires_23_24_gw0020.000file://\\CAPWKSLAP165\D$\1_GIS_work\9_00_CNC2024_1161_Fires\1_FINAL\All_fires_23_24_gw.shpLocal Area NetworkGeographicGCS_WGS_1984Angular Unit: Degree (0.017453)<GeographicCoordinateSystem xsi:type='typens:GeographicCoordinateSystem' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xmlns:xs='http://www.w3.org/2001/XMLSchema' xmlns:typens='http://www.esri.com/schemas/ArcGIS/10.8'><WKT>GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433],AUTHORITY["EPSG",4326]]</WKT><XOrigin>-400</XOrigin><YOrigin>-400</YOrigin><XYScale>11258999068426.238</XYScale><ZOrigin>-100000</ZOrigin><ZScale>10000</ZScale><MOrigin>-100000</MOrigin><MScale>10000</MScale><XYTolerance>8.983152841195215e-09</XYTolerance><ZTolerance>0.001</ZTolerance><MTolerance>0.001</MTolerance><HighPrecision>true</HighPrecision><LeftLongitude>-180</LeftLongitude><WKID>4326</WKID><LatestWKID>4326</LatestWKID></GeographicCoordinateSystem>20240805112217002024080511221700 Version 6.2 (Build 9200) ; Esri ArcGIS 10.8.1.14362All_fires_23_24_gwShapefile0.000datasetEPSG6.14(3.0.1)0SimpleFALSE0FALSEFALSEAll_fires_23_24_gwFeature Class0FIDFIDOID400Internal feature number.EsriSequential unique whole numbers that are automatically generated.ShapeShapeGeometry000Feature geometry.EsriCoordinates defining the features.IDIDDouble16160FIRE_CODEFIRE_CODEString1600FILE_NAMEFILE_NAMEString1600FIREWEBFIREWEBString1600RES_CODERES_CODEString500LAND_UNITLAND_UNITString5000MONTHMONTHInteger550YEARYEARInteger10100RES_CENTRERES_CENTREString25400RES_NAMERES_NAMEString10000LOCAL_DESCLOCAL_DESCString10000DATE_STARTDATE_STARTString1500DATE_EXTINDATE_EXTINString1500DATE_WITHDDATE_WITHDString1500REPORT_OFFREPORT_OFFString7500POLIC_CASEPOLIC_CASEString5000IGNITIONCAIGNITIONCAString5000Yr_mnthYr_mnthString1000Area_haArea_haDouble1900RESCODE_LURESCODE_LUString100020240805
diff --git a/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shx b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shx
new file mode 100644
index 00000000..6b52ca09
Binary files /dev/null and b/data/manual_download/All_fires_23_24_gw/All_fires_23_24_gw.shx differ
diff --git a/data/manual_download/README.md b/data/manual_download/README.md
new file mode 100644
index 00000000..32b307d4
--- /dev/null
+++ b/data/manual_download/README.md
@@ -0,0 +1,3 @@
+# Raw Data
+
+This folder contains files that cannot be programmatically downloaded (such as those from BGIS)
diff --git a/data/stac/catalog.json b/data/stac/catalog.json
new file mode 100644
index 00000000..b55f10b1
--- /dev/null
+++ b/data/stac/catalog.json
@@ -0,0 +1,32 @@
+{
+ "stac_version": "1.0.0",
+ "type": "Catalog",
+ "id": "emma",
+ "description": "EMMA Environmental Data Catalog - A curated collection of environmental datasets for the Eastern Mediterranean and Maghreb region.",
+ "links": [
+ {
+ "rel": "root",
+ "href": "catalog.json",
+ "type": "application/json",
+ "title": "EMMA Catalog"
+ },
+ {
+ "rel": "license",
+ "href": "https://creativecommons.org/licenses/by/4.0/",
+ "type": "text/html",
+ "title": "Creative Commons Attribution 4.0"
+ },
+ {
+ "rel": "about",
+ "href": "https://github.com/AdamWilsonLab/emma_envdata",
+ "type": "text/html",
+ "title": "EMMA Project Repository"
+ },
+ {
+ "rel": "child",
+ "href": "modis_vi/collection.json",
+ "type": "application/json",
+ "title": "modis_vi - Dynamic VI observations"
+ }
+ ]
+}
diff --git a/data/stac/modis_vi/collection.json b/data/stac/modis_vi/collection.json
new file mode 100644
index 00000000..c9c2de2e
--- /dev/null
+++ b/data/stac/modis_vi/collection.json
@@ -0,0 +1,88 @@
+{
+ "stac_version": "1.0.0",
+ "stac_extensions": "https://stac-extensions.github.io/scientific/v1.0.0/schema.json",
+ "type": "Collection",
+ "id": "modis_vi",
+ "description": "MODIS Enhanced Vegetation Index (EVI) observations from Terra and Aqua satellites. 500m resolution, 16-day composites. Data processed from AppEEARS.",
+ "license": "CC-BY-4.0",
+ "keywords": ["MODIS", "EVI", "vegetation", "Terra", "Aqua", "500m", "16-day"],
+ "extent": {
+ "spatial": {
+ "bbox": [
+ [-180, -90, 180, 90]
+ ]
+ },
+ "temporal": {
+ "interval": [
+ ["2026-02-01T00:00:00Z", "2026-02-01T23:59:59Z"]
+ ]
+ }
+ },
+ "links": [
+ {
+ "rel": "root",
+ "href": "../catalog.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "parent",
+ "href": "../catalog.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "license",
+ "href": "https://creativecommons.org/licenses/by/4.0/",
+ "type": "text/html"
+ },
+ {
+ "rel": "about",
+ "href": "https://lpdaac.usgs.gov/products/mod13a1v061/",
+ "title": "MOD13A1.061 Product Information",
+ "type": "text/html"
+ },
+ {
+ "rel": "item",
+ "href": "modis_vi_202602.json",
+ "type": "application/json",
+ "title": "MODIS VI 202602"
+ }
+ ],
+ "providers": [
+ {
+ "name": "USGS LP DAAC",
+ "description": "Data source for MOD13A1 and MYD13A1",
+ "roles": ["producer", "licensor"],
+ "url": "https://lpdaac.usgs.gov/"
+ },
+ {
+ "name": "NASA AppEEARS",
+ "description": "Data access and subsetting service",
+ "roles": "processor",
+ "url": "https://appeears.org/"
+ },
+ {
+ "name": "EMMA Lab",
+ "description": "Data processing and aggregation",
+ "roles": "processor",
+ "url": "https://adamwilsonlab.github.io/"
+ }
+ ],
+ "summaries": {
+ "sci_doi": "10.5067/MODIS/MOD13A1.061|10.5067/MODIS/MYD13A1.061",
+ "platforms": ["Terra", "Aqua"],
+ "instruments": "MODIS",
+ "gsd": [
+ 500
+ ],
+ "bands": [
+ {
+ "name": "EVI",
+ "description": "Enhanced Vegetation Index",
+ "data_type": "int32",
+ "scale": 0.01,
+ "offset": 0,
+ "nodata": -9999
+ }
+ ]
+ }
+}
diff --git a/data/stac/modis_vi/modis_vi_202601.json b/data/stac/modis_vi/modis_vi_202601.json
new file mode 100644
index 00000000..0d1c68f8
--- /dev/null
+++ b/data/stac/modis_vi/modis_vi_202601.json
@@ -0,0 +1,54 @@
+{
+ "stac_version": "1.0.0",
+ "stac_extensions": "https://stac-extensions.github.io/scientific/v1.0.0/schema.json",
+ "type": "Feature",
+ "id": "modis_vi_202601",
+ "description": "MODIS EVI observations for January 2026",
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [-180, -90],
+ [180, -90],
+ [180, 90],
+ [-180, 90],
+ [-180, -90]
+ ]
+ ]
+ },
+ "properties": {
+ "datetime": "2026-01-01T00:00:00Z",
+ "start_datetime": "2026-01-01T00:00:00Z",
+ "end_datetime": "2026-01-31T23:59:59Z",
+ "platforms": ["Terra", "Aqua"],
+ "instruments": "MODIS",
+ "gsd": 500,
+ "dataset": "modis_vi"
+ },
+ "links": [
+ {
+ "rel": "collection",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "root",
+ "href": "../catalog.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "parent",
+ "href": "collection.json",
+ "type": "application/json"
+ }
+ ],
+ "assets": {
+ "data": {
+ "href": "https://github.com/AdamWilsonLab/emma_envdata/releases/download/data_modis_vi_current/dynamic_modis_vi_202601.gz.parquet",
+ "title": "MODIS VI Parquet - 202601",
+ "description": "Enhanced Vegetation Index observations in parquet format",
+ "type": "application/octet-stream",
+ "roles": "data"
+ }
+ }
+}
diff --git a/data/stac/modis_vi/modis_vi_202602.json b/data/stac/modis_vi/modis_vi_202602.json
new file mode 100644
index 00000000..899a8638
--- /dev/null
+++ b/data/stac/modis_vi/modis_vi_202602.json
@@ -0,0 +1,54 @@
+{
+ "stac_version": "1.0.0",
+ "stac_extensions": "https://stac-extensions.github.io/scientific/v1.0.0/schema.json",
+ "type": "Feature",
+ "id": "modis_vi_202602",
+ "description": "MODIS EVI observations for February 2026",
+ "geometry": {
+ "type": "Polygon",
+ "coordinates": [
+ [
+ [-180, -90],
+ [180, -90],
+ [180, 90],
+ [-180, 90],
+ [-180, -90]
+ ]
+ ]
+ },
+ "properties": {
+ "datetime": "2026-02-01T00:00:00Z",
+ "start_datetime": "2026-02-01T00:00:00Z",
+ "end_datetime": "2026-02-28T23:59:59Z",
+ "platforms": ["Terra", "Aqua"],
+ "instruments": "MODIS",
+ "gsd": 500,
+ "dataset": "modis_vi"
+ },
+ "links": [
+ {
+ "rel": "collection",
+ "href": "collection.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "root",
+ "href": "../catalog.json",
+ "type": "application/json"
+ },
+ {
+ "rel": "parent",
+ "href": "collection.json",
+ "type": "application/json"
+ }
+ ],
+ "assets": {
+ "data": {
+ "href": "https://github.com/AdamWilsonLab/emma_envdata/releases/download/data_modis_vi_current/dynamic_modis_vi_202602.gz.parquet",
+ "title": "MODIS VI Parquet - 202602",
+ "description": "Enhanced Vegetation Index observations in parquet format",
+ "type": "application/octet-stream",
+ "roles": "data"
+ }
+ }
+}
diff --git a/decrypt_secret.sh b/decrypt_secret.sh
new file mode 100644
index 00000000..81deface
--- /dev/null
+++ b/decrypt_secret.sh
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# Decrypt the file
+
+mkdir -p $HOME/.config/earthengine/ndef/
+
+# --batch to prevent interactive command
+# --yes to assume "yes" for questions
+
+gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
+--output $HOME/.config/earthengine/ndef/20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner@gmail.com $HOME/emma_envdata/scratch_code/20061abcbc1c6ecf51bd9cf7e37350f6_bmaitner@gmail.com.gpg
+
+gpg --quiet --batch --yes --decrypt --passphrase="$RGEE_SECRET" \
+--output $HOME/.config/earthengine/ndef/credentials $HOME/emma_envdata/scratch_code/credentials.gpg
diff --git a/docs/APPEEARS_SETUP.md b/docs/APPEEARS_SETUP.md
new file mode 100644
index 00000000..74abefde
--- /dev/null
+++ b/docs/APPEEARS_SETUP.md
@@ -0,0 +1,110 @@
+# AppEEARS Setup Guide
+
+This project now uses NASA AppEEARS instead of Google Earth Engine for satellite data access.
+
+## Prerequisites
+
+1. **NASA EarthData Account**: Register at https://urs.earthdata.nasa.gov/
+2. **AppEEARS Access**: Approve AppEEARS application at https://appeears.earthdatacloud.nasa.gov/
+
+## Local Setup
+
+Set environment variables in your `.Renviron` file:
+
+```r
+# Edit .Renviron
+usethis::edit_r_environ()
+
+# Add these lines:
+EARTHDATA_USER=your_username
+EARTHDATA_PASSWORD=your_password
+```
+
+Restart R session for changes to take effect.
+
+## GitHub Actions Setup
+
+Add secrets to your repository:
+
+1. Go to repository Settings β Secrets and variables β Actions
+2. Add two secrets:
+ - `EARTHDATA_USER`: Your NASA EarthData username
+ - `EARTHDATA_PASSWORD`: Your NASA EarthData password
+
+## Verify Setup
+
+Test authentication in R:
+
+```r
+library(appeears)
+rs_login(
+ user = Sys.getenv("EARTHDATA_USER"),
+ password = Sys.getenv("EARTHDATA_PASSWORD")
+)
+
+# List available products
+rs_products()
+```
+
+## Migration Notes
+
+### Changes from rgee to AppEEARS
+
+- **Authentication**: Simple username/password instead of service account JSON
+- **Output format**: Monthly NetCDF files instead of individual GeoTIFFs
+- **Storage efficiency**: ~20% reduction in file sizes
+- **Simpler CI/CD**: No Python/conda dependencies required
+
+### Product Mapping
+
+| Current (rgee) | AppEEARS Product | Status |
+|----------------|------------------|--------|
+| MODIS Fire (MCD64A1) | MCD64A1.061 | β
Migrated |
+| MODIS NDVI (MOD13A1) | MOD13A1.061 | β
Migrated |
+| VIIRS NDVI | VNP13A1.001 | π Pending |
+| KNDVI | Calculate from reflectances | π Pending |
+| CHELSA Climate | Direct download (unchanged) | β
No change |
+| NASADEM | NASADEM.001 | π Pending |
+
+### Release Tags
+
+NetCDF outputs use new tag names to distinguish from GeoTIFF:
+
+- `raw_fire_modis_nc` - Raw fire monthly NetCDF
+- `raw_ndvi_modis_nc` - Raw NDVI monthly NetCDF
+- etc.
+
+### File Naming Convention
+
+Monthly NetCDF files follow the pattern: `{product}_{collection}_{YYYY-MM}.nc`
+
+Examples:
+- `fire_MCD64A1_2025-12.nc`
+- `ndvi_MOD13A1_2025-12.nc`
+
+## Troubleshooting
+
+### Authentication Errors
+
+If you see "EARTHDATA credentials not found":
+1. Check environment variables are set: `Sys.getenv("EARTHDATA_USER")`
+2. Restart R session after setting `.Renviron`
+3. Verify credentials at https://urs.earthdata.nasa.gov/
+
+### AppEEARS Task Failures
+
+Check task status:
+```r
+appeears::rs_status(task_id = "your_task_id")
+```
+
+Common issues:
+- Date range too large (split into smaller requests)
+- Invalid area of interest (ensure valid WGS84 coordinates)
+- Service temporarily unavailable (retry after delay)
+
+## Resources
+
+- [AppEEARS Documentation](https://appeears.earthdatacloud.nasa.gov/help)
+- [appeears R package](https://docs.ropensci.org/appeears/)
+- [NASA EarthData](https://earthdata.nasa.gov/)
diff --git a/emma-targets.Rproj b/emma-targets.Rproj
index aad884da..699e3ae1 100644
--- a/emma-targets.Rproj
+++ b/emma-targets.Rproj
@@ -1,4 +1,5 @@
Version: 1.0
+ProjectId: d3bc4aff-88e6-458a-b2fc-6bc8e9e06f1c
RestoreWorkspace: No
SaveWorkspace: No
diff --git a/emma_envdata.code-workspace b/emma_envdata.code-workspace
new file mode 100644
index 00000000..876a1499
--- /dev/null
+++ b/emma_envdata.code-workspace
@@ -0,0 +1,8 @@
+{
+ "folders": [
+ {
+ "path": "."
+ }
+ ],
+ "settings": {}
+}
\ No newline at end of file
diff --git a/img/network.png b/img/network.png
new file mode 100644
index 00000000..a74d2d3f
Binary files /dev/null and b/img/network.png differ
diff --git a/R/clean_up.R b/old/clean_up.R
similarity index 100%
rename from R/clean_up.R
rename to old/clean_up.R
diff --git a/R/domain_remnants.R b/old/domain_remnants.R
similarity index 100%
rename from R/domain_remnants.R
rename to old/domain_remnants.R
diff --git a/R/domain_remnants_release.R b/old/domain_remnants_release.R
similarity index 100%
rename from R/domain_remnants_release.R
rename to old/domain_remnants_release.R
diff --git a/R/get_domain.R b/old/get_domain.R
similarity index 100%
rename from R/get_domain.R
rename to old/get_domain.R
diff --git a/old/get_modis_vi.R b/old/get_modis_vi.R
new file mode 100644
index 00000000..220598ec
--- /dev/null
+++ b/old/get_modis_vi.R
@@ -0,0 +1,624 @@
+
+#' @title Submit monthly MODIS VI request via AppEEARS
+#' @description Submits an AppEEARS area request for MOD13A1.061 and MYD13A1.061
+#' EVI, and QA (500m resolution, 16-day composite) for a monthly period.
+#' @author EMMA Team
+#' @param domain_vector A SpatVector or sf polygon defining the domain boundary
+#' @param month_start Start date for the month (YYYY-MM-DD)
+#' @param month_end End date for the month (YYYY-MM-DD)
+#' @param verbose Logical for progress messages
+#' @return Character string with AppEEARS task ID
+#' @export
+submit_modis_vi <- function(
+ domain_vector,
+ month_start,
+ month_end,
+ verbose = TRUE
+) {
+
+ # Convert domain vector to sf, fix geometry, simplify, merge, and reproject to WGS84
+ domain_sf <- st_as_sf(domain_vector) %>%
+ st_simplify(dTolerance = 100, preserveTopology = TRUE) %>%
+ st_buffer(0) %>%
+ st_make_valid() %>%
+ st_transform(crs = 4326) %>%
+ geojsonsf::sf_geojson(simplify = FALSE) %>%
+ jsonlite::fromJSON()
+
+ # Validate dates
+ month_start <- as.Date(month_start)
+ month_end <- as.Date(month_end)
+
+ if (verbose) {
+ message("AppEEARS MODIS VI monthly request: ", format(month_start, "%Y-%m-%d"),
+ " to ", format(month_end, "%Y-%m-%d"))
+ }
+
+ # Resolve layer names dynamically (same as full-range version)
+ evi_layer <- "_500m_16_days_EVI"
+ qa_layer <- "_500m_16_days_VI_Quality"
+ date_layer <- "_500m_16_days_composite_day_of_the_year"
+
+ try({
+ lyr <- appeears::rs_layers("MOD13A1.061")
+ cand_cols <- intersect(c("Layer", "Name", "layer", "name"), names(lyr))
+ if (length(cand_cols)) {
+ vals <- unlist(lapply(cand_cols, function(cc) lyr[[cc]]))
+ evi_cand <- vals[grepl("EVI", vals, ignore.case = TRUE)][1]
+ qa_cand <- vals[grepl("VI.*Quality|Quality", vals, ignore.case = TRUE)][1]
+ date_cand <- vals[grepl("composite_day_of_the_year", vals, ignore.case = TRUE)][1]
+ if (!is.na(evi_cand)) evi_layer <- evi_cand
+ if (!is.na(qa_cand)) qa_layer <- qa_cand
+ if (!is.na(date_cand)) date_layer <- date_cand
+ }
+ }, silent = TRUE)
+
+ if (verbose) message("Using layers: ", evi_layer, ", ", qa_layer, ", ", date_layer)
+
+ # Build request payload for monthly period
+ req <- list(
+ task_type = "area",
+ task_name = paste0("MODIS_VI_", format(month_start, "%Y%m"), "_", format(Sys.time(), "%H%M%S")),
+ params = list(
+ dates = list(list(
+ startDate = format(month_start, "%m-%d-%Y"),
+ endDate = format(month_end, "%m-%d-%Y")
+ )),
+ layers = list(
+ # MOD13A1.061 (Terra)
+ list(product = "MOD13A1.061", layer = evi_layer),
+ list(product = "MOD13A1.061", layer = qa_layer),
+ list(product = "MOD13A1.061", layer = date_layer),
+ # MYD13A1.061 (Aqua)
+ list(product = "MYD13A1.061", layer = evi_layer),
+ list(product = "MYD13A1.061", layer = qa_layer),
+ list(product = "MYD13A1.061", layer = date_layer)
+ ),
+ output = list(
+ format = list(type = "netcdf4"),
+ projection = "native"
+ ),
+ geo = domain_sf
+ )
+ )
+
+ # Submit task
+ if (verbose) message("Submitting AppEEARS MODIS VI monthly task...")
+ task <- appeears::rs_request(
+ request = req,
+ user = Sys.getenv("EARTHDATA_USER"),
+ transfer = FALSE,
+ verbose = verbose
+ )
+
+ task_id <- task$get_task_id()
+ if (verbose) message("Task submitted with ID: ", task_id)
+
+ task_id
+}
+
+
+#' @title Download and process MODIS VI for a monthly period
+#' @description Polls for completion of AppEEARS task and downloads results,
+#' then processes into a NetCDF file with EVI and QA variables for that monthly period.
+#' @author EMMA Team
+#' @param task_id Character string with AppEEARS task ID
+#' @param domain_vector A SpatVector or sf polygon defining the domain boundary
+#' @param domain_raster A SpatRaster (domain.tif) defining the output grid and mask
+#' @param month_start Start date for monthly period (YYYY-MM-DD)
+#' @param out_dir Output directory for monthly NetCDF files
+#' @param temp_directory Temporary working directory for downloads
+#' @param verbose Logical for progress messages
+#' @return Character path to output NetCDF file (format: modis_vi_YYYYMM_monthly.nc)
+#' @details
+#' Implements AppEEARS polling with timeout protection and QA masking.
+#' Output is a single month of data; multiple outputs are aggregated separately.
+#' @export
+download_modis_vi <- function(
+ task_id,
+ domain_vector,
+ domain_raster,
+ month_start,
+ out_dir = "data/target_outputs/modis_vi",
+ temp_directory = "data/temp/raw_data/modis_vi_month/",
+ cleanup = TRUE,
+ verbose = TRUE
+) {
+
+ terra_tmp <- file.path(getwd(), "data/temp/terra")
+ month_start <- as.Date(month_start)
+ # Convert to YYYYMM format used in filenames
+ yyyymm <- format(month_start, "%Y%m")
+ file_name <- sprintf("modis_vi_%s_monthly.nc", yyyymm)
+
+ # Clean and create temp directory
+ unlink(temp_directory, recursive = TRUE, force = TRUE)
+ dir.create(temp_directory, recursive = TRUE, showWarnings = FALSE)
+
+ # Clean terra temp
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ dir.create(terra_tmp, recursive = TRUE, showWarnings = FALSE)
+ terraOptions(tempdir = terra_tmp, memfrac = 0.8)
+
+ # Poll for task completion
+ if (verbose) message("Polling task ", task_id, " for completion...")
+
+ max_retries <- 120 # 2 hours at 60s intervals
+ retry_count <- 0
+ task_status <- "pending"
+
+ repeat {
+ retry_count <- retry_count + 1
+
+ # Check task status
+ task_info <- appeears::rs_list_task(task_id = task_id, user = Sys.getenv("EARTHDATA_USER"))
+ task_status <- task_info$status
+
+ if (task_status == "done") {
+ if (verbose) message("Task completed successfully")
+ break
+ }
+
+ if (task_status %in% c("failed", "error")) {
+ stop("AppEEARS task ", task_id, " failed with status: ", task_status)
+ }
+
+ if (retry_count >= max_retries) {
+ stop("Task ", task_id, " polling timed out after ", max_retries, " minutes")
+ }
+
+ if (verbose && retry_count %% 10 == 0) {
+ message("Task status: ", task_status, " (", retry_count, "/", max_retries, ")")
+ }
+
+ Sys.sleep(60)
+ }
+
+ # Download results
+ if (verbose) message("Downloading files for task: ", task_id)
+ appeears::rs_transfer(
+ task_id = task_id,
+ user = Sys.getenv("EARTHDATA_USER"),
+ path = temp_directory,
+ verbose = verbose
+ )
+
+ # Load the NetCDF files
+ nc_paths <- list.files(temp_directory, pattern = "\\.nc$", full.names = TRUE, recursive = TRUE)
+ if (length(nc_paths) == 0) {
+ # No data available for this month (common for edge cases, polar regions, etc.)
+ if (verbose) message("No NetCDF files returned from AppEEARS for month ", file_name, " - writing skip marker")
+
+ # Create a lightweight skip marker file instead of fake data
+ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
+ skip_file <- file.path(out_dir, paste0("modis_vi_", yyyymm, "_monthly.skip"))
+
+ writeLines(
+ c(
+ paste("Month:", yyyymm),
+ paste("Task ID:", task_id),
+ paste("Reason: No NetCDF files returned from AppEEARS"),
+ paste("Note: Possible causes - polar region, cloud cover, instrument malfunction, or outside data availability period"),
+ paste("Timestamp:", Sys.time())
+ ),
+ skip_file
+ )
+
+ if (verbose) message("Created skip marker: ", skip_file)
+
+ # Cleanup
+ if(cleanup) {
+ unlink(temp_directory, recursive = TRUE, force = TRUE)
+ gc()
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ }
+
+ return(skip_file)
+ }
+
+ if (verbose) message("Reading ", length(nc_paths), " NetCDF files from AppEEARS")
+
+
+# Vargas et al.,15 pixels with any of the following QA flags were removed:
+# not confidently clear, adjacent to cloud, cloud shadow, snow or ice, thin cirrus cloud,
+# high aerosol loading, solar zenith angle >65 deg, and not over land.
+
+ # Apply QA mask to VI layers
+ qa_lookup <- list.files(
+ temp_directory,
+ pattern = "(VI-Quality-lookup).*\\.csv$",
+ full.names = TRUE,
+ recursive = TRUE
+ )
+ if (!length(qa_lookup)) {
+ stop("QA lookup table (VI_Quality*.csv) not found in temp_directory; cannot mask VI data")
+ }
+
+ extract_keep <- function(path) {
+ tab <- tryCatch(read.csv(path, stringsAsFactors = FALSE), error = function(e) NULL)
+ if (is.null(tab)) return(NULL)
+
+ # Find required columns (case-insensitive)
+ value_col <- names(tab)[grepl("^value$", tolower(names(tab)))][1]
+ modland_col <- names(tab)[grepl("modland", tolower(names(tab)))][1]
+ adj_cloud_col <- names(tab)[grepl("adjacent.*cloud", tolower(names(tab)))][1]
+ snow_col <- names(tab)[grepl("possible.*snow|snow.*ice", tolower(names(tab)))][1]
+ shadow_col <- names(tab)[grepl("possible.*shadow", tolower(names(tab)))][1]
+ land_col <- names(tab)[grepl("land/water|land.*water", tolower(names(tab)))][1]
+ aerosol_col <- names(tab)[grepl("aerosol", tolower(names(tab)))][1]
+
+ # Check all required columns are present
+ required_cols <- c(value_col, modland_col, adj_cloud_col, snow_col, shadow_col, land_col, aerosol_col)
+ if (any(is.na(required_cols))) {
+ warning("QA lookup missing required columns. Found: value=", !is.na(value_col),
+ ", modland=", !is.na(modland_col), ", adj_cloud=", !is.na(adj_cloud_col),
+ ", snow=", !is.na(snow_col), ", shadow=", !is.na(shadow_col),
+ ", land=", !is.na(land_col), ", aerosol=", !is.na(aerosol_col))
+ return(NULL)
+ }
+
+ # Filter for pixels that meet ALL QA criteria:
+ # 1. VI produced with good quality OR VI produced but check other QA (conservative approach)
+ # 2. No adjacent cloud detected
+ # 3. No cloud shadow (possible shadow = No)
+ # 4. No snow/ice (possible snow/ice = No)
+ # 5. Over land (not ocean or water)
+ # 6. Not high aerosol loading
+ keep <- (grepl("vi produced", tolower(tab[[modland_col]]))) &
+ (grepl("^no$", tolower(tab[[adj_cloud_col]]))) &
+ (grepl("^no$", tolower(tab[[shadow_col]]))) &
+ (grepl("^no$", tolower(tab[[snow_col]]))) &
+ (grepl("land", tolower(tab[[land_col]]))) &
+ (!grepl("high", tolower(tab[[aerosol_col]])))
+
+ tab[[value_col]][keep & !is.na(tab[[value_col]])]
+ }
+
+ keep_values <- unique(unlist(lapply(qa_lookup, extract_keep)))
+ if (!length(keep_values)) {
+ stop("No 'good quality' entries found in any QA table; refusing to proceed")
+ }
+
+ mask_vi_with_qa <- function(stack, domain_template, product_name = "terra") {
+ evi_idx <- which(grepl("EVI", names(stack), ignore.case = TRUE))[1] # Take first EVI layer
+ qa_idx <- which(grepl("Quality", names(stack), ignore.case = TRUE))[1] # Take first QA layer
+ date_idx <- which(grepl("composite_day_of_the_year", names(stack), ignore.case = TRUE))[1] # Take first date layer
+
+ if (is.na(evi_idx) || is.na(qa_idx) || is.na(date_idx)) {
+ message("Skipping file: missing EVI (", !is.na(evi_idx), "), QA (", !is.na(qa_idx), "), or date (", !is.na(date_idx), ") layer")
+ return(NULL)
+ }
+
+ qa_r <- stack[[qa_idx]]
+ keep_mask <- terra::app(qa_r, function(x) x %in% keep_values)
+
+ # Mask, project, and scale EVI
+ evi <- terra::mask(stack[[evi_idx]], keep_mask, maskvalue = FALSE) |>
+ terra::project(domain_template, method = "average") |>
+ terra::app(function(x) x * 100)
+
+ # Mask and project date (composite day of year)
+ date <- terra::mask(stack[[date_idx]], keep_mask, maskvalue = FALSE) |>
+ terra::project(domain_template, method = "mode")
+
+ # Name variables with product suffix
+ names(evi) <- paste0("evi_", product_name)
+ names(date) <- paste0("date_", product_name)
+
+ c(date, evi) # Return: date layer first, then EVI
+ }
+
+ # Project to domain grid
+ domain_template <- if (is.character(domain_raster)) terra::rast(domain_raster) else domain_raster
+
+ # Process all files, detecting product from file metadata and extracting time info
+ # AppEEARS downloads files grouped by product (MOD13A1 files, then MYD13A1 files)
+ all_layers <- list()
+ all_times <- numeric() # Store time values (days since epoch)
+ file_counter <- 0
+
+ for (nc_path in nc_paths) {
+ file_counter <- file_counter + 1
+ rast_obj <- terra::rast(nc_path)
+
+ # Detect product and extract time dimension
+ product_name <- "terra"
+ nc_times <- NULL
+
+ tryCatch({
+ nc <- ncdf4::nc_open(nc_path)
+ # Check global attributes for product name
+ global_attrs <- names(ncdf4::ncatt_get(nc, 0))
+
+ for (attr_name in global_attrs) {
+ attr_val <- ncdf4::ncatt_get(nc, 0, attr_name)$value
+ if (!is.null(attr_val) && is.character(attr_val)) {
+ if (grepl("MYD13", attr_val, ignore.case = TRUE)) {
+ product_name <- "aqua"
+ break
+ } else if (grepl("MOD13", attr_val, ignore.case = TRUE)) {
+ product_name <- "terra"
+ break
+ }
+ }
+ }
+
+ # Extract time values from NetCDF
+ if ("time" %in% names(nc$dim)) {
+ nc_times <- ncdf4::ncvar_get(nc, "time")
+ if (!is.null(nc_times) && length(nc_times) > 0) {
+ all_times <- c(all_times, nc_times)
+ }
+ }
+
+ ncdf4::nc_close(nc)
+ }, error = function(e) {
+ # If reading file fails, use file order heuristic
+ if (file_counter > 1) product_name <<- "aqua"
+ })
+
+ masked_layers <- mask_vi_with_qa(rast_obj, domain_template, product_name = product_name)
+ if (!is.null(masked_layers)) {
+ all_layers[[length(all_layers) + 1]] <- masked_layers
+ }
+ }
+
+ # Combine all layers into single stack
+ if (length(all_layers) == 0) {
+ stop("No valid EVI/date layers found after QA masking")
+ }
+
+ raster_stack <- do.call(c, all_layers)
+
+ # Mask to domain
+ mask_layer <- if ("domain" %in% names(domain_template)) domain_template[["domain"]] else domain_template
+ masked_stack <- terra::mask(raster_stack, mask_layer)
+
+ # Add pixel ID layer
+ if (!"pid" %in% names(domain_template)) {
+ stop("domain_raster must include a 'pid' layer")
+ }
+ pid_raster <- terra::mask(domain_template[["pid"]], mask_layer)
+ names(pid_raster) <- "pid"
+ output_stack <- c(masked_stack, pid_raster)
+
+ # Write to NetCDF using ncdf4 to ensure separate variables
+ dir.create(out_dir, recursive = TRUE, showWarnings = FALSE)
+ out_file <- file.path(out_dir, file_name)
+ unlink(out_file)
+
+ if (verbose) message("Writing NetCDF with daily time dimension and date-based observation placement: ", out_file)
+
+ # Get dimensions and coordinates from output_stack
+ nrow <- nrow(output_stack)
+ ncol <- ncol(output_stack)
+ crs <- terra::crs(output_stack)
+
+ # Get actual x and y coordinates from the raster itself
+ # These are the coordinates of the cell centers in row/column order
+ x_coords <- terra::xFromCol(output_stack, col = 1:ncol)
+ y_coords <- terra::yFromRow(output_stack, row = 1:nrow)
+
+ # Create daily time dimension for entire month
+ # Calculate the number of days in the month by finding last day of month
+ year_int <- as.integer(format(month_start, "%Y"))
+ month_int <- as.integer(format(month_start, "%m"))
+ if (month_int < 12) {
+ next_month_first <- as.Date(paste0(year_int, "-", sprintf("%02d", month_int + 1), "-01"))
+ } else {
+ next_month_first <- as.Date(paste0(year_int + 1, "-01-01"))
+ }
+ last_day_of_month <- next_month_first - 1
+ days_in_month <- as.integer(format(last_day_of_month, "%d"))
+ time_coords <- seq(0, days_in_month - 1, by = 1)
+ time_units <- paste0("days since ", format(month_start, "%Y-%m-%d"))
+
+ if (verbose) message("Using daily time dimension with ", days_in_month, " days")
+
+ # Note: terra raster y coordinates are top-to-bottom; NetCDF expects bottom-to-top
+ # We will handle the flip during matrix write, coordinates stay in raster order
+
+ # Create dimensions with actual coordinates (using UDUNITS-compliant units)
+ x_dim <- ncdf4::ncdim_def("x", "m", vals = x_coords)
+ y_dim <- ncdf4::ncdim_def("y", "m", vals = y_coords)
+ time_dim <- ncdf4::ncdim_def("time", time_units, vals = time_coords, unlim = TRUE)
+
+ # Create variables for each unique variable name
+ var_list <- list()
+ unique_var_names <- unique(names(output_stack))
+
+ for (var_name in unique_var_names) {
+ if (var_name == "pid") {
+ var_list[[var_name]] <- ncdf4::ncvar_def(
+ name = var_name,
+ units = "1",
+ dim = list(x_dim, y_dim),
+ missval = NA_real_,
+ compression = 9
+ )
+ } else if (var_name %in% c("date_terra", "date_aqua", "evi_terra", "evi_aqua")) {
+ var_list[[var_name]] <- ncdf4::ncvar_def(
+ name = var_name,
+ units = if (grepl("evi", var_name, ignore.case = TRUE)) "1" else "1",
+ dim = list(time_dim, x_dim, y_dim),
+ missval = NA_real_,
+ compression = 9
+ )
+ }
+ }
+
+ # Create NetCDF file
+ nc <- ncdf4::nc_create(out_file, var_list)
+
+ # Function to convert day-of-year to day-of-month for the given month
+ doy_to_dom <- function(doy, year_val, month_val) {
+ if (is.na(doy) || !is.finite(doy)) return(NA_integer_)
+ # Create date from day-of-year
+ date <- as.Date(paste0(year_val, "-01-01")) + (as.integer(doy) - 1)
+ # Check if it falls in the target month
+ if (lubridate::month(date) == month_val) {
+ return(lubridate::mday(date))
+ } else {
+ return(NA_integer_)
+ }
+ }
+
+ # Get indices for date layers
+ date_terra_idx <- which(names(output_stack) == "date_terra")
+ date_aqua_idx <- which(names(output_stack) == "date_aqua")
+ evi_terra_idx <- which(names(output_stack) == "evi_terra")
+ evi_aqua_idx <- which(names(output_stack) == "evi_aqua")
+
+ # Write each variable's data using date fields for temporal placement
+ for (var_name in unique_var_names) {
+ if (var_name == "pid") {
+ # pid is constant across time
+ matching_indices <- which(names(output_stack) == var_name)
+ layer_data <- terra::as.matrix(output_stack[[matching_indices[1]]], wide = TRUE)
+ layer_data <- layer_data[nrow(layer_data):1, ]
+ layer_data <- t(layer_data)
+ ncdf4::ncvar_put(nc, var_name, layer_data)
+ } else if (var_name %in% c("date_terra", "date_aqua", "evi_terra", "evi_aqua")) {
+ # Time-varying variables: place observations using date fields
+ daily_array <- array(NA_real_, dim = c(ncol, nrow, days_in_month))
+
+ # Determine which date field and observation index to use
+ if (var_name %in% c("evi_terra", "date_terra")) {
+ date_idx <- date_terra_idx
+ obs_idx <- if (var_name == "evi_terra") evi_terra_idx else date_terra_idx
+ } else {
+ date_idx <- date_aqua_idx
+ obs_idx <- if (var_name == "evi_aqua") evi_aqua_idx else date_aqua_idx
+ }
+
+ # For each observation, place it at the correct daily time slot based on date field
+ if (length(obs_idx) > 0 && length(date_idx) > 0) {
+ for (obs_seq in seq_along(obs_idx)) {
+ if (obs_seq <= length(date_idx)) {
+ obs_layer <- terra::as.matrix(output_stack[[obs_idx[obs_seq]]], wide = TRUE)
+ date_layer <- terra::as.matrix(output_stack[[date_idx[obs_seq]]], wide = TRUE)
+
+ # Reverse rows and transpose to match NetCDF order
+ obs_layer <- obs_layer[nrow(obs_layer):1, ]
+ obs_layer <- t(obs_layer)
+ date_layer <- date_layer[nrow(date_layer):1, ]
+ date_layer <- t(date_layer)
+
+ # Convert day-of-year to day-of-month (vectorized apply to matrix)
+ dom_matrix <- apply(date_layer, c(1, 2), function(x) {
+ doy_to_dom(x, lubridate::year(month_start), lubridate::month(month_start))
+ })
+
+ # Place observations at correct day-of-month indices
+ for (x in 1:ncol) {
+ for (y in 1:nrow) {
+ if (!is.na(obs_layer[x, y]) && !is.na(dom_matrix[x, y])) {
+ dom_val <- as.integer(dom_matrix[x, y])
+ if (dom_val >= 1 && dom_val <= days_in_month) {
+ daily_array[x, y, dom_val] <- obs_layer[x, y]
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ ncdf4::ncvar_put(nc, var_name, daily_array)
+ }
+ }
+
+ # Close file temporarily to add grid_mapping variable properly
+ ncdf4::nc_close(nc)
+
+ # Reopen file for writing and add grid_mapping variable
+ nc <- ncdf4::nc_open(out_file, write = TRUE)
+
+ # Add global attributes
+ ncdf4::ncatt_put(nc, 0, "title", paste0("MODIS Terra/Aqua EVI for month ", yyyymm, " resampled to domain"))
+ ncdf4::ncatt_put(nc, 0, "source", "MOD13A1.061 (Terra) and MYD13A1.061 (Aqua) via AppEEARS")
+ ncdf4::ncatt_put(nc, 0, "temporal_resolution", "Daily time dimension; observations placed according to composite day-of-year (16-day composites)")
+ ncdf4::ncatt_put(nc, 0, "time_structure", "Irregular: each pixel has observations on days corresponding to date_terra and date_aqua values; missing days are NA")
+ ncdf4::ncatt_put(nc, 0, "spatial_resolution", "500m native, resampled to domain grid")
+ ncdf4::ncatt_put(nc, 0, "month", yyyymm)
+ ncdf4::ncatt_put(nc, 0, "history", paste0("created: ", Sys.time()))
+ ncdf4::ncatt_put(nc, 0, "Conventions", "CF-1.8")
+ ncdf4::ncatt_put(nc, 0, "note_evi_scaling", "EVI values are scaled by 100 for storage. Divide by 100 to recover original values.")
+ ncdf4::ncatt_put(nc, 0, "scale_factor_evi", 0.01)
+ ncdf4::ncatt_put(nc, 0, "qa_filter", "Pixels retained: VI produced with good/checkable quality, adjacent cloud=No, cloud shadow=No, snow/ice=No, over land, aerosol loading not high. Excludes: not confidently clear, adjacent to cloud, cloud shadow, snow or ice, high aerosol loading, not over land. Note: Solar zenith angle >65 deg filter not implemented in current QA table version.")
+
+ # Get CRS information per CF-1.8 grid_mapping specification
+ crs_str <- as.character(crs)
+
+ if (!is.na(crs)) {
+ # Get WKT in latest format (OGC WKT 2)
+ crs_wkt <- terra::crs(output_stack, proj = TRUE)
+
+ if (!is.null(crs_wkt) && nchar(crs_wkt) > 0) {
+ # Per CF-1.8 Section 5.6.1, store CRS WKT following OGC WKT-CRS standard
+ # This attribute is recognized by GDAL, QGIS, and other geospatial tools
+ ncdf4::ncatt_put(nc, 0, "crs_wkt", crs_wkt)
+
+ # Extract and store EPSG code if available (for quick reference and validation)
+ epsg <- tryCatch(as.numeric(gsub("EPSG:", "", crs_str)), error = function(e) NA)
+ if (!is.na(epsg)) {
+ ncdf4::ncatt_put(nc, 0, "crs_epsg", as.integer(epsg))
+ }
+ }
+ }
+
+ # Add coordinate variable attributes per CF-1.8
+ # Coordinate variable attributes per CF-1.8 Section 4.1-4.4 and 5.6
+ ncdf4::ncatt_put(nc, "time", "axis", "T")
+ ncdf4::ncatt_put(nc, "time", "long_name", "Time of composite")
+ ncdf4::ncatt_put(nc, "time", "standard_name", "time")
+
+ ncdf4::ncatt_put(nc, "x", "axis", "X")
+ ncdf4::ncatt_put(nc, "x", "long_name", "X coordinate of projection")
+ ncdf4::ncatt_put(nc, "x", "units", "m")
+ ncdf4::ncatt_put(nc, "x", "standard_name", "projection_x_coordinate")
+
+ ncdf4::ncatt_put(nc, "y", "axis", "Y")
+ ncdf4::ncatt_put(nc, "y", "long_name", "Y coordinate of projection")
+ ncdf4::ncatt_put(nc, "y", "units", "m")
+ ncdf4::ncatt_put(nc, "y", "standard_name", "projection_y_coordinate")
+
+ # Add metadata to each data variable
+ for (var_name in unique_var_names) {
+ if (var_name != "pid") {
+ ncdf4::ncatt_put(nc, var_name, "coordinates", "time x y")
+ }
+
+ # Add units and descriptive names per CF-1.8 Section 3.1 and Appendix A
+ if (grepl("evi", var_name, ignore.case = TRUE)) {
+ ncdf4::ncatt_put(nc, var_name, "units", "1") # Dimensionless
+ ncdf4::ncatt_put(nc, var_name, "long_name", paste0("Enhanced Vegetation Index (scaled by 100) - ", toupper(sub("evi_", "", var_name))))
+ ncdf4::ncatt_put(nc, var_name, "scale_factor", 0.01) # Divide by 100 to recover original
+ ncdf4::ncatt_put(nc, var_name, "valid_min", 0)
+ ncdf4::ncatt_put(nc, var_name, "valid_max", 10000)
+ } else if (grepl("date", var_name, ignore.case = TRUE)) {
+ ncdf4::ncatt_put(nc, var_name, "units", "day") # Day of year (1-366), not absolute time
+ ncdf4::ncatt_put(nc, var_name, "long_name", paste0("Composite day of year - ", toupper(sub("date_", "", var_name))))
+ ncdf4::ncatt_put(nc, var_name, "valid_min", 1)
+ ncdf4::ncatt_put(nc, var_name, "valid_max", 366)
+ }
+ }
+
+ # Add units to pid variable
+ ncdf4::ncatt_put(nc, "pid", "units", "1")
+ ncdf4::ncatt_put(nc, "pid", "long_name", "Pixel ID")
+ ncdf4::ncatt_put(nc, "pid", "coordinates", "x y")
+
+ # Close the NetCDF file
+ ncdf4::nc_close(nc)
+
+ # Cleanup
+ if(cleanup) {
+ unlink(temp_directory, recursive = TRUE, force = TRUE)
+ gc()
+ unlink(terra_tmp, recursive = TRUE, force = TRUE)
+ }
+
+ if (verbose) message("MODIS VI monthly data saved to: ", out_file)
+ out_file
+}
diff --git a/R/get_release_elevation_nasadem.R b/old/get_release_elevation_nasadem.R
similarity index 100%
rename from R/get_release_elevation_nasadem.R
rename to old/get_release_elevation_nasadem.R
diff --git a/R/get_release_fire_modis.R b/old/get_release_fire_modis.R
similarity index 100%
rename from R/get_release_fire_modis.R
rename to old/get_release_fire_modis.R
diff --git a/R/get_release_mean_ndvi_modis.R b/old/get_release_mean_ndvi_modis.R
similarity index 100%
rename from R/get_release_mean_ndvi_modis.R
rename to old/get_release_mean_ndvi_modis.R
diff --git a/R/get_release_ndvi_dates_modis.R b/old/get_release_ndvi_dates_modis.R
similarity index 100%
rename from R/get_release_ndvi_dates_modis.R
rename to old/get_release_ndvi_dates_modis.R
diff --git a/R/get_release_ndvi_dates_viirs.R b/old/get_release_ndvi_dates_viirs.R
similarity index 100%
rename from R/get_release_ndvi_dates_viirs.R
rename to old/get_release_ndvi_dates_viirs.R
diff --git a/R/get_release_ndvi_modis.R b/old/get_release_ndvi_modis.R
similarity index 100%
rename from R/get_release_ndvi_modis.R
rename to old/get_release_ndvi_modis.R
diff --git a/R/get_release_ndvi_viirs.R b/old/get_release_ndvi_viirs.R
similarity index 100%
rename from R/get_release_ndvi_viirs.R
rename to old/get_release_ndvi_viirs.R
diff --git a/R/get_release_ndwi_modis.R b/old/get_release_ndwi_modis.R
similarity index 100%
rename from R/get_release_ndwi_modis.R
rename to old/get_release_ndwi_modis.R
diff --git a/old/hexgrid.R b/old/hexgrid.R
new file mode 100644
index 00000000..582b16e8
--- /dev/null
+++ b/old/hexgrid.R
@@ -0,0 +1,13 @@
+hexgrid <- st_make_grid(
+ domain,
+ cellsize = c(500,500),
+ offset = st_bbox(domain)[c("xmin", "ymin")],
+ crs = st_crs(domain),
+ what = "polygons",
+ square = FALSE,
+ flat_topped = FALSE
+ ) |>
+ st_as_sf() |>
+ st_intersection(domain) |> # keep all hexagons that intersect domain, not just those with centroids in domain
+ mutate(pid = row_number())
+
diff --git a/qgis/emma.qgz b/qgis/emma.qgz
new file mode 100644
index 00000000..d9955518
Binary files /dev/null and b/qgis/emma.qgz differ
diff --git a/renv.lock b/renv.lock
deleted file mode 100644
index 7d3133c1..00000000
--- a/renv.lock
+++ /dev/null
@@ -1,76 +0,0 @@
-{
- "R": {
- "Version": "4.1.2",
- "Repositories": [
- {
- "Name": "CRAN",
- "URL": "https://cloud.r-project.org"
- }
- ]
- },
- "Packages": {
- "KernSmooth": {
- "Package": "KernSmooth",
- "Version": "2.23-20",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "8dcfa99b14c296bc9f1fd64d52fd3ce7"
- },
- "MASS": {
- "Package": "MASS",
- "Version": "7.3-54",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "0e59129db205112e3963904db67fd0dc"
- },
- "Matrix": {
- "Package": "Matrix",
- "Version": "1.3-4",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "4ed05e9c9726267e4a5872e09c04587c"
- },
- "class": {
- "Package": "class",
- "Version": "7.3-19",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "1593b7beb067c8381c0d24e38bd778e0"
- },
- "codetools": {
- "Package": "codetools",
- "Version": "0.2-18",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "019388fc48e48b3da0d3a76ff94608a8"
- },
- "lattice": {
- "Package": "lattice",
- "Version": "0.20-45",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "b64cdbb2b340437c4ee047a1f4c4377b"
- },
- "mgcv": {
- "Package": "mgcv",
- "Version": "1.8-38",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "be3c61ffbb1e3d3b3df214d192ac5444"
- },
- "nlme": {
- "Package": "nlme",
- "Version": "3.1-153",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "2d632e0d963a653a0329756ce701ecdd"
- },
- "renv": {
- "Package": "renv",
- "Version": "0.14.0",
- "Source": "Repository",
- "Repository": "CRAN",
- "Hash": "30e5eba91b67f7f4d75d31de14bbfbdc"
- }
- }
-}
diff --git a/renv/.gitignore b/renv/.gitignore
deleted file mode 100644
index 21296311..00000000
--- a/renv/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-library/
-local/
-lock/
-python/
-staging/
diff --git a/renv/activate.R b/renv/activate.R
deleted file mode 100644
index 304fd900..00000000
--- a/renv/activate.R
+++ /dev/null
@@ -1,668 +0,0 @@
-
-local({
-
- # the requested version of renv
- version <- "0.14.0"
-
- # the project directory
- project <- getwd()
-
- # allow environment variable to control activation
- activate <- Sys.getenv("RENV_ACTIVATE_PROJECT")
- if (!nzchar(activate)) {
-
- # don't auto-activate when R CMD INSTALL is running
- if (nzchar(Sys.getenv("R_INSTALL_PKG")))
- return(FALSE)
-
- }
-
- # bail if activation was explicitly disabled
- if (tolower(activate) %in% c("false", "f", "0"))
- return(FALSE)
-
- # avoid recursion
- if (nzchar(Sys.getenv("RENV_R_INITIALIZING")))
- return(invisible(TRUE))
-
- # signal that we're loading renv during R startup
- Sys.setenv("RENV_R_INITIALIZING" = "true")
- on.exit(Sys.unsetenv("RENV_R_INITIALIZING"), add = TRUE)
-
- # signal that we've consented to use renv
- options(renv.consent = TRUE)
-
- # load the 'utils' package eagerly -- this ensures that renv shims, which
- # mask 'utils' packages, will come first on the search path
- library(utils, lib.loc = .Library)
-
- # check to see if renv has already been loaded
- if ("renv" %in% loadedNamespaces()) {
-
- # if renv has already been loaded, and it's the requested version of renv,
- # nothing to do
- spec <- .getNamespaceInfo(.getNamespace("renv"), "spec")
- if (identical(spec[["version"]], version))
- return(invisible(TRUE))
-
- # otherwise, unload and attempt to load the correct version of renv
- unloadNamespace("renv")
-
- }
-
- # load bootstrap tools
- bootstrap <- function(version, library) {
-
- # attempt to download renv
- tarball <- tryCatch(renv_bootstrap_download(version), error = identity)
- if (inherits(tarball, "error"))
- stop("failed to download renv ", version)
-
- # now attempt to install
- status <- tryCatch(renv_bootstrap_install(version, tarball, library), error = identity)
- if (inherits(status, "error"))
- stop("failed to install renv ", version)
-
- }
-
- renv_bootstrap_tests_running <- function() {
- getOption("renv.tests.running", default = FALSE)
- }
-
- renv_bootstrap_repos <- function() {
-
- # check for repos override
- repos <- Sys.getenv("RENV_CONFIG_REPOS_OVERRIDE", unset = NA)
- if (!is.na(repos))
- return(repos)
-
- # if we're testing, re-use the test repositories
- if (renv_bootstrap_tests_running())
- return(getOption("renv.tests.repos"))
-
- # retrieve current repos
- repos <- getOption("repos")
-
- # ensure @CRAN@ entries are resolved
- repos[repos == "@CRAN@"] <- getOption(
- "renv.repos.cran",
- "https://cloud.r-project.org"
- )
-
- # add in renv.bootstrap.repos if set
- default <- c(FALLBACK = "https://cloud.r-project.org")
- extra <- getOption("renv.bootstrap.repos", default = default)
- repos <- c(repos, extra)
-
- # remove duplicates that might've snuck in
- dupes <- duplicated(repos) | duplicated(names(repos))
- repos[!dupes]
-
- }
-
- renv_bootstrap_download <- function(version) {
-
- # if the renv version number has 4 components, assume it must
- # be retrieved via github
- nv <- numeric_version(version)
- components <- unclass(nv)[[1]]
-
- methods <- if (length(components) == 4L) {
- list(
- renv_bootstrap_download_github
- )
- } else {
- list(
- renv_bootstrap_download_cran_latest,
- renv_bootstrap_download_cran_archive
- )
- }
-
- for (method in methods) {
- path <- tryCatch(method(version), error = identity)
- if (is.character(path) && file.exists(path))
- return(path)
- }
-
- stop("failed to download renv ", version)
-
- }
-
- renv_bootstrap_download_impl <- function(url, destfile) {
-
- mode <- "wb"
-
- # https://bugs.r-project.org/bugzilla/show_bug.cgi?id=17715
- fixup <-
- Sys.info()[["sysname"]] == "Windows" &&
- substring(url, 1L, 5L) == "file:"
-
- if (fixup)
- mode <- "w+b"
-
- utils::download.file(
- url = url,
- destfile = destfile,
- mode = mode,
- quiet = TRUE
- )
-
- }
-
- renv_bootstrap_download_cran_latest <- function(version) {
-
- spec <- renv_bootstrap_download_cran_latest_find(version)
-
- message("* Downloading renv ", version, " ... ", appendLF = FALSE)
-
- type <- spec$type
- repos <- spec$repos
-
- info <- tryCatch(
- utils::download.packages(
- pkgs = "renv",
- destdir = tempdir(),
- repos = repos,
- type = type,
- quiet = TRUE
- ),
- condition = identity
- )
-
- if (inherits(info, "condition")) {
- message("FAILED")
- return(FALSE)
- }
-
- # report success and return
- message("OK (downloaded ", type, ")")
- info[1, 2]
-
- }
-
- renv_bootstrap_download_cran_latest_find <- function(version) {
-
- # check whether binaries are supported on this system
- binary <-
- getOption("renv.bootstrap.binary", default = TRUE) &&
- !identical(.Platform$pkgType, "source") &&
- !identical(getOption("pkgType"), "source") &&
- Sys.info()[["sysname"]] %in% c("Darwin", "Windows")
-
- types <- c(if (binary) "binary", "source")
-
- # iterate over types + repositories
- for (type in types) {
- for (repos in renv_bootstrap_repos()) {
-
- # retrieve package database
- db <- tryCatch(
- as.data.frame(
- utils::available.packages(type = type, repos = repos),
- stringsAsFactors = FALSE
- ),
- error = identity
- )
-
- if (inherits(db, "error"))
- next
-
- # check for compatible entry
- entry <- db[db$Package %in% "renv" & db$Version %in% version, ]
- if (nrow(entry) == 0)
- next
-
- # found it; return spec to caller
- spec <- list(entry = entry, type = type, repos = repos)
- return(spec)
-
- }
- }
-
- # if we got here, we failed to find renv
- fmt <- "renv %s is not available from your declared package repositories"
- stop(sprintf(fmt, version))
-
- }
-
- renv_bootstrap_download_cran_archive <- function(version) {
-
- name <- sprintf("renv_%s.tar.gz", version)
- repos <- renv_bootstrap_repos()
- urls <- file.path(repos, "src/contrib/Archive/renv", name)
- destfile <- file.path(tempdir(), name)
-
- message("* Downloading renv ", version, " ... ", appendLF = FALSE)
-
- for (url in urls) {
-
- status <- tryCatch(
- renv_bootstrap_download_impl(url, destfile),
- condition = identity
- )
-
- if (identical(status, 0L)) {
- message("OK")
- return(destfile)
- }
-
- }
-
- message("FAILED")
- return(FALSE)
-
- }
-
- renv_bootstrap_download_github <- function(version) {
-
- enabled <- Sys.getenv("RENV_BOOTSTRAP_FROM_GITHUB", unset = "TRUE")
- if (!identical(enabled, "TRUE"))
- return(FALSE)
-
- # prepare download options
- pat <- Sys.getenv("GITHUB_PAT")
- if (nzchar(Sys.which("curl")) && nzchar(pat)) {
- fmt <- "--location --fail --header \"Authorization: token %s\""
- extra <- sprintf(fmt, pat)
- saved <- options("download.file.method", "download.file.extra")
- options(download.file.method = "curl", download.file.extra = extra)
- on.exit(do.call(base::options, saved), add = TRUE)
- } else if (nzchar(Sys.which("wget")) && nzchar(pat)) {
- fmt <- "--header=\"Authorization: token %s\""
- extra <- sprintf(fmt, pat)
- saved <- options("download.file.method", "download.file.extra")
- options(download.file.method = "wget", download.file.extra = extra)
- on.exit(do.call(base::options, saved), add = TRUE)
- }
-
- message("* Downloading renv ", version, " from GitHub ... ", appendLF = FALSE)
-
- url <- file.path("https://api.github.com/repos/rstudio/renv/tarball", version)
- name <- sprintf("renv_%s.tar.gz", version)
- destfile <- file.path(tempdir(), name)
-
- status <- tryCatch(
- renv_bootstrap_download_impl(url, destfile),
- condition = identity
- )
-
- if (!identical(status, 0L)) {
- message("FAILED")
- return(FALSE)
- }
-
- message("OK")
- return(destfile)
-
- }
-
- renv_bootstrap_install <- function(version, tarball, library) {
-
- # attempt to install it into project library
- message("* Installing renv ", version, " ... ", appendLF = FALSE)
- dir.create(library, showWarnings = FALSE, recursive = TRUE)
-
- # invoke using system2 so we can capture and report output
- bin <- R.home("bin")
- exe <- if (Sys.info()[["sysname"]] == "Windows") "R.exe" else "R"
- r <- file.path(bin, exe)
- args <- c("--vanilla", "CMD", "INSTALL", "-l", shQuote(library), shQuote(tarball))
- output <- system2(r, args, stdout = TRUE, stderr = TRUE)
- message("Done!")
-
- # check for successful install
- status <- attr(output, "status")
- if (is.numeric(status) && !identical(status, 0L)) {
- header <- "Error installing renv:"
- lines <- paste(rep.int("=", nchar(header)), collapse = "")
- text <- c(header, lines, output)
- writeLines(text, con = stderr())
- }
-
- status
-
- }
-
- renv_bootstrap_platform_prefix <- function() {
-
- # construct version prefix
- version <- paste(R.version$major, R.version$minor, sep = ".")
- prefix <- paste("R", numeric_version(version)[1, 1:2], sep = "-")
-
- # include SVN revision for development versions of R
- # (to avoid sharing platform-specific artefacts with released versions of R)
- devel <-
- identical(R.version[["status"]], "Under development (unstable)") ||
- identical(R.version[["nickname"]], "Unsuffered Consequences")
-
- if (devel)
- prefix <- paste(prefix, R.version[["svn rev"]], sep = "-r")
-
- # build list of path components
- components <- c(prefix, R.version$platform)
-
- # include prefix if provided by user
- prefix <- renv_bootstrap_platform_prefix_impl()
- if (!is.na(prefix) && nzchar(prefix))
- components <- c(prefix, components)
-
- # build prefix
- paste(components, collapse = "/")
-
- }
-
- renv_bootstrap_platform_prefix_impl <- function() {
-
- # if an explicit prefix has been supplied, use it
- prefix <- Sys.getenv("RENV_PATHS_PREFIX", unset = NA)
- if (!is.na(prefix))
- return(prefix)
-
- # if the user has requested an automatic prefix, generate it
- auto <- Sys.getenv("RENV_PATHS_PREFIX_AUTO", unset = NA)
- if (auto %in% c("TRUE", "True", "true", "1"))
- return(renv_bootstrap_platform_prefix_auto())
-
- # empty string on failure
- ""
-
- }
-
- renv_bootstrap_platform_prefix_auto <- function() {
-
- prefix <- tryCatch(renv_bootstrap_platform_os(), error = identity)
- if (inherits(prefix, "error") || prefix %in% "unknown") {
-
- msg <- paste(
- "failed to infer current operating system",
- "please file a bug report at https://github.com/rstudio/renv/issues",
- sep = "; "
- )
-
- warning(msg)
-
- }
-
- prefix
-
- }
-
- renv_bootstrap_platform_os <- function() {
-
- sysinfo <- Sys.info()
- sysname <- sysinfo[["sysname"]]
-
- # handle Windows + macOS up front
- if (sysname == "Windows")
- return("windows")
- else if (sysname == "Darwin")
- return("macos")
-
- # check for os-release files
- for (file in c("/etc/os-release", "/usr/lib/os-release"))
- if (file.exists(file))
- return(renv_bootstrap_platform_os_via_os_release(file, sysinfo))
-
- # check for redhat-release files
- if (file.exists("/etc/redhat-release"))
- return(renv_bootstrap_platform_os_via_redhat_release())
-
- "unknown"
-
- }
-
- renv_bootstrap_platform_os_via_os_release <- function(file, sysinfo) {
-
- # read /etc/os-release
- release <- utils::read.table(
- file = file,
- sep = "=",
- quote = c("\"", "'"),
- col.names = c("Key", "Value"),
- comment.char = "#",
- stringsAsFactors = FALSE
- )
-
- vars <- as.list(release$Value)
- names(vars) <- release$Key
-
- # get os name
- os <- tolower(sysinfo[["sysname"]])
-
- # read id
- id <- "unknown"
- for (field in c("ID", "ID_LIKE")) {
- if (field %in% names(vars) && nzchar(vars[[field]])) {
- id <- vars[[field]]
- break
- }
- }
-
- # read version
- version <- "unknown"
- for (field in c("UBUNTU_CODENAME", "VERSION_CODENAME", "VERSION_ID", "BUILD_ID")) {
- if (field %in% names(vars) && nzchar(vars[[field]])) {
- version <- vars[[field]]
- break
- }
- }
-
- # join together
- paste(c(os, id, version), collapse = "-")
-
- }
-
- renv_bootstrap_platform_os_via_redhat_release <- function() {
-
- # read /etc/redhat-release
- contents <- readLines("/etc/redhat-release", warn = FALSE)
-
- # infer id
- id <- if (grepl("centos", contents, ignore.case = TRUE))
- "centos"
- else if (grepl("redhat", contents, ignore.case = TRUE))
- "redhat"
- else
- "unknown"
-
- # try to find a version component (very hacky)
- version <- "unknown"
-
- parts <- strsplit(contents, "[[:space:]]")[[1L]]
- for (part in parts) {
-
- nv <- tryCatch(numeric_version(part), error = identity)
- if (inherits(nv, "error"))
- next
-
- version <- nv[1, 1]
- break
-
- }
-
- paste(c("linux", id, version), collapse = "-")
-
- }
-
- renv_bootstrap_library_root_name <- function(project) {
-
- # use project name as-is if requested
- asis <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT_ASIS", unset = "FALSE")
- if (asis)
- return(basename(project))
-
- # otherwise, disambiguate based on project's path
- id <- substring(renv_bootstrap_hash_text(project), 1L, 8L)
- paste(basename(project), id, sep = "-")
-
- }
-
- renv_bootstrap_library_root <- function(project) {
-
- path <- Sys.getenv("RENV_PATHS_LIBRARY", unset = NA)
- if (!is.na(path))
- return(path)
-
- path <- Sys.getenv("RENV_PATHS_LIBRARY_ROOT", unset = NA)
- if (!is.na(path)) {
- name <- renv_bootstrap_library_root_name(project)
- return(file.path(path, name))
- }
-
- prefix <- renv_bootstrap_profile_prefix()
- paste(c(project, prefix, "renv/library"), collapse = "/")
-
- }
-
- renv_bootstrap_validate_version <- function(version) {
-
- loadedversion <- utils::packageDescription("renv", fields = "Version")
- if (version == loadedversion)
- return(TRUE)
-
- # assume four-component versions are from GitHub; three-component
- # versions are from CRAN
- components <- strsplit(loadedversion, "[.-]")[[1]]
- remote <- if (length(components) == 4L)
- paste("rstudio/renv", loadedversion, sep = "@")
- else
- paste("renv", loadedversion, sep = "@")
-
- fmt <- paste(
- "renv %1$s was loaded from project library, but this project is configured to use renv %2$s.",
- "Use `renv::record(\"%3$s\")` to record renv %1$s in the lockfile.",
- "Use `renv::restore(packages = \"renv\")` to install renv %2$s into the project library.",
- sep = "\n"
- )
-
- msg <- sprintf(fmt, loadedversion, version, remote)
- warning(msg, call. = FALSE)
-
- FALSE
-
- }
-
- renv_bootstrap_hash_text <- function(text) {
-
- hashfile <- tempfile("renv-hash-")
- on.exit(unlink(hashfile), add = TRUE)
-
- writeLines(text, con = hashfile)
- tools::md5sum(hashfile)
-
- }
-
- renv_bootstrap_load <- function(project, libpath, version) {
-
- # try to load renv from the project library
- if (!requireNamespace("renv", lib.loc = libpath, quietly = TRUE))
- return(FALSE)
-
- # warn if the version of renv loaded does not match
- renv_bootstrap_validate_version(version)
-
- # load the project
- renv::load(project)
-
- TRUE
-
- }
-
- renv_bootstrap_profile_load <- function(project) {
-
- # if RENV_PROFILE is already set, just use that
- profile <- Sys.getenv("RENV_PROFILE", unset = NA)
- if (!is.na(profile) && nzchar(profile))
- return(profile)
-
- # check for a profile file (nothing to do if it doesn't exist)
- path <- file.path(project, "renv/local/profile")
- if (!file.exists(path))
- return(NULL)
-
- # read the profile, and set it if it exists
- contents <- readLines(path, warn = FALSE)
- if (length(contents) == 0L)
- return(NULL)
-
- # set RENV_PROFILE
- profile <- contents[[1L]]
- if (nzchar(profile))
- Sys.setenv(RENV_PROFILE = profile)
-
- profile
-
- }
-
- renv_bootstrap_profile_prefix <- function() {
- profile <- renv_bootstrap_profile_get()
- if (!is.null(profile))
- return(file.path("renv/profiles", profile))
- }
-
- renv_bootstrap_profile_get <- function() {
- profile <- Sys.getenv("RENV_PROFILE", unset = "")
- renv_bootstrap_profile_normalize(profile)
- }
-
- renv_bootstrap_profile_set <- function(profile) {
- profile <- renv_bootstrap_profile_normalize(profile)
- if (is.null(profile))
- Sys.unsetenv("RENV_PROFILE")
- else
- Sys.setenv(RENV_PROFILE = profile)
- }
-
- renv_bootstrap_profile_normalize <- function(profile) {
-
- if (is.null(profile) || profile %in% c("", "default"))
- return(NULL)
-
- profile
-
- }
-
- # load the renv profile, if any
- renv_bootstrap_profile_load(project)
-
- # construct path to library root
- root <- renv_bootstrap_library_root(project)
-
- # construct library prefix for platform
- prefix <- renv_bootstrap_platform_prefix()
-
- # construct full libpath
- libpath <- file.path(root, prefix)
-
- # attempt to load
- if (renv_bootstrap_load(project, libpath, version))
- return(TRUE)
-
- # load failed; inform user we're about to bootstrap
- prefix <- paste("# Bootstrapping renv", version)
- postfix <- paste(rep.int("-", 77L - nchar(prefix)), collapse = "")
- header <- paste(prefix, postfix)
- message(header)
-
- # perform bootstrap
- bootstrap(version, libpath)
-
- # exit early if we're just testing bootstrap
- if (!is.na(Sys.getenv("RENV_BOOTSTRAP_INSTALL_ONLY", unset = NA)))
- return(TRUE)
-
- # try again to load
- if (requireNamespace("renv", lib.loc = libpath, quietly = TRUE)) {
- message("* Successfully installed and loaded renv ", version, ".")
- return(renv::load())
- }
-
- # failed to download or load renv; warn the user
- msg <- c(
- "Failed to find an renv installation: the project will not be loaded.",
- "Use `renv::activate()` to re-initialize the project."
- )
-
- warning(paste(msg, collapse = "\n"), call. = FALSE)
-
-})