diff --git a/shared-opt-dir/agency-parser/MT.gitignore b/shared-opt-dir/agency-parser/MT.gitignore index 00360166..2ff19e3b 100644 --- a/shared-opt-dir/agency-parser/MT.gitignore +++ b/shared-opt-dir/agency-parser/MT.gitignore @@ -23,6 +23,7 @@ hs_err_pid* /archive.sh /archive_selection.sh /build.gradle +/check_data_not_outdated.sh /check_data_outdated.sh /download.sh /force_single_thread_off.sh diff --git a/shared-opt-dir/agency-parser/check_data_not_outdated.sh b/shared-opt-dir/agency-parser/check_data_not_outdated.sh new file mode 100755 index 00000000..02d98eaf --- /dev/null +++ b/shared-opt-dir/agency-parser/check_data_not_outdated.sh @@ -0,0 +1,131 @@ +#!/bin/bash +SCRIPT_DIR="$(dirname "$0")"; +source ${SCRIPT_DIR}/../commons/commons.sh + +echo ">> Checking if data is outdated..." + +# Check if app-android directory exists +APP_ANDROID_DIR="${SCRIPT_DIR}/../app-android/src/main"; +if [[ ! -d "${APP_ANDROID_DIR}" ]]; then + echo ">> No app-android directory found. Data check not applicable."; + exit 0; +fi + +# Get current timestamp in seconds +NOW_TIMESTAMP_SEC=$(date +%s); +echo "> Current timestamp: '$NOW_TIMESTAMP_SEC'"; + +# Check current and next data files +CURRENT_VALUES="${APP_ANDROID_DIR}/res-current/values/current_gtfs_rts_values_gen.xml"; +NEXT_VALUES="${APP_ANDROID_DIR}/res-next/values/next_gtfs_rts_values_gen.xml"; + +# Prefer "next" file if available, otherwise fallback to "current" +DEPLOYED_LAST_DEPARTURE_SEC=""; +DATA_FILE_USED=""; + +requireCommand "xmllint" "libxml2-utils"; + +if [[ -f "$NEXT_VALUES" ]]; then + DEPLOYED_LAST_DEPARTURE_SEC=$(xmllint --xpath "//resources/integer[@name='next_gtfs_rts_last_departure_in_sec']/text()" "$NEXT_VALUES") + DATA_FILE_USED="next"; + if [[ -n "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then + echo "> Using next data file."; + else + echo "> Next data file found but timestamp not found."; + fi +elif [[ -f "$CURRENT_VALUES" ]]; then + DEPLOYED_LAST_DEPARTURE_SEC=$(xmllint --xpath "//resources/integer[@name='current_gtfs_rts_last_departure_in_sec']/text()" "$CURRENT_VALUES") + DATA_FILE_USED="current"; + if [[ -n "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then + echo "> Using current data file."; + else + echo "> Current data file found but timestamp not found."; + fi +fi + +if [[ -z "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then + if [[ ! -f "$NEXT_VALUES" ]] && [[ ! -f "$CURRENT_VALUES" ]]; then + echo ">> No data files found. Cannot determine if data is outdated."; + else + echo ">> Data files found but last departure timestamp not found. Cannot determine if data is outdated."; + fi + exit 0; # Exit code 0 - avoid triggering sync when uncertain +fi + +echo "> Deployed last departure timestamp: '$DEPLOYED_LAST_DEPARTURE_SEC' (from $DATA_FILE_USED)"; + +# Check if deployed data is not outdated (last departure is in the future) +if [[ "$DEPLOYED_LAST_DEPARTURE_SEC" -gt "$NOW_TIMESTAMP_SEC" ]]; then + DIFF_SEC=$((DEPLOYED_LAST_DEPARTURE_SEC - NOW_TIMESTAMP_SEC)); + DIFF_DAYS=$((DIFF_SEC / 86400)); + echo ">> Deployed data has not expired. Last departure will be in $DIFF_DAYS days."; + echo ">> Data is not outdated. Sync not recommended."; + exit 0; # Exit code 0 indicates data is not outdated +fi + +DIFF_SEC=$((NOW_TIMESTAMP_SEC - DEPLOYED_LAST_DEPARTURE_SEC)); +DIFF_DAYS=$((DIFF_SEC / 86400)); +echo ">> Deployed data has expired! Last departure was $DIFF_DAYS days ago."; +echo ">> Data is OUTDATED. Sync recommended. Looking for available archives..."; + +# Check archive directory for available data +ARCHIVE_DIR="${SCRIPT_DIR}/archive"; +echo "> Archive dir: '$ARCHIVE_DIR'"; + +if [[ ! -d "$ARCHIVE_DIR" ]]; then + echo ">> No archive directory found. Cannot check for newer data."; + echo ">> Data is up-to-date (based on deployed data only)."; + exit 0; # No archive, so we can't determine if newer data available +fi + +# Find archives +mapfile -t ARCHIVES < <(find "$ARCHIVE_DIR" -name "*.zip" -type f 2>/dev/null | sort) +echo "> Archives found: ${#ARCHIVES[@]}"; + +if [[ "${#ARCHIVES[@]}" -eq 0 ]]; then + echo ">> No archives available. Data cannot be updated."; + echo ">> Data is up-to-date (based on deployed data only)."; + exit 0; +fi + +# Check if any archive has data that extends beyond the deployed last departure +ARCHIVE_HAS_NEWER_DATA=false; + +# Find archives and check them +for ARCHIVE in "${ARCHIVES[@]}" ; do + ARCHIVE_BASENAME=$(basename "$ARCHIVE"); + ARCHIVE_BASENAME_NO_EXT="${ARCHIVE_BASENAME%.*}"; + + # Validate archive date format + if ! [[ "$ARCHIVE_BASENAME_NO_EXT" =~ ^[0-9]{8}-[0-9]{8}$ ]]; then + echo "> Archive: $ARCHIVE_BASENAME"; + echo " - WARNING: Archive filename doesn't match expected format (YYYYMMDD-YYYYMMDD.zip)"; + continue; + fi + + ARCHIVE_START_DATE=${ARCHIVE_BASENAME_NO_EXT:0:8} + ARCHIVE_END_DATE=${ARCHIVE_BASENAME_NO_EXT:9:8} + echo "> Archive: $ARCHIVE_BASENAME (${ARCHIVE_START_DATE} to ${ARCHIVE_END_DATE})"; + + # Convert archive end date (YYYYMMDD) to timestamp at end of day (23:59:59) + ARCHIVE_END_TIMESTAMP=$(date -d "${ARCHIVE_END_DATE} 23:59:59" +%s 2>/dev/null); + + if [[ -n "$ARCHIVE_END_TIMESTAMP" ]]; then + echo " - Archive end timestamp: $ARCHIVE_END_TIMESTAMP"; + # If archive has data beyond what's currently deployed, we need to sync + if [[ "$ARCHIVE_END_TIMESTAMP" -gt "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then + echo " - Archive has newer data than deployed!"; + ARCHIVE_HAS_NEWER_DATA=true; + break; # Found newer data, no need to check other archives + fi + fi +done + +# Determine if data is outdated based on archive availability +if [[ "$ARCHIVE_HAS_NEWER_DATA" == true ]]; then + echo ">> Archive contains newer data. Data is OUTDATED. Sync recommended."; + exit 1; # Exit code 1 indicates data is outdated +else + echo ">> Data is up-to-date."; + exit 0; # Exit code 0 indicates data is current +fi diff --git a/shared-opt-dir/agency-parser/check_data_outdated.sh b/shared-opt-dir/agency-parser/check_data_outdated.sh index 02d98eaf..9ae77258 100755 --- a/shared-opt-dir/agency-parser/check_data_outdated.sh +++ b/shared-opt-dir/agency-parser/check_data_outdated.sh @@ -1,131 +1,4 @@ #!/bin/bash SCRIPT_DIR="$(dirname "$0")"; -source ${SCRIPT_DIR}/../commons/commons.sh -echo ">> Checking if data is outdated..." - -# Check if app-android directory exists -APP_ANDROID_DIR="${SCRIPT_DIR}/../app-android/src/main"; -if [[ ! -d "${APP_ANDROID_DIR}" ]]; then - echo ">> No app-android directory found. Data check not applicable."; - exit 0; -fi - -# Get current timestamp in seconds -NOW_TIMESTAMP_SEC=$(date +%s); -echo "> Current timestamp: '$NOW_TIMESTAMP_SEC'"; - -# Check current and next data files -CURRENT_VALUES="${APP_ANDROID_DIR}/res-current/values/current_gtfs_rts_values_gen.xml"; -NEXT_VALUES="${APP_ANDROID_DIR}/res-next/values/next_gtfs_rts_values_gen.xml"; - -# Prefer "next" file if available, otherwise fallback to "current" -DEPLOYED_LAST_DEPARTURE_SEC=""; -DATA_FILE_USED=""; - -requireCommand "xmllint" "libxml2-utils"; - -if [[ -f "$NEXT_VALUES" ]]; then - DEPLOYED_LAST_DEPARTURE_SEC=$(xmllint --xpath "//resources/integer[@name='next_gtfs_rts_last_departure_in_sec']/text()" "$NEXT_VALUES") - DATA_FILE_USED="next"; - if [[ -n "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then - echo "> Using next data file."; - else - echo "> Next data file found but timestamp not found."; - fi -elif [[ -f "$CURRENT_VALUES" ]]; then - DEPLOYED_LAST_DEPARTURE_SEC=$(xmllint --xpath "//resources/integer[@name='current_gtfs_rts_last_departure_in_sec']/text()" "$CURRENT_VALUES") - DATA_FILE_USED="current"; - if [[ -n "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then - echo "> Using current data file."; - else - echo "> Current data file found but timestamp not found."; - fi -fi - -if [[ -z "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then - if [[ ! -f "$NEXT_VALUES" ]] && [[ ! -f "$CURRENT_VALUES" ]]; then - echo ">> No data files found. Cannot determine if data is outdated."; - else - echo ">> Data files found but last departure timestamp not found. Cannot determine if data is outdated."; - fi - exit 0; # Exit code 0 - avoid triggering sync when uncertain -fi - -echo "> Deployed last departure timestamp: '$DEPLOYED_LAST_DEPARTURE_SEC' (from $DATA_FILE_USED)"; - -# Check if deployed data is not outdated (last departure is in the future) -if [[ "$DEPLOYED_LAST_DEPARTURE_SEC" -gt "$NOW_TIMESTAMP_SEC" ]]; then - DIFF_SEC=$((DEPLOYED_LAST_DEPARTURE_SEC - NOW_TIMESTAMP_SEC)); - DIFF_DAYS=$((DIFF_SEC / 86400)); - echo ">> Deployed data has not expired. Last departure will be in $DIFF_DAYS days."; - echo ">> Data is not outdated. Sync not recommended."; - exit 0; # Exit code 0 indicates data is not outdated -fi - -DIFF_SEC=$((NOW_TIMESTAMP_SEC - DEPLOYED_LAST_DEPARTURE_SEC)); -DIFF_DAYS=$((DIFF_SEC / 86400)); -echo ">> Deployed data has expired! Last departure was $DIFF_DAYS days ago."; -echo ">> Data is OUTDATED. Sync recommended. Looking for available archives..."; - -# Check archive directory for available data -ARCHIVE_DIR="${SCRIPT_DIR}/archive"; -echo "> Archive dir: '$ARCHIVE_DIR'"; - -if [[ ! -d "$ARCHIVE_DIR" ]]; then - echo ">> No archive directory found. Cannot check for newer data."; - echo ">> Data is up-to-date (based on deployed data only)."; - exit 0; # No archive, so we can't determine if newer data available -fi - -# Find archives -mapfile -t ARCHIVES < <(find "$ARCHIVE_DIR" -name "*.zip" -type f 2>/dev/null | sort) -echo "> Archives found: ${#ARCHIVES[@]}"; - -if [[ "${#ARCHIVES[@]}" -eq 0 ]]; then - echo ">> No archives available. Data cannot be updated."; - echo ">> Data is up-to-date (based on deployed data only)."; - exit 0; -fi - -# Check if any archive has data that extends beyond the deployed last departure -ARCHIVE_HAS_NEWER_DATA=false; - -# Find archives and check them -for ARCHIVE in "${ARCHIVES[@]}" ; do - ARCHIVE_BASENAME=$(basename "$ARCHIVE"); - ARCHIVE_BASENAME_NO_EXT="${ARCHIVE_BASENAME%.*}"; - - # Validate archive date format - if ! [[ "$ARCHIVE_BASENAME_NO_EXT" =~ ^[0-9]{8}-[0-9]{8}$ ]]; then - echo "> Archive: $ARCHIVE_BASENAME"; - echo " - WARNING: Archive filename doesn't match expected format (YYYYMMDD-YYYYMMDD.zip)"; - continue; - fi - - ARCHIVE_START_DATE=${ARCHIVE_BASENAME_NO_EXT:0:8} - ARCHIVE_END_DATE=${ARCHIVE_BASENAME_NO_EXT:9:8} - echo "> Archive: $ARCHIVE_BASENAME (${ARCHIVE_START_DATE} to ${ARCHIVE_END_DATE})"; - - # Convert archive end date (YYYYMMDD) to timestamp at end of day (23:59:59) - ARCHIVE_END_TIMESTAMP=$(date -d "${ARCHIVE_END_DATE} 23:59:59" +%s 2>/dev/null); - - if [[ -n "$ARCHIVE_END_TIMESTAMP" ]]; then - echo " - Archive end timestamp: $ARCHIVE_END_TIMESTAMP"; - # If archive has data beyond what's currently deployed, we need to sync - if [[ "$ARCHIVE_END_TIMESTAMP" -gt "$DEPLOYED_LAST_DEPARTURE_SEC" ]]; then - echo " - Archive has newer data than deployed!"; - ARCHIVE_HAS_NEWER_DATA=true; - break; # Found newer data, no need to check other archives - fi - fi -done - -# Determine if data is outdated based on archive availability -if [[ "$ARCHIVE_HAS_NEWER_DATA" == true ]]; then - echo ">> Archive contains newer data. Data is OUTDATED. Sync recommended."; - exit 1; # Exit code 1 indicates data is outdated -else - echo ">> Data is up-to-date."; - exit 0; # Exit code 0 indicates data is current -fi +exec "${SCRIPT_DIR}/check_data_not_outdated.sh" "$@"; diff --git a/shared-overwrite/.github/workflows/mt-download-data.yml b/shared-overwrite/.github/workflows/mt-download-data.yml index 2f850e09..03dd99b7 100644 --- a/shared-overwrite/.github/workflows/mt-download-data.yml +++ b/shared-overwrite/.github/workflows/mt-download-data.yml @@ -76,7 +76,7 @@ jobs: if: steps.check-input-url.outputs.has_input_url == 'true' continue-on-error: true run: | - if [[ "${MT_SKIP_PUSH_COMMIT}" != false ]] && ./agency-parser/check_data_outdated.sh; then + if [[ "${MT_SKIP_PUSH_COMMIT}" != false ]] && ./agency-parser/check_data_not_outdated.sh; then echo ">> No new data downloaded and data is up-to-date. No sync needed."; else echo ">> New data downloaded or data is outdated. Triggering mt-sync-code-data.yml workflow (skip download data)...";