From 8d4abfcb81caabd28eac92c67264234f95281b48 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 28 Oct 2025 21:44:04 +0000
Subject: [PATCH 1/8] Initial plan


From ba76c01103dcf4d7041c96c651e370bd9a70c12f Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 28 Oct 2025 21:51:53 +0000
Subject: [PATCH 2/8] Implement ETag caching, 502 fallback, and first-run
 handling in pull-config.sh

Co-authored-by: runleveldev <44057501+runleveldev@users.noreply.github.com>
---
 nginx-reverse-proxy/pull-config.sh       |  84 +++++-
 nginx-reverse-proxy/test-502-fallback.sh | 174 ++++++++++++
 nginx-reverse-proxy/test-pull-config.sh  | 326 +++++++++++++++++++++++
 3 files changed, 580 insertions(+), 4 deletions(-)
 create mode 100755 nginx-reverse-proxy/test-502-fallback.sh
 create mode 100755 nginx-reverse-proxy/test-pull-config.sh

diff --git a/nginx-reverse-proxy/pull-config.sh b/nginx-reverse-proxy/pull-config.sh
index 85aa7318..4453df23 100755
--- a/nginx-reverse-proxy/pull-config.sh
+++ b/nginx-reverse-proxy/pull-config.sh
@@ -3,15 +3,91 @@
 set -euo pipefail
 
 CONF_FILE=/etc/nginx/conf.d/reverse-proxy.conf
+ETAG_FILE=/etc/nginx/conf.d/reverse-proxy.etag
 CONF_URL=https://create-a-container.opensource.mieweb.org/nginx.conf
+FALLBACK_URL=http://create-a-container.cluster.mieweb.org:3000/nginx.conf
 
-mv "${CONF_FILE}" "${CONF_FILE}.bak"
-curl -fsSL -o "${CONF_FILE}" "${CONF_URL}"
+# Function to download config and extract ETag
+download_config() {
+  local url="$1"
+  local temp_file="${CONF_FILE}.tmp"
+  local headers_file="${CONF_FILE}.headers"
+  
+  # Read existing ETag if it exists
+  local etag_header=""
+  if [[ -f "${ETAG_FILE}" ]]; then
+    local etag
+    etag=$(cat "${ETAG_FILE}")
+    etag_header="If-None-Match: ${etag}"
+  fi
+  
+  # Download with headers, capture HTTP status code
+  # Note: We don't use -f flag to allow handling of all HTTP status codes
+  local http_code
+  if [[ -n "${etag_header}" ]]; then
+    http_code=$(curl -w "%{http_code}" -D "${headers_file}" -H "${etag_header}" -o "${temp_file}" -sSL "${url}" 2>/dev/null || echo "000")
+  else
+    http_code=$(curl -w "%{http_code}" -D "${headers_file}" -o "${temp_file}" -sSL "${url}" 2>/dev/null || echo "000")
+  fi
+  
+  # Return the http_code
+  echo "${http_code}"
+}
 
+# Try primary URL
+HTTP_CODE=$(download_config "${CONF_URL}")
+
+# Handle 502 error with fallback
+if [[ ${HTTP_CODE} -eq 502 ]] || [[ ${HTTP_CODE} -eq 000 ]]; then
+  echo "Primary URL failed (HTTP ${HTTP_CODE}), trying fallback URL..." >&2
+  rm -f "${CONF_FILE}.tmp" "${CONF_FILE}.headers"
+  HTTP_CODE=$(download_config "${FALLBACK_URL}")
+fi
+
+# Check if we got a 304 Not Modified
+if [[ ${HTTP_CODE} -eq 304 ]]; then
+  # No changes, clean up temp files and exit
+  rm -f "${CONF_FILE}.tmp" "${CONF_FILE}.headers"
+  exit 0
+fi
+
+# Check if we got a successful response
+if [[ ${HTTP_CODE} -ne 200 ]]; then
+  rm -f "${CONF_FILE}.tmp" "${CONF_FILE}.headers"
+  echo "Failed to download configuration (HTTP ${HTTP_CODE})" >&2
+  exit 1
+fi
+
+# Extract new ETag from headers
+NEW_ETAG=""
+if [[ -f "${CONF_FILE}.headers" ]]; then
+  NEW_ETAG=$(grep -i '^etag:' "${CONF_FILE}.headers" | sed 's/^etag: *//i' | tr -d '\r\n' || echo "")
+fi
+
+# Backup existing config if it exists
+if [[ -f "${CONF_FILE}" ]]; then
+  mv "${CONF_FILE}" "${CONF_FILE}.bak"
+fi
+
+# Move new config into place
+mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+
+# Test the new configuration
 if ! nginx -t; then
-  mv "${CONF_FILE}.bak" "${CONF_FILE}"
+  # Restore backup if it exists
+  if [[ -f "${CONF_FILE}.bak" ]]; then
+    mv "${CONF_FILE}.bak" "${CONF_FILE}"
+  else
+    # No backup, just remove the bad config
+    rm -f "${CONF_FILE}"
+  fi
+  rm -f "${CONF_FILE}.headers"
   exit 1
 fi
 
-rm -f "${CONF_FILE}.bak"
+# Configuration is valid, clean up backup and save new ETag
+rm -f "${CONF_FILE}.bak" "${CONF_FILE}.headers"
+if [[ -n "${NEW_ETAG}" ]]; then
+  echo "${NEW_ETAG}" > "${ETAG_FILE}"
+fi
 nginx -s reload
\ No newline at end of file
diff --git a/nginx-reverse-proxy/test-502-fallback.sh b/nginx-reverse-proxy/test-502-fallback.sh
new file mode 100755
index 00000000..63ae44a8
--- /dev/null
+++ b/nginx-reverse-proxy/test-502-fallback.sh
@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+#
+# Additional test for 502 fallback behavior
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_DIR="/tmp/nginx-pull-config-502-test-$$"
+
+# Colors
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+# Setup test environment
+setup() {
+  echo -e "${YELLOW}Setting up test environment for 502 fallback test...${NC}"
+  mkdir -p "${TEST_DIR}/etc/nginx/conf.d"
+  mkdir -p "${TEST_DIR}/bin"
+  
+  # Create a mock nginx command
+  cat > "${TEST_DIR}/bin/nginx" <<'EOF'
+#!/bin/bash
+case "$1" in
+  -t)
+    exit 0
+    ;;
+  -s)
+    if [[ "$2" == "reload" ]]; then
+      echo "nginx reloaded"
+      exit 0
+    fi
+    ;;
+esac
+exit 1
+EOF
+  chmod +x "${TEST_DIR}/bin/nginx"
+  export PATH="${TEST_DIR}/bin:${PATH}"
+}
+
+# Cleanup
+cleanup() {
+  echo -e "${YELLOW}Cleaning up...${NC}"
+  rm -rf "${TEST_DIR}"
+}
+
+# Test 502 fallback
+test_502_fallback() {
+  echo -e "\n${YELLOW}Test: 502 error with fallback to internal URL${NC}"
+  
+  # Create modified script
+  local test_script="${TEST_DIR}/pull-config-502-test.sh"
+  sed "s|/etc/nginx/conf.d|${TEST_DIR}/etc/nginx/conf.d|g" "${SCRIPT_DIR}/pull-config.sh" > "${test_script}"
+  chmod +x "${test_script}"
+  
+  # Create curl mock that returns 502 for primary URL, 200 for fallback
+  cat > "${TEST_DIR}/bin/curl" <<'EOF'
+#!/bin/bash
+url=""
+output_file=""
+headers_file=""
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -o)
+      output_file="$2"
+      shift 2
+      ;;
+    -D)
+      headers_file="$2"
+      shift 2
+      ;;
+    -H|-w)
+      shift 2
+      ;;
+    -sSL)
+      shift
+      ;;
+    *)
+      url="$1"
+      shift
+      ;;
+  esac
+done
+
+# Check which URL is being requested
+if [[ "${url}" == *"create-a-container.opensource.mieweb.org"* ]]; then
+  # Primary URL - return 502
+  if [[ -n "${headers_file}" ]]; then
+    cat > "${headers_file}" <<HEADERS
+HTTP/1.1 502 Bad Gateway
+Content-Type: text/html
+Content-Length: 150
+
+HEADERS
+  fi
+  if [[ -n "${output_file}" ]]; then
+    echo "<html><body>502 Bad Gateway</body></html>" > "${output_file}"
+  fi
+  echo "502"
+elif [[ "${url}" == *"create-a-container.cluster.mieweb.org"* ]]; then
+  # Fallback URL - return 200
+  if [[ -n "${headers_file}" ]]; then
+    cat > "${headers_file}" <<HEADERS
+HTTP/1.1 200 OK
+Content-Type: text/plain
+ETag: "fallback-etag-123"
+Content-Length: 30
+
+HEADERS
+  fi
+  if [[ -n "${output_file}" ]]; then
+    echo "# Config from fallback URL" > "${output_file}"
+  fi
+  echo "200"
+else
+  echo "000"
+fi
+EOF
+  chmod +x "${TEST_DIR}/bin/curl"
+  export PATH="${TEST_DIR}/bin:${PATH}"
+  
+  # Run the script
+  if "${test_script}" 2>&1 | tee "${TEST_DIR}/output.log"; then
+    # Check that config was created from fallback
+    if [[ -f "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf" ]]; then
+      if grep -q "fallback URL" "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"; then
+        echo -e "${GREEN}✓ Test passed: Fallback URL used successfully${NC}"
+        
+        # Check that fallback message was logged
+        if grep -q "Primary URL failed" "${TEST_DIR}/output.log"; then
+          echo -e "${GREEN}✓ Test passed: Fallback message logged${NC}"
+          return 0
+        else
+          echo -e "${RED}✗ Test failed: No fallback message in output${NC}"
+          return 1
+        fi
+      else
+        echo -e "${RED}✗ Test failed: Config not from fallback URL${NC}"
+        cat "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"
+        return 1
+      fi
+    else
+      echo -e "${RED}✗ Test failed: Config file not created${NC}"
+      return 1
+    fi
+  else
+    echo -e "${RED}✗ Test failed: Script exited with error${NC}"
+    cat "${TEST_DIR}/output.log"
+    return 1
+  fi
+}
+
+# Main
+main() {
+  echo -e "${YELLOW}=== Testing 502 Fallback Behavior ===${NC}"
+  
+  setup
+  
+  if test_502_fallback; then
+    cleanup
+    echo -e "\n${GREEN}502 fallback test passed!${NC}"
+    exit 0
+  else
+    cleanup
+    echo -e "\n${RED}502 fallback test failed${NC}"
+    exit 1
+  fi
+}
+
+main "$@"
diff --git a/nginx-reverse-proxy/test-pull-config.sh b/nginx-reverse-proxy/test-pull-config.sh
new file mode 100755
index 00000000..478e1cf0
--- /dev/null
+++ b/nginx-reverse-proxy/test-pull-config.sh
@@ -0,0 +1,326 @@
+#!/usr/bin/env bash
+#
+# Test script for pull-config.sh
+# This script validates the three main improvements:
+# 1. Handling missing config files (first run)
+# 2. ETag-based caching
+# 3. Fallback to internal URL on 502 errors
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_DIR="/tmp/nginx-pull-config-test-$$"
+TEST_RESULTS=0
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Setup test environment
+setup() {
+  echo -e "${YELLOW}Setting up test environment...${NC}"
+  mkdir -p "${TEST_DIR}/etc/nginx/conf.d"
+  mkdir -p "${TEST_DIR}/bin"
+  
+  # Create a mock nginx command
+  cat > "${TEST_DIR}/bin/nginx" <<'EOF'
+#!/bin/bash
+case "$1" in
+  -t)
+    # Always succeed for testing
+    exit 0
+    ;;
+  -s)
+    if [[ "$2" == "reload" ]]; then
+      echo "nginx reloaded"
+      exit 0
+    fi
+    ;;
+esac
+exit 1
+EOF
+  chmod +x "${TEST_DIR}/bin/nginx"
+  
+  # Add mock nginx to PATH
+  export PATH="${TEST_DIR}/bin:${PATH}"
+  
+  echo -e "${GREEN}✓ Test environment ready${NC}"
+}
+
+# Cleanup test environment
+cleanup() {
+  echo -e "${YELLOW}Cleaning up test environment...${NC}"
+  rm -rf "${TEST_DIR}"
+}
+
+# Test 1: First run without existing config
+test_first_run() {
+  echo -e "\n${YELLOW}Test 1: First run without existing config${NC}"
+  
+  # Create a modified version of the script that uses our test directory
+  local test_script="${TEST_DIR}/pull-config-test.sh"
+  sed "s|/etc/nginx/conf.d|${TEST_DIR}/etc/nginx/conf.d|g" "${SCRIPT_DIR}/pull-config.sh" > "${test_script}"
+  chmod +x "${test_script}"
+  
+  # Replace curl with a mock that simulates a successful download
+  cat > "${TEST_DIR}/bin/curl" <<'EOF'
+#!/bin/bash
+# Parse arguments
+output_file=""
+headers_file=""
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -o)
+      output_file="$2"
+      shift 2
+      ;;
+    -D)
+      headers_file="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+
+# Write mock config
+if [[ -n "${output_file}" ]]; then
+  echo "# Mock nginx config" > "${output_file}"
+fi
+
+# Write mock headers with ETag
+if [[ -n "${headers_file}" ]]; then
+  cat > "${headers_file}" <<HEADERS
+HTTP/1.1 200 OK
+Content-Type: text/plain
+ETag: "mock-etag-12345"
+Content-Length: 20
+
+HEADERS
+fi
+
+# Return 200 status code
+echo "200"
+EOF
+  chmod +x "${TEST_DIR}/bin/curl"
+  export PATH="${TEST_DIR}/bin:${PATH}"
+  
+  # Run the script
+  if "${test_script}"; then
+    if [[ -f "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf" ]]; then
+      echo -e "${GREEN}✓ Test passed: Config file created${NC}"
+      if [[ -f "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.etag" ]]; then
+        echo -e "${GREEN}✓ Test passed: ETag file created${NC}"
+      else
+        echo -e "${RED}✗ Test failed: ETag file not created${NC}"
+        TEST_RESULTS=1
+      fi
+    else
+      echo -e "${RED}✗ Test failed: Config file not created${NC}"
+      TEST_RESULTS=1
+    fi
+  else
+    echo -e "${RED}✗ Test failed: Script exited with error${NC}"
+    TEST_RESULTS=1
+  fi
+}
+
+# Test 2: Subsequent run with ETag (304 Not Modified)
+test_etag_cache_hit() {
+  echo -e "\n${YELLOW}Test 2: ETag cache hit (304 Not Modified)${NC}"
+  
+  # Setup: Create existing config and ETag
+  echo "# Existing config" > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"
+  echo '"mock-etag-12345"' > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.etag"
+  
+  # Create a modified version of the script
+  local test_script="${TEST_DIR}/pull-config-etag-test.sh"
+  sed "s|/etc/nginx/conf.d|${TEST_DIR}/etc/nginx/conf.d|g" "${SCRIPT_DIR}/pull-config.sh" > "${test_script}"
+  chmod +x "${test_script}"
+  
+  # Replace curl with a mock that returns 304
+  cat > "${TEST_DIR}/bin/curl" <<'EOF'
+#!/bin/bash
+# Parse arguments to check for If-None-Match header
+has_etag=0
+output_file=""
+headers_file=""
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -H)
+      if [[ "$2" == If-None-Match* ]]; then
+        has_etag=1
+      fi
+      shift 2
+      ;;
+    -o)
+      output_file="$2"
+      shift 2
+      ;;
+    -D)
+      headers_file="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+
+# If ETag header was sent, return 304
+if [[ ${has_etag} -eq 1 ]]; then
+  # Write mock headers
+  if [[ -n "${headers_file}" ]]; then
+    cat > "${headers_file}" <<HEADERS
+HTTP/1.1 304 Not Modified
+ETag: "mock-etag-12345"
+
+HEADERS
+  fi
+  echo "304"
+else
+  # Write mock config and headers for 200
+  if [[ -n "${output_file}" ]]; then
+    echo "# New mock config" > "${output_file}"
+  fi
+  if [[ -n "${headers_file}" ]]; then
+    cat > "${headers_file}" <<HEADERS
+HTTP/1.1 200 OK
+ETag: "mock-etag-67890"
+
+HEADERS
+  fi
+  echo "200"
+fi
+EOF
+  chmod +x "${TEST_DIR}/bin/curl"
+  
+  # Store original config modification time
+  local orig_mtime
+  orig_mtime=$(stat -c %Y "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf")
+  
+  # Run the script
+  if "${test_script}"; then
+    local new_mtime
+    new_mtime=$(stat -c %Y "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf")
+    if [[ ${orig_mtime} -eq ${new_mtime} ]]; then
+      echo -e "${GREEN}✓ Test passed: Config not modified on 304${NC}"
+    else
+      echo -e "${RED}✗ Test failed: Config was modified on 304${NC}"
+      TEST_RESULTS=1
+    fi
+  else
+    echo -e "${RED}✗ Test failed: Script exited with error${NC}"
+    TEST_RESULTS=1
+  fi
+}
+
+# Test 3: Config validation failure and rollback
+test_validation_rollback() {
+  echo -e "\n${YELLOW}Test 3: Config validation failure and rollback${NC}"
+  
+  # Setup: Create existing valid config
+  echo "# Valid existing config" > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"
+  echo '"mock-etag-old"' > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.etag"
+  
+  # Create a modified version of the script
+  local test_script="${TEST_DIR}/pull-config-rollback-test.sh"
+  sed "s|/etc/nginx/conf.d|${TEST_DIR}/etc/nginx/conf.d|g" "${SCRIPT_DIR}/pull-config.sh" > "${test_script}"
+  chmod +x "${test_script}"
+  
+  # Replace nginx with one that fails validation
+  cat > "${TEST_DIR}/bin/nginx" <<'EOF'
+#!/bin/bash
+case "$1" in
+  -t)
+    # Fail validation
+    echo "nginx: configuration file test failed"
+    exit 1
+    ;;
+  -s)
+    if [[ "$2" == "reload" ]]; then
+      echo "nginx reloaded"
+      exit 0
+    fi
+    ;;
+esac
+exit 1
+EOF
+  chmod +x "${TEST_DIR}/bin/nginx"
+  
+  # Replace curl to return a new config
+  cat > "${TEST_DIR}/bin/curl" <<'EOF'
+#!/bin/bash
+output_file=""
+headers_file=""
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -o)
+      output_file="$2"
+      shift 2
+      ;;
+    -D)
+      headers_file="$2"
+      shift 2
+      ;;
+    *)
+      shift
+      ;;
+  esac
+done
+
+if [[ -n "${output_file}" ]]; then
+  echo "# Invalid new config" > "${output_file}"
+fi
+if [[ -n "${headers_file}" ]]; then
+  cat > "${headers_file}" <<HEADERS
+HTTP/1.1 200 OK
+ETag: "mock-etag-new"
+
+HEADERS
+fi
+echo "200"
+EOF
+  chmod +x "${TEST_DIR}/bin/curl"
+  
+  # Run the script (should fail and rollback)
+  if "${test_script}"; then
+    echo -e "${RED}✗ Test failed: Script should have exited with error${NC}"
+    TEST_RESULTS=1
+  else
+    # Check that the original config was restored
+    if grep -q "Valid existing config" "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"; then
+      echo -e "${GREEN}✓ Test passed: Original config restored after validation failure${NC}"
+    else
+      echo -e "${RED}✗ Test failed: Original config not restored${NC}"
+      TEST_RESULTS=1
+    fi
+  fi
+}
+
+# Main test execution
+main() {
+  echo -e "${YELLOW}=== Testing pull-config.sh ===${NC}"
+  
+  setup
+  
+  test_first_run
+  test_etag_cache_hit
+  test_validation_rollback
+  
+  cleanup
+  
+  echo -e "\n${YELLOW}=== Test Summary ===${NC}"
+  if [[ ${TEST_RESULTS} -eq 0 ]]; then
+    echo -e "${GREEN}All tests passed!${NC}"
+    exit 0
+  else
+    echo -e "${RED}Some tests failed${NC}"
+    exit 1
+  fi
+}
+
+main "$@"

From e8b39c32f2acc10ba9cc619327b6e18bb9970d33 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 28 Oct 2025 21:54:47 +0000
Subject: [PATCH 3/8] Update README documentation for ETag caching and 502
 fallback features

Co-authored-by: runleveldev <44057501+runleveldev@users.noreply.github.com>
---
 nginx-reverse-proxy/README.md | 71 +++++++++++++++++++++++++++++++----
 1 file changed, 64 insertions(+), 7 deletions(-)

diff --git a/nginx-reverse-proxy/README.md b/nginx-reverse-proxy/README.md
index 5b529261..68ab72b3 100644
--- a/nginx-reverse-proxy/README.md
+++ b/nginx-reverse-proxy/README.md
@@ -10,11 +10,20 @@ The reverse proxy configuration is automatically synchronized from the create-a-
 
 ### `pull-config.sh`
 Bash script that:
-1. Backs up the current nginx configuration
-2. Downloads the latest configuration from the API endpoint
-3. Tests the new configuration with `nginx -t`
-4. Rolls back to the backup if validation fails
-5. Reloads nginx if validation succeeds
+1. Checks for an existing ETag and sends it with the request (If-None-Match header)
+2. Skips all operations if the server returns 304 Not Modified (no changes)
+3. Falls back to internal cluster URL if the primary URL returns a 502 error
+4. Backs up the current nginx configuration (if it exists)
+5. Downloads the latest configuration from the API endpoint
+6. Tests the new configuration with `nginx -t`
+7. Rolls back to the backup if validation fails (if backup exists)
+8. Saves the ETag for future requests
+9. Reloads nginx if validation succeeds
+
+This approach ensures:
+- **Efficient bandwidth usage**: Only downloads when configuration has changed
+- **High availability**: Automatically falls back to internal URL on gateway errors
+- **Bootstrap-friendly**: Works correctly on first run when no config exists yet
 
 ### `pull-config.cron`
 Cron job definition that runs `pull-config.sh` every minute to keep the nginx configuration synchronized with the database.
@@ -71,7 +80,7 @@ sudo chmod +x /opt/opensource-server/nginx-reverse-proxy/pull-config.sh
 ### 4. Initial Configuration Pull
 ```bash
 # Run script manually to get initial configuration
-sudo touch /etc/nginx/conf.d/reverse-proxy.conf
+# No need to pre-create the config file - the script handles first run
 sudo /opt/opensource-server/nginx-reverse-proxy/pull-config.sh
 ```
 
@@ -93,7 +102,9 @@ sudo tail -f /var/log/syslog | grep pull-config
 The scripts use these default paths (can be modified in the scripts):
 
 - `CONF_FILE`: `/etc/nginx/conf.d/reverse-proxy.conf` - Target nginx config file
-- `CONF_URL`: `https://create-a-container.opensource.mieweb.org/nginx.conf` - API endpoint
+- `ETAG_FILE`: `/etc/nginx/conf.d/reverse-proxy.etag` - Stores ETag for caching
+- `CONF_URL`: `https://create-a-container.opensource.mieweb.org/nginx.conf` - Primary API endpoint
+- `FALLBACK_URL`: `http://create-a-container.cluster.mieweb.org:3000/nginx.conf` - Fallback endpoint (used on 502 errors)
 
 ### Cron Schedule
 By default, the configuration is pulled every minute:
@@ -109,6 +120,22 @@ To change the schedule, edit `/etc/cron.d/nginx-pull-config` with standard cron
 # Every day at midnight: 0 0 * * *
 ```
 
+**Note**: Thanks to ETag-based caching, running the script every minute has minimal overhead. The script only downloads and reloads nginx when the configuration actually changes. Most runs will exit early with a 304 Not Modified response.
+
+### Optimizations
+
+#### ETag-Based Caching
+The script uses HTTP ETags to avoid unnecessary downloads. On each run:
+- If an ETag exists from a previous run, it's sent with the request
+- If the server returns 304 Not Modified, the script exits immediately
+- This reduces bandwidth usage and prevents unnecessary nginx reloads
+
+#### High Availability Fallback
+If the primary URL (via reverse proxy) returns a 502 Bad Gateway error:
+- The script automatically falls back to the internal cluster URL
+- This handles the bootstrapping problem where the reverse proxy isn't configured yet
+- The internal URL bypasses the reverse proxy and connects directly to the service
+
 ## Troubleshooting
 
 ### Configuration Pull Failures
@@ -185,6 +212,13 @@ ls -lt /etc/nginx/conf.d/reverse-proxy.conf
 sudo /opt/opensource-server/nginx-reverse-proxy/pull-config.sh
 ```
 
+### Force Download (Bypass ETag Cache)
+```bash
+# Remove the ETag file to force a fresh download
+sudo rm -f /etc/nginx/conf.d/reverse-proxy.etag
+sudo /opt/opensource-server/nginx-reverse-proxy/pull-config.sh
+```
+
 ### Disable Automatic Updates
 ```bash
 # Temporarily disable cron job
@@ -193,3 +227,26 @@ sudo chmod 000 /etc/cron.d/nginx-pull-config
 # Or remove it completely
 sudo rm /etc/cron.d/nginx-pull-config
 ```
+
+## Testing
+
+The repository includes comprehensive test scripts to validate the pull-config.sh behavior:
+
+### Run All Tests
+```bash
+cd /opt/opensource-server/nginx-reverse-proxy
+./test-pull-config.sh
+```
+
+This tests:
+- First run without existing configuration
+- ETag caching (304 Not Modified handling)
+- Configuration validation and rollback on failure
+
+### Test 502 Fallback
+```bash
+cd /opt/opensource-server/nginx-reverse-proxy
+./test-502-fallback.sh
+```
+
+This validates the fallback mechanism when the primary URL returns a 502 error.

From 06e8ac442f8d43b8d2788a889df2117ea6543448 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 28 Oct 2025 21:56:38 +0000
Subject: [PATCH 4/8] Address code review feedback on documentation clarity

Co-authored-by: runleveldev <44057501+runleveldev@users.noreply.github.com>
---
 nginx-reverse-proxy/README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nginx-reverse-proxy/README.md b/nginx-reverse-proxy/README.md
index 68ab72b3..1b9b91b2 100644
--- a/nginx-reverse-proxy/README.md
+++ b/nginx-reverse-proxy/README.md
@@ -80,7 +80,7 @@ sudo chmod +x /opt/opensource-server/nginx-reverse-proxy/pull-config.sh
 ### 4. Initial Configuration Pull
 ```bash
 # Run script manually to get initial configuration
-# No need to pre-create the config file - the script handles first run
+# No need to pre-create the config file - the script handles first run when no existing configuration exists
 sudo /opt/opensource-server/nginx-reverse-proxy/pull-config.sh
 ```
 
@@ -120,7 +120,7 @@ To change the schedule, edit `/etc/cron.d/nginx-pull-config` with standard cron
 # Every day at midnight: 0 0 * * *
 ```
 
-**Note**: Thanks to ETag-based caching, running the script every minute has minimal overhead. The script only downloads and reloads nginx when the configuration actually changes. Most runs will exit early with a 304 Not Modified response.
+**Note**: Thanks to ETag-based caching, running the script every minute has minimal overhead. After the first successful run, the script only downloads and reloads nginx when the configuration actually changes. Most runs will exit early with a 304 Not Modified response.
 
 ### Optimizations
 

From 379ae8db37d623e282ce784acdce2403d7a4cb30 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 28 Oct 2025 23:38:35 +0000
Subject: [PATCH 5/8] Refactor pull-config.sh per code review feedback

- Derive ETAG_FILE from CONF_FILE (line 6)
- Use global variables for TEMP_FILE, HEADERS_FILE, BACKUP_FILE (lines 7-9)
- Build curl command as array instead of conditionals (lines 26-42)
- Add cleanup() function with trap EXIT to ensure cleanup (lines 14-19)
- Update test scripts to match new etag filename (.conf.etag)

Co-authored-by: runleveldev <44057501+runleveldev@users.noreply.github.com>
---
 nginx-reverse-proxy/pull-config.sh      | 62 ++++++++++++++-----------
 nginx-reverse-proxy/test-pull-config.sh |  6 +--
 2 files changed, 39 insertions(+), 29 deletions(-)

diff --git a/nginx-reverse-proxy/pull-config.sh b/nginx-reverse-proxy/pull-config.sh
index 4453df23..ff0a1c9d 100755
--- a/nginx-reverse-proxy/pull-config.sh
+++ b/nginx-reverse-proxy/pull-config.sh
@@ -3,32 +3,47 @@
 set -euo pipefail
 
 CONF_FILE=/etc/nginx/conf.d/reverse-proxy.conf
-ETAG_FILE=/etc/nginx/conf.d/reverse-proxy.etag
+ETAG_FILE="${CONF_FILE}.etag"
+TEMP_FILE="${CONF_FILE}.tmp"
+HEADERS_FILE="${CONF_FILE}.headers"
+BACKUP_FILE="${CONF_FILE}.bak"
 CONF_URL=https://create-a-container.opensource.mieweb.org/nginx.conf
 FALLBACK_URL=http://create-a-container.cluster.mieweb.org:3000/nginx.conf
 
+# Cleanup function
+cleanup() {
+  rm -f "${TEMP_FILE}" "${HEADERS_FILE}" "${BACKUP_FILE}"
+}
+
+# Set trap to always cleanup on exit
+trap cleanup EXIT
+
 # Function to download config and extract ETag
 download_config() {
   local url="$1"
-  local temp_file="${CONF_FILE}.tmp"
-  local headers_file="${CONF_FILE}.headers"
   
-  # Read existing ETag if it exists
-  local etag_header=""
+  # Build curl command as array
+  local curl_cmd=(
+    curl
+    -w "%{http_code}"
+    -D "${HEADERS_FILE}"
+    -o "${TEMP_FILE}"
+    -sSL
+  )
+  
+  # Add ETag header if it exists
   if [[ -f "${ETAG_FILE}" ]]; then
     local etag
     etag=$(cat "${ETAG_FILE}")
-    etag_header="If-None-Match: ${etag}"
+    curl_cmd+=(-H "If-None-Match: ${etag}")
   fi
   
-  # Download with headers, capture HTTP status code
-  # Note: We don't use -f flag to allow handling of all HTTP status codes
+  # Add URL
+  curl_cmd+=("${url}")
+  
+  # Execute curl and capture HTTP status code
   local http_code
-  if [[ -n "${etag_header}" ]]; then
-    http_code=$(curl -w "%{http_code}" -D "${headers_file}" -H "${etag_header}" -o "${temp_file}" -sSL "${url}" 2>/dev/null || echo "000")
-  else
-    http_code=$(curl -w "%{http_code}" -D "${headers_file}" -o "${temp_file}" -sSL "${url}" 2>/dev/null || echo "000")
-  fi
+  http_code=$("${curl_cmd[@]}" 2>/dev/null || echo "000")
   
   # Return the http_code
   echo "${http_code}"
@@ -40,53 +55,48 @@ HTTP_CODE=$(download_config "${CONF_URL}")
 # Handle 502 error with fallback
 if [[ ${HTTP_CODE} -eq 502 ]] || [[ ${HTTP_CODE} -eq 000 ]]; then
   echo "Primary URL failed (HTTP ${HTTP_CODE}), trying fallback URL..." >&2
-  rm -f "${CONF_FILE}.tmp" "${CONF_FILE}.headers"
   HTTP_CODE=$(download_config "${FALLBACK_URL}")
 fi
 
 # Check if we got a 304 Not Modified
 if [[ ${HTTP_CODE} -eq 304 ]]; then
-  # No changes, clean up temp files and exit
-  rm -f "${CONF_FILE}.tmp" "${CONF_FILE}.headers"
+  # No changes, exit (cleanup handled by trap)
   exit 0
 fi
 
 # Check if we got a successful response
 if [[ ${HTTP_CODE} -ne 200 ]]; then
-  rm -f "${CONF_FILE}.tmp" "${CONF_FILE}.headers"
   echo "Failed to download configuration (HTTP ${HTTP_CODE})" >&2
   exit 1
 fi
 
 # Extract new ETag from headers
 NEW_ETAG=""
-if [[ -f "${CONF_FILE}.headers" ]]; then
-  NEW_ETAG=$(grep -i '^etag:' "${CONF_FILE}.headers" | sed 's/^etag: *//i' | tr -d '\r\n' || echo "")
+if [[ -f "${HEADERS_FILE}" ]]; then
+  NEW_ETAG=$(grep -i '^etag:' "${HEADERS_FILE}" | sed 's/^etag: *//i' | tr -d '\r\n' || echo "")
 fi
 
 # Backup existing config if it exists
 if [[ -f "${CONF_FILE}" ]]; then
-  mv "${CONF_FILE}" "${CONF_FILE}.bak"
+  mv "${CONF_FILE}" "${BACKUP_FILE}"
 fi
 
 # Move new config into place
-mv "${CONF_FILE}.tmp" "${CONF_FILE}"
+mv "${TEMP_FILE}" "${CONF_FILE}"
 
 # Test the new configuration
 if ! nginx -t; then
   # Restore backup if it exists
-  if [[ -f "${CONF_FILE}.bak" ]]; then
-    mv "${CONF_FILE}.bak" "${CONF_FILE}"
+  if [[ -f "${BACKUP_FILE}" ]]; then
+    mv "${BACKUP_FILE}" "${CONF_FILE}"
   else
     # No backup, just remove the bad config
     rm -f "${CONF_FILE}"
   fi
-  rm -f "${CONF_FILE}.headers"
   exit 1
 fi
 
-# Configuration is valid, clean up backup and save new ETag
-rm -f "${CONF_FILE}.bak" "${CONF_FILE}.headers"
+# Configuration is valid, save new ETag (cleanup handled by trap)
 if [[ -n "${NEW_ETAG}" ]]; then
   echo "${NEW_ETAG}" > "${ETAG_FILE}"
 fi
diff --git a/nginx-reverse-proxy/test-pull-config.sh b/nginx-reverse-proxy/test-pull-config.sh
index 478e1cf0..c9a38bfc 100755
--- a/nginx-reverse-proxy/test-pull-config.sh
+++ b/nginx-reverse-proxy/test-pull-config.sh
@@ -112,7 +112,7 @@ EOF
   if "${test_script}"; then
     if [[ -f "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf" ]]; then
       echo -e "${GREEN}✓ Test passed: Config file created${NC}"
-      if [[ -f "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.etag" ]]; then
+      if [[ -f "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf.etag" ]]; then
         echo -e "${GREEN}✓ Test passed: ETag file created${NC}"
       else
         echo -e "${RED}✗ Test failed: ETag file not created${NC}"
@@ -134,7 +134,7 @@ test_etag_cache_hit() {
   
   # Setup: Create existing config and ETag
   echo "# Existing config" > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"
-  echo '"mock-etag-12345"' > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.etag"
+  echo '"mock-etag-12345"' > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf.etag"
   
   # Create a modified version of the script
   local test_script="${TEST_DIR}/pull-config-etag-test.sh"
@@ -224,7 +224,7 @@ test_validation_rollback() {
   
   # Setup: Create existing valid config
   echo "# Valid existing config" > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf"
-  echo '"mock-etag-old"' > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.etag"
+  echo '"mock-etag-old"' > "${TEST_DIR}/etc/nginx/conf.d/reverse-proxy.conf.etag"
   
   # Create a modified version of the script
   local test_script="${TEST_DIR}/pull-config-rollback-test.sh"

From 1641cecf595c12cc64d10da753869858da9e175f Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 30 Oct 2025 15:00:14 +0000
Subject: [PATCH 6/8] Fix robustness and reduce cron noise

- Check CONF_FILE exists before using ETAG_FILE (line 35)
- Suppress nginx -t and reload output to avoid cron warnings (lines 88, 103)
- Handles edge case where etag exists but conf doesn't

Co-authored-by: runleveldev <44057501+runleveldev@users.noreply.github.com>
---
 nginx-reverse-proxy/pull-config.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/nginx-reverse-proxy/pull-config.sh b/nginx-reverse-proxy/pull-config.sh
index ff0a1c9d..222e6c48 100755
--- a/nginx-reverse-proxy/pull-config.sh
+++ b/nginx-reverse-proxy/pull-config.sh
@@ -31,8 +31,8 @@ download_config() {
     -sSL
   )
   
-  # Add ETag header if it exists
-  if [[ -f "${ETAG_FILE}" ]]; then
+  # Add ETag header if it exists and conf file exists
+  if [[ -f "${CONF_FILE}" ]] && [[ -f "${ETAG_FILE}" ]]; then
     local etag
     etag=$(cat "${ETAG_FILE}")
     curl_cmd+=(-H "If-None-Match: ${etag}")
@@ -84,8 +84,8 @@ fi
 # Move new config into place
 mv "${TEMP_FILE}" "${CONF_FILE}"
 
-# Test the new configuration
-if ! nginx -t; then
+# Test the new configuration (suppress output to avoid cron noise)
+if ! nginx -t >/dev/null 2>&1; then
   # Restore backup if it exists
   if [[ -f "${BACKUP_FILE}" ]]; then
     mv "${BACKUP_FILE}" "${CONF_FILE}"
@@ -100,4 +100,4 @@ fi
 if [[ -n "${NEW_ETAG}" ]]; then
   echo "${NEW_ETAG}" > "${ETAG_FILE}"
 fi
-nginx -s reload
\ No newline at end of file
+nginx -s reload >/dev/null 2>&1
\ No newline at end of file

From 10fa0823ea5da3bd3c18a72c5d17bfbea39c6aa3 Mon Sep 17 00:00:00 2001
From: Robert Gingras <rgingras@mieweb.com>
Date: Thu, 30 Oct 2025 15:35:41 +0000
Subject: [PATCH 7/8] Add tests for pull-config.sh: validate first run, ETag
 caching, and config rollback

---
 nginx-reverse-proxy/{ => tests}/test-502-fallback.sh | 0
 nginx-reverse-proxy/{ => tests}/test-pull-config.sh  | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename nginx-reverse-proxy/{ => tests}/test-502-fallback.sh (100%)
 rename nginx-reverse-proxy/{ => tests}/test-pull-config.sh (100%)

diff --git a/nginx-reverse-proxy/test-502-fallback.sh b/nginx-reverse-proxy/tests/test-502-fallback.sh
similarity index 100%
rename from nginx-reverse-proxy/test-502-fallback.sh
rename to nginx-reverse-proxy/tests/test-502-fallback.sh
diff --git a/nginx-reverse-proxy/test-pull-config.sh b/nginx-reverse-proxy/tests/test-pull-config.sh
similarity index 100%
rename from nginx-reverse-proxy/test-pull-config.sh
rename to nginx-reverse-proxy/tests/test-pull-config.sh

From 2ce75af50ccd7707554111786499bcac23d6d6f4 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Thu, 30 Oct 2025 18:14:44 +0000
Subject: [PATCH 8/8] Add 404 handling to fallback logic

- Also fallback to internal URL on 404 responses
- Handles case where unknown subdomains return 404 instead of 502

Co-authored-by: runleveldev <44057501+runleveldev@users.noreply.github.com>
---
 nginx-reverse-proxy/pull-config.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/nginx-reverse-proxy/pull-config.sh b/nginx-reverse-proxy/pull-config.sh
index 222e6c48..e305ed9c 100755
--- a/nginx-reverse-proxy/pull-config.sh
+++ b/nginx-reverse-proxy/pull-config.sh
@@ -52,8 +52,8 @@ download_config() {
 # Try primary URL
 HTTP_CODE=$(download_config "${CONF_URL}")
 
-# Handle 502 error with fallback
-if [[ ${HTTP_CODE} -eq 502 ]] || [[ ${HTTP_CODE} -eq 000 ]]; then
+# Handle 502, 404 error with fallback
+if [[ ${HTTP_CODE} -eq 502 ]] || [[ ${HTTP_CODE} -eq 404 ]] || [[ ${HTTP_CODE} -eq 000 ]]; then
   echo "Primary URL failed (HTTP ${HTTP_CODE}), trying fallback URL..." >&2
   HTTP_CODE=$(download_config "${FALLBACK_URL}")
 fi