From 37d0c16960ed7e22dcfbd6a92763336ea55e2214 Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Mon, 30 Jun 2025 15:00:20 -0700 Subject: [PATCH 1/3] Creation of prune_iptables.sh Solution for Issue 3 that does the following. --- intern-phxdc-pve1/prune_iptables.sh | 72 +++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 intern-phxdc-pve1/prune_iptables.sh diff --git a/intern-phxdc-pve1/prune_iptables.sh b/intern-phxdc-pve1/prune_iptables.sh new file mode 100644 index 00000000..92cfa9f6 --- /dev/null +++ b/intern-phxdc-pve1/prune_iptables.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +# Script to prune iptables rules for containers that no longer exist +# It fetches the port_map.json file from a remote host, checks for existing containers, and removes iptables rules for containers that are not present in the current Proxmox setup +# Location: /usr/local/bin/prune_iptables.sh +# Author: Carter Myers +set -euo pipefail + +# Remote source of port_map.json +REMOTE_HOST="intern-nginx" +REMOTE_FILE="/etc/nginx/port_map.json" +LOCAL_FILE="/tmp/port_map.json" +LOG_FILE="/var/log/prune_iptables.log" + +# List of Proxmox nodes in the cluster (add more if needed) +PVE_NODES=("localhost" "intern-phxdc-pve2") + +# === 1. Fetch port_map.json === +echo "[$(date)] Fetching port_map.json from $REMOTE_HOST..." >> "$LOG_FILE" +if ! scp "$REMOTE_HOST:$REMOTE_FILE" "$LOCAL_FILE" >/dev/null 2>&1; then + echo "$(date '+%Y-%m-%d %H:%M:%S') - ERROR: Could not fetch $REMOTE_FILE from $REMOTE_HOST" >> "$LOG_FILE" + exit 1 +fi + +# === 2. Build list of existing hostnames across all PVE nodes === +EXISTING_HOSTNAMES="" +for node in "${PVE_NODES[@]}"; do + echo "[$(date)] Checking containers on $node..." >> "$LOG_FILE" + if [[ "$node" == "localhost" ]]; then + CTIDS=$(pct list | awk 'NR>1 {print $1}') + for id in $CTIDS; do + hn=$(pct config "$id" 2>/dev/null | grep -i '^hostname:' | awk '{print $2}') + [[ -n "$hn" ]] && EXISTING_HOSTNAMES+="$hn"$'\n' + done + else + CTIDS=$(ssh "$node" "pct list | awk 'NR>1 {print \\$1}'" 2>/dev/null || true) + for id in $CTIDS; do + hn=$(ssh "$node" "pct config $id 2>/dev/null | grep -i '^hostname:' | awk '{print \\$2}'" 2>/dev/null || true) + [[ -n "$hn" ]] && EXISTING_HOSTNAMES+="$hn"$'\n' + done + fi +done + +# === 3. Delete iptables rules from PREROUTING and POSTROUTING === +delete_rules_for_ip() { + local ip="$1" + local chain="$2" + local hostname="$3" + + while true; do + match=$(iptables -t nat -L "$chain" -n --line-numbers | grep "$ip" | head -n 1 || true) + [[ -z "$match" ]] && break + + rule_num=$(echo "$match" | awk '{print $1}') + port=$(echo "$match" | grep -oP 'dpt:\K[0-9]+' || echo "N/A") + date_str=$(date '+%Y-%m-%d %H:%M:%S') + + echo "$date_str - Removed '$hostname' $ip : $port from $chain table" >> "$LOG_FILE" + iptables -t nat -D "$chain" "$rule_num" + done +} + +# === 4. Process each entry in port_map.json === +jq -r 'keys[]' "$LOCAL_FILE" | while read hostname; do + if ! echo "$EXISTING_HOSTNAMES" | grep -iqx "$hostname"; then + echo "[$(date)] Hostname $hostname not found on any node. Cleaning up iptables..." >> "$LOG_FILE" + ip=$(jq -r --arg hn "$hostname" '.[$hn].ip' "$LOCAL_FILE") + + delete_rules_for_ip "$ip" "PREROUTING" "$hostname" + delete_rules_for_ip "$ip" "POSTROUTING" "$hostname" + fi +done \ No newline at end of file From f9c2291b60fc73b78bda521d01df9636b428d177 Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Tue, 8 Jul 2025 09:49:39 -0700 Subject: [PATCH 2/3] Fix issues related to failed prunes --- intern-phxdc-pve1/prune_iptables.sh | 173 ++++++++++++++++++++++------ 1 file changed, 137 insertions(+), 36 deletions(-) diff --git a/intern-phxdc-pve1/prune_iptables.sh b/intern-phxdc-pve1/prune_iptables.sh index 92cfa9f6..e647f269 100644 --- a/intern-phxdc-pve1/prune_iptables.sh +++ b/intern-phxdc-pve1/prune_iptables.sh @@ -1,72 +1,173 @@ #!/bin/bash # Script to prune iptables rules for containers that no longer exist -# It fetches the port_map.json file from a remote host, checks for existing containers, and removes iptables rules for containers that are not present in the current Proxmox setup -# Location: /usr/local/bin/prune_iptables.sh # Author: Carter Myers + +# Enable strict mode: +# -e: Exit immediately if a command exits with a non-zero status. +# -u: Treat unset variables as an error when substituting. +# -o pipefail: The return value of a pipeline is the status of the last command +# to exit with a non-zero status, or zero if all commands exit successfully. set -euo pipefail -# Remote source of port_map.json +# --- Configuration --- REMOTE_HOST="intern-nginx" REMOTE_FILE="/etc/nginx/port_map.json" LOCAL_FILE="/tmp/port_map.json" LOG_FILE="/var/log/prune_iptables.log" +PVE_NODES=("localhost" "10.15.0.5") -# List of Proxmox nodes in the cluster (add more if needed) -PVE_NODES=("localhost" "intern-phxdc-pve2") +# Function to log messages with a timestamp +log_message() { + echo "[$(date)] $1" >> "$LOG_FILE" +} -# === 1. Fetch port_map.json === -echo "[$(date)] Fetching port_map.json from $REMOTE_HOST..." >> "$LOG_FILE" +# --- 1. Fetch port_map.json from remote host --- +log_message "Fetching port_map.json from $REMOTE_HOST..." if ! scp "$REMOTE_HOST:$REMOTE_FILE" "$LOCAL_FILE" >/dev/null 2>&1; then - echo "$(date '+%Y-%m-%d %H:%M:%S') - ERROR: Could not fetch $REMOTE_FILE from $REMOTE_HOST" >> "$LOG_FILE" + log_message "ERROR: Could not fetch $REMOTE_FILE from $REMOTE_HOST" exit 1 fi +log_message "Successfully fetched $REMOTE_FILE to $LOCAL_FILE." -# === 2. Build list of existing hostnames across all PVE nodes === +# --- 2. Build list of existing hostnames --- EXISTING_HOSTNAMES="" for node in "${PVE_NODES[@]}"; do - echo "[$(date)] Checking containers on $node..." >> "$LOG_FILE" + log_message "Checking containers on $node..." if [[ "$node" == "localhost" ]]; then - CTIDS=$(pct list | awk 'NR>1 {print $1}') + CTIDS=$(pct list | awk 'NR>1 {print $1}' || true) + log_message "DEBUG: Local CTIDs: [${CTIDS:-}]" for id in $CTIDS; do - hn=$(pct config "$id" 2>/dev/null | grep -i '^hostname:' | awk '{print $2}') + hn=$(pct config "$id" 2>/dev/null | grep -i '^hostname:' | awk '{print $2}' | tr -d '[:space:]' || true) [[ -n "$hn" ]] && EXISTING_HOSTNAMES+="$hn"$'\n' done else - CTIDS=$(ssh "$node" "pct list | awk 'NR>1 {print \\$1}'" 2>/dev/null || true) - for id in $CTIDS; do - hn=$(ssh "$node" "pct config $id 2>/dev/null | grep -i '^hostname:' | awk '{print \\$2}'" 2>/dev/null || true) + log_message "DEBUG: Checking remote node: $node" + CTIDS_CMD="pct list | awk 'NR>1 {print \$1}'" + CTIDS_OUTPUT=$(ssh "$node" "$CTIDS_CMD" 2>&1 || true) + if [[ "$CTIDS_OUTPUT" =~ "Permission denied" || "$CTIDS_OUTPUT" =~ "Connection refused" || "$CTIDS_OUTPUT" =~ "Host key verification failed" ]]; then + log_message "ERROR: SSH to $node failed: $CTIDS_OUTPUT" + continue + fi + log_message "DEBUG: CTIDs on $node: [${CTIDS_OUTPUT:-}]" + for id in $CTIDS_OUTPUT; do + HN_CMD="pct config $id 2>/dev/null | grep -i '^hostname:' | awk '{print \$2}'" + HN_OUTPUT=$(ssh "$node" "$HN_CMD" 2>&1 || true) + if [[ "$HN_OUTPUT" =~ "Permission denied" || "$HN_OUTPUT" =~ "No such file" ]]; then + log_message "ERROR: Failed to get hostname for $id on $node: $HN_OUTPUT" + continue + fi + hn=$(echo "$HN_OUTPUT" | tr -d '[:space:]') [[ -n "$hn" ]] && EXISTING_HOSTNAMES+="$hn"$'\n' done fi done -# === 3. Delete iptables rules from PREROUTING and POSTROUTING === -delete_rules_for_ip() { - local ip="$1" - local chain="$2" - local hostname="$3" +# Remove any empty lines from EXISTING_HOSTNAMES +EXISTING_HOSTNAMES=$(echo "$EXISTING_HOSTNAMES" | sed '/^$/d') +log_message "Existing hostnames collected:" +log_message "$EXISTING_HOSTNAMES" - while true; do - match=$(iptables -t nat -L "$chain" -n --line-numbers | grep "$ip" | head -n 1 || true) - [[ -z "$match" ]] && break +# --- 3. Prune iptables and port_map.json --- +log_message "Pruning iptables and port_map.json..." +cp "$LOCAL_FILE" "$LOCAL_FILE.bak" +log_message "Created backup of $LOCAL_FILE at $LOCAL_FILE.bak" - rule_num=$(echo "$match" | awk '{print $1}') - port=$(echo "$match" | grep -oP 'dpt:\K[0-9]+' || echo "N/A") - date_str=$(date '+%Y-%m-%d %H:%M:%S') +HOSTNAMES_IN_JSON=$(jq -r 'keys[]' "$LOCAL_FILE") +mapfile -t EXISTING_ARRAY <<< "$EXISTING_HOSTNAMES" - echo "$date_str - Removed '$hostname' $ip : $port from $chain table" >> "$LOG_FILE" - iptables -t nat -D "$chain" "$rule_num" +# Helper function to check if a hostname exists in the collected list +hostname_exists() { + local h=$(echo "$1" | tr -d '[:space:]') + for existing in "${EXISTING_ARRAY[@]}"; do + if [[ "${h,,}" == "${existing,,}" ]]; then # Case-insensitive comparison + return 0 + fi done + return 1 } -# === 4. Process each entry in port_map.json === -jq -r 'keys[]' "$LOCAL_FILE" | while read hostname; do - if ! echo "$EXISTING_HOSTNAMES" | grep -iqx "$hostname"; then - echo "[$(date)] Hostname $hostname not found on any node. Cleaning up iptables..." >> "$LOG_FILE" - ip=$(jq -r --arg hn "$hostname" '.[$hn].ip' "$LOCAL_FILE") +for hostname in $HOSTNAMES_IN_JSON; do + trimmed_hostname=$(echo "$hostname" | tr -d '[:space:]') + if hostname_exists "$trimmed_hostname"; then + log_message "Keeping entry: $trimmed_hostname" + else + ip=$(jq -r --arg h "$hostname" '.[$h].ip // "unknown"' "$LOCAL_FILE") + ports=$(jq -c --arg h "$hostname" '.[$h].ports // {}' "$LOCAL_FILE") + log_message "Stale entry detected: $hostname (IP: $ip, Ports: $ports) - removing..." + + # --- IPTABLES REMOVAL --- + # Capture rules into an array first to avoid subshell issues with 'while read' + mapfile -t RULES_TO_DELETE < <(sudo iptables -t nat -S | grep "$hostname" || true) # Added sudo, || true to prevent pipefail if grep finds nothing + + if [[ ${#RULES_TO_DELETE[@]} -gt 0 ]]; then + log_message "Found ${#RULES_TO_DELETE[@]} iptables rules for $hostname. Attempting removal..." + for rule in "${RULES_TO_DELETE[@]}"; do + cleaned_rule=$(echo "$rule" | sed 's/^-A /-D /') + log_message "Attempting to remove iptables rule: sudo iptables -t nat $cleaned_rule" + if sudo iptables -t nat $cleaned_rule; then + log_message "Removed iptables rule: $cleaned_rule" + else + log_message "ERROR: Failed to remove iptables rule: $cleaned_rule (Exit status: $?)" + fi + done + else + log_message "No iptables rules found for $hostname to remove." + fi + + # --- JSON ENTRY REMOVAL --- + log_message "Attempting to remove $hostname from local port_map.json..." + if jq "del(.\"$hostname\")" "$LOCAL_FILE" > "${LOCAL_FILE}.tmp"; then + if mv "${LOCAL_FILE}.tmp" "$LOCAL_FILE"; then + log_message "Successfully removed $hostname from local port_map.json." + else + log_message "ERROR: Failed to move temporary file to $LOCAL_FILE for $hostname." + exit 1 # Critical failure, exit + fi + else + log_message "ERROR: jq failed to delete $hostname from $LOCAL_FILE." + exit 1 # Critical failure, exit + fi + + # Confirm deletion from local file + if jq -e --arg h "$hostname" 'has($h)' "$LOCAL_FILE" >/dev/null; then + log_message "ERROR: $hostname still exists in local port_map.json after deletion attempt!" + else + log_message "Confirmed $hostname removed from local port_map.json." + fi + fi +done + +# --- 4. Upload and verify updated file on remote --- +log_message "Uploading updated port_map.json to $REMOTE_HOST..." +TEMP_REMOTE="/tmp/port_map.json" - delete_rules_for_ip "$ip" "PREROUTING" "$hostname" - delete_rules_for_ip "$ip" "POSTROUTING" "$hostname" +if scp "$LOCAL_FILE" "$REMOTE_HOST:$TEMP_REMOTE" >/dev/null 2>&1; then + log_message "Uploaded to $REMOTE_HOST:$TEMP_REMOTE" +else + log_message "ERROR: Failed to upload $TEMP_REMOTE to $REMOTE_HOST" + exit 1 +fi + +# Check if deleted hostnames still exist in uploaded file +log_message "Verifying remote file content..." +for hostname in $HOSTNAMES_IN_JSON; do + if ! hostname_exists "$hostname"; then # Only check for hostnames that *should* have been deleted + if ssh "$REMOTE_HOST" "grep -q '\"$hostname\"' $TEMP_REMOTE"; then + log_message "WARNING: $hostname still exists in uploaded $TEMP_REMOTE on $REMOTE_HOST!" + else + log_message "Verified $hostname was removed in uploaded file on $REMOTE_HOST." + fi fi -done \ No newline at end of file +done + +# Move uploaded file into place on the remote host +log_message "Moving uploaded file into final position on $REMOTE_HOST..." +if ssh "$REMOTE_HOST" "sudo cp $TEMP_REMOTE $REMOTE_FILE && sudo chown root:root $REMOTE_FILE && sudo chmod 644 $REMOTE_FILE && rm $TEMP_REMOTE"; then + log_message "Copied updated port_map.json to $REMOTE_FILE on $REMOTE_HOST" +else + log_message "ERROR: Failed to replace $REMOTE_FILE on $REMOTE_HOST" + exit 1 +fi + +log_message "Prune complete." \ No newline at end of file From ca4e65e59970e4889909d5d4c95ffc48ad31dc01 Mon Sep 17 00:00:00 2001 From: Carter Myers <206+cmyers@users.noreply.github.mieweb.com> Date: Thu, 10 Jul 2025 14:47:48 -0700 Subject: [PATCH 3/3] Fix iptable rule check skip --- intern-phxdc-pve1/prune_iptables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intern-phxdc-pve1/prune_iptables.sh b/intern-phxdc-pve1/prune_iptables.sh index e647f269..abc1aec2 100644 --- a/intern-phxdc-pve1/prune_iptables.sh +++ b/intern-phxdc-pve1/prune_iptables.sh @@ -98,7 +98,7 @@ for hostname in $HOSTNAMES_IN_JSON; do # --- IPTABLES REMOVAL --- # Capture rules into an array first to avoid subshell issues with 'while read' - mapfile -t RULES_TO_DELETE < <(sudo iptables -t nat -S | grep "$hostname" || true) # Added sudo, || true to prevent pipefail if grep finds nothing + mapfile -t RULES_TO_DELETE < <(sudo iptables -t nat -S | grep -w "$ip" || true) # Added sudo, || true to prevent pipefail if grep finds nothing if [[ ${#RULES_TO_DELETE[@]} -gt 0 ]]; then log_message "Found ${#RULES_TO_DELETE[@]} iptables rules for $hostname. Attempting removal..."