From f7e20cb684e1ff7383c2c8c947c8a32c1231df92 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sun, 17 Aug 2025 13:55:10 -0400 Subject: [PATCH 01/13] root_start_command fixes for launchpad --- ci-cd automation/Proxmox-Launchpad | 2 +- ci-cd automation/check-container-exists.sh | 8 +++---- ci-cd automation/update-container.sh | 26 ++++++++++++++-------- container creation/create-container.sh | 9 ++++---- container creation/start_services.sh | 10 +++++++-- 5 files changed, 35 insertions(+), 20 deletions(-) diff --git a/ci-cd automation/Proxmox-Launchpad b/ci-cd automation/Proxmox-Launchpad index 038aff5a..4a77ef7f 160000 --- a/ci-cd automation/Proxmox-Launchpad +++ b/ci-cd automation/Proxmox-Launchpad @@ -1 +1 @@ -Subproject commit 038aff5ad0eacd9f77935ae8819ab59da13fc981 +Subproject commit 4a77ef7f87efddceaaeaf6dd00915e8718c67417 diff --git a/ci-cd automation/check-container-exists.sh b/ci-cd automation/check-container-exists.sh index 77c8fd8f..791b24ee 100644 --- a/ci-cd automation/check-container-exists.sh +++ b/ci-cd automation/check-container-exists.sh @@ -5,7 +5,7 @@ outputError() { echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" + echo -e "${BOLD}${MAGENTA} Script Failed. Exiting... ${RESET}" echo -e "$2" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" exit $1 @@ -32,15 +32,15 @@ REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") # Check if repository folder is present. if [ "$PVE1" == "true" ]; then - if pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then + if [ ! -z $CONTAINER_ID ] && pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then exit 2; # Update Repository else exit 0; # Clone Repository fi else - if ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log"; then + if [ ! -z $CONTAINER_ID ] && ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log" ; then exit 2; # Update Repository else exit 0; # Clone Repository fi -fi \ No newline at end of file +fi diff --git a/ci-cd automation/update-container.sh b/ci-cd automation/update-container.sh index 484f4996..c8522e6c 100644 --- a/ci-cd automation/update-container.sh +++ b/ci-cd automation/update-container.sh @@ -3,6 +3,8 @@ # Last Modified on August 5th, 2025 by Maxwell Klema # ---------------------------------------- +set -x + RESET="\033[0m" BOLD="\033[1m" MAGENTA='\033[35m' @@ -148,15 +150,15 @@ startComponentPVE1() { INSTALL_CMD="$5" if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD && $BUILD_CMD" > /dev/null 2>&1 - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 elif [ "${RUNTIME^^}" == "PYTHON" ]; then - pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 + pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull" > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD && $BUILD_CMD" > /dev/null 2>&1 - pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 + pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 fi } @@ -174,14 +176,14 @@ startComponentPVE2() { pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/ && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " + " elif [ "${RUNTIME^^}" == "PYTHON" ]; then ssh root@10.15.0.5 " pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 && pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && git fetch origin && git reset --hard origin/$PROJECT_BRANCH && git pull' > /dev/null 2>&1 pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $INSTALL_CMD' && '$BUILD_CMD' > /dev/null 2>&1 pct set $CONTAINER_ID --memory 2048 --swap 0 --cores 2 - " + " fi } @@ -208,17 +210,23 @@ if [ "${MULTI_COMPONENT^^}" == "Y" ]; then done if [ ! -z "$ROOT_START_COMMAND" ]; then if (( $CONTAINER_ID % 2 == 0 )); then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" else - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" fi fi # startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." else if (( $CONTAINER_ID % 2 == 0 )); then startComponentPVE2 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." "$INSTALL_COMMAND" + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi else startComponentPVE1 "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." "$INSTALL_COMMAND" + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi fi fi @@ -262,5 +270,5 @@ QUOTED_CMD=$(printf ' %q' "${CMD[@]}") tmux new-session -d -s "$CONTAINER_NAME" "$QUOTED_CMD" echo "✅ Container $CONTAINER_ID has been updated with new contents from branch \"$PROJECT_BRANCH\" on repository \"$PROJECT_REPOSITORY\"." +echo "Wait a few minutes for all background processes to complete before accessing the container." exit 0 - diff --git a/container creation/create-container.sh b/container creation/create-container.sh index 16013e6e..66ac88c2 100644 --- a/container creation/create-container.sh +++ b/container creation/create-container.sh @@ -42,7 +42,7 @@ echoContainerDetails() { echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${BLUE}Still not working? Contact Max K. at maxklema@gmail.com${RESET}" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - + } trap cleanup SIGINT SIGTERM SIGHUP @@ -71,6 +71,7 @@ SERVICES_BASE_FILE="${16}" LINUX_DISTRO="${17}" MULTI_COMPONENTS="${18}" ROOT_START_COMMAND="${19}" +SELF_HOSTED_RUNNER="${20}" # Pick the correct template to clone ===== @@ -96,7 +97,7 @@ fi # Create the Container Clone ==== -if [ "${GH_ACTION^^}" != "Y" ]; then +if [ "${GH_ACTION^^}" != "Y" ] || [ "${SELF_HOSTED_RUNNER^^}" == "N" ]; then CONTAINER_ID=$(pvesh get /cluster/nextid) #Get the next available LXC ID echo "⏳ Cloning Container..." @@ -178,10 +179,10 @@ pct exec $CONTAINER_ID -- bash -c "cd /root && touch container-updates.log" # Run Contianer Provision Script to add container to port_map.json echo "⏳ Running Container Provision Script..." if [ -f "/var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE" ]; then - /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE "$USERNAME_ONLY" "$ROOT_PSWD" + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE "$USERNAME_ONLY" rm -rf /var/lib/vz/snippets/container-port-maps/$PROTOCOL_FILE > /dev/null 2>&1 else - /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT "" "$PROXMOX_USERNAME" "$ROOT_PSWD" + /var/lib/vz/snippets/register-container.sh $CONTAINER_ID $HTTP_PORT "" "$PROXMOX_USERNAME" fi SSH_PORT=$(iptables -t nat -S PREROUTING | grep "to-destination $CONTAINER_IP:22" | awk -F'--dport ' '{print $2}' | awk '{print $1}' | head -n 1 || true) diff --git a/container creation/start_services.sh b/container creation/start_services.sh index 7942ad02..46ccfc50 100644 --- a/container creation/start_services.sh +++ b/container creation/start_services.sh @@ -48,7 +48,7 @@ if (( $CONTAINER_ID % 2 == 0 )); then if [ "${GH_ACTION^^}" == "Y" ]; then ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- systemctl start github-runner" - fi + fi startProject() { @@ -85,6 +85,9 @@ if (( $CONTAINER_ID % 2 == 0 )); then fi else startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if [ ! -z "$ROOT_START_COMMAND" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi fi fi @@ -94,7 +97,7 @@ else sleep 5 if [ "${GH_ACTION^^}" == "Y" ]; then pct exec $CONTAINER_ID -- bash -c "systemctl start github-runner" - fi + fi startComponent() { @@ -131,5 +134,8 @@ else fi else startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if [ ! -z "$ROOT_START_COMMAND" ]; then + pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi fi fi From 99b7d85b2e87ad7be226b5f712b5f9b8d231b846 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sun, 17 Aug 2025 14:45:54 -0400 Subject: [PATCH 02/13] single component env variable are appended on each push --- ci-cd automation/update-container.sh | 57 +++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/ci-cd automation/update-container.sh b/ci-cd automation/update-container.sh index c8522e6c..535ac3ea 100644 --- a/ci-cd automation/update-container.sh +++ b/ci-cd automation/update-container.sh @@ -67,7 +67,7 @@ if [ "$REPOSITORY_BRANCH_EXISTS" != "200" ]; then fi -# # Get Project Root Directroy +# Get Project Root Directroy if [ "$PROJECT_ROOT" == "." ] || [ -z "$PROJECT_ROOT" ]; then PROJECT_ROOT="/" @@ -118,7 +118,6 @@ if [ ! -z "$HTTP_PORT" ]; then && mv -f /tmp/port_map.json.new /etc/nginx/port_map.json " fi - # Clone repository if needed ==== if (( "$CONTAINER_ID" % 2 == 0 )); then @@ -139,6 +138,60 @@ fi EOF fi +# Update Environment Variables + +# Helper Function to write environment variables to a file inside container +writeEnvToFile() { + env_file_path="$1" + component_path="$2" + env_vars=$(cat "$env_file_path") + if (( $CONTAINER_ID % 2 == 0)); then + ssh 10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'touch \"$component_path/.env\" && echo \"$env_vars\" > \"$component_path/.env\"'" + else + pct exec $CONTAINER_ID -- bash -c "touch \"$component_path/.env\" && echo \"$env_vars\" > \"$component_path/.env\"" + fi +} + +# Check If there are environment variables +if [ ! -z "$CONTAINER_ENV_VARS" ]; then + # generate random temp .env folder to store all env files for different components + RANDOM_NUM=$(shuf -i 100000-999999 -n 1) + ENV_FOLDER="env_$RANDOM_NUM" + ENV_FOLDER_PATH="/var/lib/vz/snippets/container-env-vars/$ENV_FOLDER" + mkdir -p "$ENV_FOLDER_PATH" + + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then # Multi-Component + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + for key in $(echo "$CONTAINER_ENV_VARS" | jq -r 'keys[]'); do + COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT/$key" + ENV_FILE_NAME=$(echo "$COMPONENT_PATH" | tr '/' '_') + ENV_FILE_NAME="$ENV_FILE_NAME.txt" + ENV_FILE_PATH="$ENV_FOLDER_PATH/$ENV_FILE_NAME" + touch "$ENV_FILE_PATH" + echo "$CONTAINER_ENV_VARS" | jq -r --arg key "$key" '.[$key] | to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" + writeEnvToFile "$ENV_FILE_PATH" "$COMPONENT_PATH" + done + else + outputError "Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in CONTAINER_ENV_VARS (GH_ACTION mode)" + exit 16 + fi + else # Single Component + ENV_FILE="env_$RANDOM_NUM.txt" + ENV_FILE_PATH="$ENV_FOLDER_PATH/$ENV_FILE" + touch "$ENV_FILE_PATH" + if echo "$CONTAINER_ENV_VARS" | jq -e > /dev/null 2>&1; then #if exit status of jq is 0 (valid JSON) // success + COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT" + echo "$CONTAINER_ENV_VARS " | jq -r 'to_entries[] | "\(.key)=\(.value)"' > "$ENV_FILE_PATH" #k=v pairs + writeEnvToFile "$ENV_FILE_PATH" "$COMPONENT_PATH" + else + outputError "Your \"CONTAINER_ENV_VARS\" is not valid JSON. Please re-format and try again." + writeLog "Invalid JSON in CONTAINER_ENV_VARS for single component (GH_ACTION mode)" + exit 16 + fi + fi +fi + # Update Container with New Contents from repository ===== startComponentPVE1() { From 19b8af9c7a092bd775050a4b3e7e26add3f716f8 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sun, 17 Aug 2025 15:48:29 -0400 Subject: [PATCH 03/13] update container script updating env variabless --- ci-cd automation/update-container.sh | 44 +++++++++++++--------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/ci-cd automation/update-container.sh b/ci-cd automation/update-container.sh index 535ac3ea..edd3af54 100644 --- a/ci-cd automation/update-container.sh +++ b/ci-cd automation/update-container.sh @@ -3,8 +3,6 @@ # Last Modified on August 5th, 2025 by Maxwell Klema # ---------------------------------------- -set -x - RESET="\033[0m" BOLD="\033[1m" MAGENTA='\033[35m' @@ -88,23 +86,23 @@ fi # Install Services ==== -echo "🛎️ Installing Services..." +# echo "🛎️ Installing Services..." -if [ -z "$LINUX_DISTRIBUTION" ]; then - LINUX_DISTRIBUTION="debian" -fi +# if [ -z "$LINUX_DISTRIBUTION" ]; then +# LINUX_DISTRIBUTION="debian" +# fi -if [ ! -z "$SERVICES" ] || [ ! -z "$CUSTOM_SERVICES" ]; then - REQUIRE_SERVICES="y" -fi +# if [ ! -z "$SERVICES" ] || [ ! -z "$CUSTOM_SERVICES" ]; then +# REQUIRE_SERVICES="y" +# fi -SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ - root@10.15.234.122 \ - "/root/bin/deployment-scripts/gatherServices.sh true") +# SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ +# root@10.15.234.122 \ +# "/root/bin/deployment-scripts/gatherServices.sh true") -echo "$SERVICE_COMMANDS" | while read -r line; do - pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 -done +# echo "$SERVICE_COMMANDS" | while read -r line; do +# pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 +# done # Change HTTP port if necessary ==== @@ -140,15 +138,19 @@ fi # Update Environment Variables +if [ ! -z "$RUNTIME_LANGUAGE" ] && echo "$RUNTIME_LANGUAGE" | jq . >/dev/null 2>&1; then # If RUNTIME_LANGUAGE is set and is valid JSON + MULTI_COMPONENT="Y" +fi + # Helper Function to write environment variables to a file inside container writeEnvToFile() { env_file_path="$1" component_path="$2" env_vars=$(cat "$env_file_path") - if (( $CONTAINER_ID % 2 == 0)); then - ssh 10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'touch \"$component_path/.env\" && echo \"$env_vars\" > \"$component_path/.env\"'" + if (( $CONTAINER_ID % 2 == 0 )); then + ssh 10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'if [ -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"'" else - pct exec $CONTAINER_ID -- bash -c "touch \"$component_path/.env\" && echo \"$env_vars\" > \"$component_path/.env\"" + pct exec $CONTAINER_ID -- bash -c "if [ -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"" fi } @@ -192,6 +194,7 @@ if [ ! -z "$CONTAINER_ENV_VARS" ]; then fi fi +exit 0 # Update Container with New Contents from repository ===== startComponentPVE1() { @@ -240,11 +243,6 @@ startComponentPVE2() { fi } - -if [ ! -z "$RUNTIME_LANGUAGE" ] && echo "$RUNTIME_LANGUAGE" | jq . >/dev/null 2>&1; then # If RUNTIME_LANGUAGE is set and is valid JSON - MULTI_COMPONENT="Y" -fi - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') From 10114a954c96ea3cd65fbbe38bf61ea15a65e128 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sun, 17 Aug 2025 16:04:38 -0400 Subject: [PATCH 04/13] fixed deployOnStart bug that did not write env variables to container --- container creation/deployOnStart.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh index 82d01a62..519b0acf 100755 --- a/container creation/deployOnStart.sh +++ b/container creation/deployOnStart.sh @@ -1,6 +1,6 @@ #!/bin/bash # Automation Script for attempting to automatically deploy projects and services on a container -# Last Modifided by Maxwell Klema on July 16th, 2025 +# Last Modifided by Maxwell Klema on August 16th, 2025 # ----------------------------------------------------- echo "🚀 Attempting Automatic Deployment" @@ -13,12 +13,12 @@ echo "Repo base name: $REPO_BASE_NAME" pct enter $CONTAINER_ID < /dev/null 2>&1 && \ cd /root/$REPO_BASE_NAME && \ -git checkout $PROJECT_BRANCH > /dev/null +git checkout $PROJECT_BRANCH > /dev/null 2>&1 else -cd /root/$REPO_BASE_NAME && git fetch && git pull && \ -git checkout $PROJECT_BRANCH +cd /root/$REPO_BASE_NAME && git fetch > /dev/null 2>&1 && git pull > /dev/null 2>&1 && \ +git checkout $PROJECT_BRANCH > /dev/null 2>&1 fi EOF @@ -28,20 +28,22 @@ pct exec $CONTAINER_ID -- bash -c "chmod 700 ~/.bashrc" # enable full R/W/X perm ENV_BASE_FOLDER="/var/lib/vz/snippets/container-env-vars/${ENV_BASE_FOLDER}" -if [ ! -d "$ENV_BASE_FOLDER"]; then +if [ -d "$ENV_BASE_FOLDER" ]; then if [ "${MULTI_COMPONENTS^^}" == "Y" ]; then for FILE in $ENV_BASE_FOLDER/*; do FILE_BASENAME=$(basename "$FILE") FILE_NAME="${FILE_BASENAME%.*}" ENV_ROUTE=$(echo "$FILE_NAME" | tr '_' '/') # acts as the route to the correct folder to place .env file in. - + ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE" + pct exec $CONTAINER_ID -- bash -c "if [ -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" > \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 done else ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt || true) - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && echo "$ENV_VARS" > .env" > /dev/null 2>&1 + COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT" + pct exec $CONTAINER_ID -- bash -c "if [ -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" > \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 fi fi @@ -83,4 +85,4 @@ if [ -f "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" ]; then while read line; do pct exec $CONTAINER_ID -- bash -c "$line" > /dev/null 2>&1 done < "/var/lib/vz/snippets/container-services/$SERVICES_BASE_FILE" -fi \ No newline at end of file +fi From f170754ffe13b6ce9d34c68aeacac1e2313a4095 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sun, 17 Aug 2025 23:27:09 -0400 Subject: [PATCH 05/13] updates for proxmox launchpad --- .../setup-runner.sh | 4 +-- ci-cd automation/update-container.sh | 29 +++++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) rename {container creation => ci-cd automation}/setup-runner.sh (99%) diff --git a/container creation/setup-runner.sh b/ci-cd automation/setup-runner.sh similarity index 99% rename from container creation/setup-runner.sh rename to ci-cd automation/setup-runner.sh index 382f4f87..ef2cda04 100644 --- a/container creation/setup-runner.sh +++ b/ci-cd automation/setup-runner.sh @@ -7,7 +7,7 @@ outputError() { echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" echo -e "$2" - echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" exit $1 } @@ -26,7 +26,7 @@ source /var/lib/vz/snippets/helper-scripts/verify_container_ownership.sh #Ensure if [ ! -z "$CONTAINER_OWNERSHIP" ]; then outputError 1 "You already own a container with name \"$CONTAINER_NAME\". Please delete it before creating a new one." fi - + # Cloning Container Template and Setting it up ===== REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") diff --git a/ci-cd automation/update-container.sh b/ci-cd automation/update-container.sh index edd3af54..562f72d1 100644 --- a/ci-cd automation/update-container.sh +++ b/ci-cd automation/update-container.sh @@ -1,6 +1,6 @@ #!/bin/bash # Script to automatically fetch new contents from a branch, push them to container, and restart intern -# Last Modified on August 5th, 2025 by Maxwell Klema +# Last Modified on August 17th, 2025 by Maxwell Klema # ---------------------------------------- RESET="\033[0m" @@ -86,23 +86,23 @@ fi # Install Services ==== -# echo "🛎️ Installing Services..." +echo "🛎️ Installing Services..." -# if [ -z "$LINUX_DISTRIBUTION" ]; then -# LINUX_DISTRIBUTION="debian" -# fi +if [ -z "$LINUX_DISTRIBUTION" ]; then + LINUX_DISTRIBUTION="debian" +fi -# if [ ! -z "$SERVICES" ] || [ ! -z "$CUSTOM_SERVICES" ]; then -# REQUIRE_SERVICES="y" -# fi +if [ ! -z "$SERVICES" ] || [ ! -z "$CUSTOM_SERVICES" ]; then + REQUIRE_SERVICES="y" +fi -# SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ -# root@10.15.234.122 \ -# "/root/bin/deployment-scripts/gatherServices.sh true") +SERVICE_COMMANDS=$(ssh -o SendEnv="LINUX_DISTRIBUTION SERVICES CUSTOM_SERVICES REQUIRE_SERVICES" \ + root@10.15.234.122 \ + "/root/bin/deployment-scripts/gatherServices.sh true") -# echo "$SERVICE_COMMANDS" | while read -r line; do -# pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 -# done +echo "$SERVICE_COMMANDS" | while read -r line; do + pct exec $CONTAINER_ID -- bash -c "$line | true" > /dev/null 2>&1 +done # Change HTTP port if necessary ==== @@ -194,7 +194,6 @@ if [ ! -z "$CONTAINER_ENV_VARS" ]; then fi fi -exit 0 # Update Container with New Contents from repository ===== startComponentPVE1() { From 6857cba61628a5b08e68f04f23f0875f6b23778b Mon Sep 17 00:00:00 2001 From: maxklema Date: Mon, 18 Aug 2025 00:40:21 -0400 Subject: [PATCH 06/13] scripts for scraping runtime environment version, verifying it, and installing it on the container if the user chose to install a specific version --- container creation/node_runtime_install.sh | 10 +++++ container creation/python_runtime_install.sh | 42 ++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 container creation/node_runtime_install.sh create mode 100755 container creation/python_runtime_install.sh diff --git a/container creation/node_runtime_install.sh b/container creation/node_runtime_install.sh new file mode 100644 index 00000000..21095418 --- /dev/null +++ b/container creation/node_runtime_install.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Script to install a specific Node.js runtime version +# Last Modified by Maxwell Klema on August 18th, 2025 +# ---------------------------------------------------------- + +pct enter "$CONTAINER_ID" -- < " + echo "Example: $0 debian 3.8.9" + exit 1 +fi + +OS=$1 +PYTHON_VERSION=$2 +PYTHON_URL="https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz" + +# Function to install dependencies based on OS +install_dependencies() { + case "${OS,,}" in + debian) + pct enter "$CONTAINER_ID" -- bash -c "sudo apt update -y && sudo apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \ + libnss3-dev libssl-dev libreadline-dev libffi-dev wget" + ;; + rocky) + pct enter "$CONTAINER_ID" -- bash -c "sudo dnf update -y && sudo dnf install -y gcc zlib-devel ncurses-devel gdbm-devel \ + nss-devel openssl-devel readline-devel wget libffi-devel" + ;; + esac +} + +# Function to install Python +install_python() { + echo "⏳ Installing Python $PYTHON_VERSION... This may take a while." + pct enter "$CONTAINER_ID" -- bash -c "cd /usr/src && sudo wget $PYTHON_URL && sudo tar xzf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && sudo ./configure --enable-optimizations && \ + sudo make -j$(nproc) && sudo make altinstall && \ + sudo ln -sfn /usr/local/bin/python${PYTHON_VERSION%.*} /usr/local/bin/python3 && \ + python3 -m ensurepip --upgrade && python3 -m pip install --upgrade pip" + echo "Python $PYTHON_VERSION installed successfully." +} + +install_dependencies +install_python From 0db63c4922a034222de0735d75cbf894fc7cdb0f Mon Sep 17 00:00:00 2001 From: maxklema Date: Mon, 18 Aug 2025 12:48:52 -0400 Subject: [PATCH 07/13] adding custom runtime version installation --- container creation/deployOnStart.sh | 53 +++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh index 519b0acf..2d5ee9c1 100755 --- a/container creation/deployOnStart.sh +++ b/container creation/deployOnStart.sh @@ -47,6 +47,59 @@ if [ -d "$ENV_BASE_FOLDER" ]; then fi fi +# Install Specific Runtime Versions (if Needed) + + +# Function to handle runtime installation +install_runtime() { + local runtime_language=$1 + local version=$2 + + if [ "${runtime_language,,}" == "nodejs" ]; then + local major=$(echo "$version" | cut -d. -f1) + local node_version_exists=$(curl -s https://nodejs.org/dist/index.json | grep "version\":\"v$major") + if [ ! -z "$node_version_exists" ]; then + source "/var/lib/vz/snippets/helper_scripts/node_runtime_install.sh" "$major" + else + echo "Node.js version $version ($major) is not available. Please check the version number. Using latest version instead." + fi + elif [ "${runtime_language,,}" == "python" ]; then + IFS='.' read -r -a parts <<< "$version" + + # Fill missing parts with 0 + while [ "${#parts[@]}" -lt 3 ]; do + parts+=("0") + done + + version="${parts[0]}.${parts[1]}.${parts[2]}" + local python_version_exists=$(curl -s https://www.python.org/ftp/python/ | grep "$version") + if [ ! -z "$python_version_exists" ]; then + source "/var/lib/vz/snippets/helper_scripts/python_runtime_install.sh" "${LINUX_DISTRO,,}" "$version" + else + echo "Python version $version is not available. Please check the version number. Using latest version instead." + fi + fi +} + +for key in $(echo "$VERSIONS_DICT" | jq -r 'keys[]'); do + if [ "$key" == "default" ] && [ "${MULTI_COMPONENT^^}" != "Y" ]; then + version=$(echo "$VERSIONS_DICT" | jq --arg k "$key" '.[$k]') + if [ "$version" != "null" ]; then + version=$(echo "$version" | sed 's/"//g') + install_runtime "$RUNTIME_LANGUAGE" "$version" + fi + else + value=$(echo "$RUNTIME_LANGUAGE" | jq --arg k "$key" '.[$k]') + value=$(echo "$value" | sed 's/"//g') + version=$(echo "$VERSIONS_DICT" | jq --arg k "$key" '.[$k]') + if [ "$version" != "null" ]; then + version=$(echo "$version" | sed 's/"//g') + install_runtime "$value" "$version" + fi + fi +done + + # Run Installation Commands ==== runInstallCommands() { From 10cf3f293c51299c5ea66a6a7a32d11d6a2ea619 Mon Sep 17 00:00:00 2001 From: maxklema Date: Mon, 18 Aug 2025 20:25:23 -0400 Subject: [PATCH 08/13] python version installation bug fixes --- container creation/python_runtime_install.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/container creation/python_runtime_install.sh b/container creation/python_runtime_install.sh index e7135ab1..362334aa 100755 --- a/container creation/python_runtime_install.sh +++ b/container creation/python_runtime_install.sh @@ -17,12 +17,10 @@ PYTHON_URL="https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VER install_dependencies() { case "${OS,,}" in debian) - pct enter "$CONTAINER_ID" -- bash -c "sudo apt update -y && sudo apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev \ - libnss3-dev libssl-dev libreadline-dev libffi-dev wget" + pct exec $CONTAINER_ID -- bash -c "sudo apt update -y && sudo apt install -y build-essential zlib1g-dev libncurses5-dev libgdbm-dev libnss3-dev libssl-dev libreadline-dev libffi-dev wget" > /dev/null 2>&1 ;; rocky) - pct enter "$CONTAINER_ID" -- bash -c "sudo dnf update -y && sudo dnf install -y gcc zlib-devel ncurses-devel gdbm-devel \ - nss-devel openssl-devel readline-devel wget libffi-devel" + pct exec $CONTAINER_ID -- bash -c "sudo dnf update -y && sudo dnf install -y gcc zlib-devel ncurses-devel gdbm-devel nss-devel openssl-devel readline-devel wget libffi-devel" > /dev/null 2>&1 ;; esac } @@ -30,12 +28,12 @@ install_dependencies() { # Function to install Python install_python() { echo "⏳ Installing Python $PYTHON_VERSION... This may take a while." - pct enter "$CONTAINER_ID" -- bash -c "cd /usr/src && sudo wget $PYTHON_URL && sudo tar xzf Python-$PYTHON_VERSION.tgz && \ + pct exec "$CONTAINER_ID" -- bash -c "cd /usr/src && sudo wget $PYTHON_URL && sudo tar xzf Python-$PYTHON_VERSION.tgz && \ cd Python-$PYTHON_VERSION && sudo ./configure --enable-optimizations && \ sudo make -j$(nproc) && sudo make altinstall && \ sudo ln -sfn /usr/local/bin/python${PYTHON_VERSION%.*} /usr/local/bin/python3 && \ python3 -m ensurepip --upgrade && python3 -m pip install --upgrade pip" - echo "Python $PYTHON_VERSION installed successfully." + echo "Python $PYTHON_VERSION installed successfully." > /dev/null 2>&1 } install_dependencies From f55d02919396e8a512a6dea52975c8182c4746fd Mon Sep 17 00:00:00 2001 From: maxklema Date: Wed, 20 Aug 2025 12:12:43 -0400 Subject: [PATCH 09/13] invalid path fix --- container creation/deployOnStart.sh | 29 +++-- container creation/start_services.sh | 151 +++++++++++++++------------ 2 files changed, 103 insertions(+), 77 deletions(-) diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh index 2d5ee9c1..aea15dc2 100755 --- a/container creation/deployOnStart.sh +++ b/container creation/deployOnStart.sh @@ -6,9 +6,12 @@ echo "🚀 Attempting Automatic Deployment" REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") -# Clone github repository from correct branch ==== +# Helper function to normalize paths by removing duplicate slashes +normalize_path() { + echo "$1" | sed 's#/\+#/#g' +} -echo "Repo base name: $REPO_BASE_NAME" +# Clone github repository from correct branch ==== pct enter $CONTAINER_ID < \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 done else ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt || true) - COMPONENT_PATH="/root/$REPO_BASE_NAME/$PROJECT_ROOT" + COMPONENT_PATH=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") pct exec $CONTAINER_ID -- bash -c "if [ -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" > \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 fi fi # Install Specific Runtime Versions (if Needed) +echo "VERSIONS_DICT: $VERSIONS_DICT" +echo "RUNTIME_LANGUAGE: $RUNTIME_LANGUAGE" + +pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 8 > /dev/null 2>&1 # Function to handle runtime installation install_runtime() { @@ -59,7 +68,7 @@ install_runtime() { local major=$(echo "$version" | cut -d. -f1) local node_version_exists=$(curl -s https://nodejs.org/dist/index.json | grep "version\":\"v$major") if [ ! -z "$node_version_exists" ]; then - source "/var/lib/vz/snippets/helper_scripts/node_runtime_install.sh" "$major" + source /var/lib/vz/snippets/helper-scripts/node_runtime_install.sh "$major" else echo "Node.js version $version ($major) is not available. Please check the version number. Using latest version instead." fi @@ -74,7 +83,7 @@ install_runtime() { version="${parts[0]}.${parts[1]}.${parts[2]}" local python_version_exists=$(curl -s https://www.python.org/ftp/python/ | grep "$version") if [ ! -z "$python_version_exists" ]; then - source "/var/lib/vz/snippets/helper_scripts/python_runtime_install.sh" "${LINUX_DISTRO,,}" "$version" + source /var/lib/vz/snippets/helper-scripts/python_runtime_install.sh "${LINUX_DISTRO,,}" "$version" else echo "Python version $version is not available. Please check the version number. Using latest version instead." fi @@ -99,19 +108,21 @@ for key in $(echo "$VERSIONS_DICT" | jq -r 'keys[]'); do fi done - # Run Installation Commands ==== runInstallCommands() { RUNTIME="$1" COMP_DIR="$2" + + # Create normalized path + WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR") if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && sudo $INSTALL_CMD" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "cd $WORK_DIR && sudo $INSTALL_CMD" > /dev/null 2>&1 elif [ "${RUNTIME^^}" == "PYTHON" ]; then pct enter $CONTAINER_ID < /dev/null -cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && \ +cd $WORK_DIR && \ python3 -m venv venv && source venv/bin/activate && \ pip install --upgrade pip && \ $INSTALL_CMD diff --git a/container creation/start_services.sh b/container creation/start_services.sh index 46ccfc50..10f9d651 100644 --- a/container creation/start_services.sh +++ b/container creation/start_services.sh @@ -22,6 +22,11 @@ PROJECT_BRANCH="${15}" UPDATE_CONTAINER="${16}" CONTAINER_NAME="${CONTAINER_NAME,,}" +# Helper function to normalize paths by removing duplicate slashes +normalize_path() { + echo "$1" | sed 's#/\+#/#g' +} + if [ "${GH_ACTION^^}" == "Y" ]; then sleep 8 # Wait for Job to Complete fi @@ -52,44 +57,49 @@ if (( $CONTAINER_ID % 2 == 0 )); then startProject() { - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + # Create normalized path + WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR") - if [ -z "$BUILD_CMD" ]; then - BUILD_CMD="true" - fi + if [ -z "$BUILD_CMD" ]; then + BUILD_CMD="true" + fi - if [ "${RUNTIME^^}" == "NODEJS" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 - fi + if [ "${RUNTIME^^}" == "NODEJS" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd $WORK_DIR && $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c \"mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp tmux new-session -d 'export HOME=/root export PATH=\\\$PATH:/usr/local/bin && cd $WORK_DIR && source venv/bin/activate $BUILD_CMD && $START_CMD'\"" > /dev/null 2>&1 + fi - } + } if [ "${DEPLOY_ON_START^^}" == "Y" ]; then - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 - fi - else - startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - if [ ! -z "$ROOT_START_COMMAND" ]; then - ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND'" > /dev/null 2>&1 - fi - fi - fi + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + startProject "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd $ROOT_WORK_DIR && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + else + startProject "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + ssh root@10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'cd $ROOT_WORK_DIR && $ROOT_START_COMMAND'" > /dev/null 2>&1 + fi + fi + fi # PVE 1 else @@ -101,41 +111,46 @@ else startComponent() { - RUNTIME="$1" - BUILD_CMD="$2" - START_CMD="$3" - COMP_DIR="$4" - - if [ -z "$BUILD_CMD" ]; then - BUILD_CMD="true" - fi - - if [ "${RUNTIME^^}" == "NODEJS" ]; then - pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root && export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && $BUILD_CMD && $START_CMD\"" - elif [ "${RUNTIME^^}" == "PYTHON" ]; then - pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root &&export PATH=\$PATH:/usr/local/bin && cd /root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR && source venv/bin/activate && $BUILD_CMD && $START_CMD\"" - fi - } + RUNTIME="$1" + BUILD_CMD="$2" + START_CMD="$3" + COMP_DIR="$4" + + # Create normalized path + WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$COMP_DIR") + + if [ -z "$BUILD_CMD" ]; then + BUILD_CMD="true" + fi + + if [ "${RUNTIME^^}" == "NODEJS" ]; then + pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root && export PATH=\$PATH:/usr/local/bin && cd $WORK_DIR && $BUILD_CMD && $START_CMD\"" + elif [ "${RUNTIME^^}" == "PYTHON" ]; then + pct exec "$CONTAINER_ID" -- bash -c "mkdir -p /tmp && chmod 1777 /tmp && mkdir -p /tmp/tmux-0 && chmod 700 /tmp/tmux-0 && TMUX_TMPDIR=/tmp/tmux-0 tmux new-session -d \"export HOME=/root &&export PATH=\$PATH:/usr/local/bin && cd $WORK_DIR && source venv/bin/activate && $BUILD_CMD && $START_CMD\"" + fi + } pct set $CONTAINER_ID --memory 4096 --swap 0 --cores 4 > /dev/null #temporarily bump up container resources for computation hungry processes (e.g. meteor) - if [ "${MULTI_COMPONENT^^}" == "Y" ]; then - for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do - START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') - BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') - if [ "$BUILD" == "null" ]; then - BUILD="" - fi - - startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" - done - if [ ! -z "$ROOT_START_COMMAND" ]; then - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 - fi - else - startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." - if [ ! -z "$ROOT_START_COMMAND" ]; then - pct exec $CONTAINER_ID -- bash -c "cd /root/$REPO_BASE_NAME/$PROJECT_ROOT && $ROOT_START_COMMAND" > /dev/null 2>&1 - fi - fi + if [ "${MULTI_COMPONENT^^}" == "Y" ]; then + for COMPONENT in $(echo "$START_COMMAND" | jq -r 'keys[]'); do + START=$(echo "$START_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + RUNTIME=$(echo "$RUNTIME_LANGUAGE" | jq -r --arg k "$COMPONENT" '.[$k]') + BUILD=$(echo "$BUILD_COMMAND" | jq -r --arg k "$COMPONENT" '.[$k]') + if [ "$BUILD" == "null" ]; then + BUILD="" + fi + + startComponent "$RUNTIME" "$BUILD" "$START" "$COMPONENT" + done + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + pct exec $CONTAINER_ID -- bash -c "cd $ROOT_WORK_DIR && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi + else + startComponent "$RUNTIME_LANGUAGE" "$BUILD_COMMAND" "$START_COMMAND" "." + if [ ! -z "$ROOT_START_COMMAND" ]; then + ROOT_WORK_DIR=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") + pct exec $CONTAINER_ID -- bash -c "cd $ROOT_WORK_DIR && $ROOT_START_COMMAND" > /dev/null 2>&1 + fi + fi fi From ee30c3e008351f2ca094b7c6d43d5588ea52bd2c Mon Sep 17 00:00:00 2001 From: Maxwell Klema <80615123+maxklema@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:28:56 -0400 Subject: [PATCH 10/13] Add port-map-server.js for host key filtering Implement a JSON server that filters and returns specific host keys from a JSON file. --- nginx reverse proxy/port-map-server.js | 49 ++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 nginx reverse proxy/port-map-server.js diff --git a/nginx reverse proxy/port-map-server.js b/nginx reverse proxy/port-map-server.js new file mode 100644 index 00000000..aeee401a --- /dev/null +++ b/nginx reverse proxy/port-map-server.js @@ -0,0 +1,49 @@ +// etc/nginx/port-map-server.js +// JSON-server that returns all (filtered) hosts to the https://opensource.mieweb.org +// Last modified on Aug 22, 2025 by Maxwell Klema + + +const jsonServer = require('json-server'); +const path = require('path'); +const fs = require('fs'); +const filePath = "/etc/nginx/port_map.json"; + +const server = jsonServer.create(); +const router = jsonServer.router(path.join(__dirname, 'port_map.json')); +const middlewares = jsonServer.defaults(); +server.use(middlewares); + +server.get('/keys', (req, res) => { + const db = router.db; + const keys = Object.keys(db.getState()).filter(key => !['container-creation', 'intern-dnsmasq', 'wazuh-server', 'wazuh-indexer', 'wazuh-dashboard', 'intern-nginx', 'mie-ldap-server', 'create-a-container', 'landing-page'].includes(key)); + + // Filter out keys that are branches of a main/master for a repository + + let content = fs.readFileSync(filePath); + let cachedMapping = JSON.parse(content); + + all_hosts = []; + proxmox_launchpad_lxcs = []; + exclude = []; + for (const key of Object.keys(cachedMapping)) { + all_hosts.push(key); + if (key.endsWith("-main") || key.endsWith("master")) { + proxmox_launchpad_lxcs.push(key.substring(0, key.lastIndexOf("-"))); + } + } + + for (const entry of proxmox_launchpad_lxcs) { + hosts_to_filter = all_hosts.filter(key => { + return key.startsWith(entry) && (!key.endsWith("-main") || !key.startsWith(entry)); + }); + hosts_to_filter.forEach(host => exclude.push(host)); + } + + const filteredKeys = keys.filter(key => !exclude.includes(key)); + res.json(filteredKeys); +}) + +server.use(router); +server.listen(3001, () => { + console.log("JSON Server Running on http://localhost:3001"); +}) From 8fcf3cd3334917cfc1a8decde4c0790418668422 Mon Sep 17 00:00:00 2001 From: Maxwell Klema <80615123+maxklema@users.noreply.github.com> Date: Fri, 22 Aug 2025 18:48:04 -0400 Subject: [PATCH 11/13] Add endpoint to retrieve specific key details --- nginx reverse proxy/port-map-server.js | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/nginx reverse proxy/port-map-server.js b/nginx reverse proxy/port-map-server.js index aeee401a..a935eb27 100644 --- a/nginx reverse proxy/port-map-server.js +++ b/nginx reverse proxy/port-map-server.js @@ -2,7 +2,6 @@ // JSON-server that returns all (filtered) hosts to the https://opensource.mieweb.org // Last modified on Aug 22, 2025 by Maxwell Klema - const jsonServer = require('json-server'); const path = require('path'); const fs = require('fs'); @@ -43,6 +42,25 @@ server.get('/keys', (req, res) => { res.json(filteredKeys); }) +server.get('/:key', (req, res) => { + const key = req.params.key; + const db = router.db; + const value = db.getState()[key]; + + const response = { + name: key, + owner: value.user, + description: value.description || "", + github_url: value.github_url || "", + }; + + if (value) { + res.json(response); + } else { + res.status(404).json({ error: 'Not found' }); + } +}); + server.use(router); server.listen(3001, () => { console.log("JSON Server Running on http://localhost:3001"); From 79ff66ba76f7ac36a6b5cc92e485d38df4476104 Mon Sep 17 00:00:00 2001 From: maxklema Date: Sat, 23 Aug 2025 18:27:24 -0400 Subject: [PATCH 12/13] opensource documentation --- .gitmodules | 3 +++ README.md | 7 +++++++ ci-cd automation/setup-runner.sh | 2 -- mie-opensource-landing | 1 + 4 files changed, 11 insertions(+), 2 deletions(-) create mode 160000 mie-opensource-landing diff --git a/.gitmodules b/.gitmodules index d7c6479b..7d6d4b66 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "ci-cd automation/Proxmox-Launchpad"] path = ci-cd automation/Proxmox-Launchpad url = https://github.com/maxklema/proxmox-launchpad.git +[submodule "mie-opensource-landing"] + path = mie-opensource-landing + url = https://github.com/maxklema/mie-opensource-landing diff --git a/README.md b/README.md index 7fec6147..1c179d68 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Configuration storage for the [opensource.mieweb.org](https://opensource.mieweb.org:8006) Proxmox project. +To learn everything there is about our cluster, see our documentation at [https://opensource.mieweb.org/docs/intro](https://opensource.mieweb.org/docs/intro). + This repository contains configuration files and scripts for managing a Proxmox-based container hosting environment, including automated DNS, NGINX reverse proxy, dynamic port mapping, and the Proxmox LaunchPad GitHub Action for automated container deployment. ## Cluster Graph @@ -99,6 +101,11 @@ If you have an account in the [opensource-mieweb](https://opensource.mieweb.org: - Use the Command Line: ssh create-container@opensource.mieweb.org (mie123!) - Use the Proxmox LaunchPad Github Action to automatically provision, update, and delete containers for you: [Proxmox LaunchPad](#proxmox-launchpad) +## MIE Opensource Landing + +Contains all the source code for [https://opensource.mieweb.org's](https://opensource.mieweb.org) landing page, built with React + Docusaurus. +- Documentation is located at [https://opensource.mieweb.org/docs/intro](https://opensource.mieweb.org/docs/intro). + ## How It Works - **DNS**: All `*.opensource.mieweb.com` requests are routed to the NGINX proxy via Dnsmasq, providing automatic subdomain resolution for containers. diff --git a/ci-cd automation/setup-runner.sh b/ci-cd automation/setup-runner.sh index ef2cda04..aaf9fafe 100644 --- a/ci-cd automation/setup-runner.sh +++ b/ci-cd automation/setup-runner.sh @@ -117,8 +117,6 @@ echo "$PUB_KEY" >> /home/update-container/.ssh/authorized_keys echo "$PUB_KEY" >> /home/delete-container/.ssh/authorized_keys echo "$PUB_KEY" >> /home/container-exists/.ssh/authorized_keys -ssh root@10.15.234.122 "echo \"$PUB_KEY\" >> /root/.ssh/authorized_keys" - echo "🔑 Creating Service File..." pct exec $NEXT_ID -- bash -c "cat < /etc/systemd/system/github-runner.service [Unit] diff --git a/mie-opensource-landing b/mie-opensource-landing new file mode 160000 index 00000000..31c40c17 --- /dev/null +++ b/mie-opensource-landing @@ -0,0 +1 @@ +Subproject commit 31c40c1717789132308c175a6961688edef221dd From 12573cd347ec51ca09cddc9f49a65ba5bd5e083d Mon Sep 17 00:00:00 2001 From: maxklema Date: Sat, 23 Aug 2025 18:40:53 -0400 Subject: [PATCH 13/13] addressing copilot fixes --- ci-cd automation/check-container-exists.sh | 8 ++++---- ci-cd automation/update-container.sh | 4 ++-- container creation/deployOnStart.sh | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci-cd automation/check-container-exists.sh b/ci-cd automation/check-container-exists.sh index 791b24ee..ba0b2c9a 100644 --- a/ci-cd automation/check-container-exists.sh +++ b/ci-cd automation/check-container-exists.sh @@ -5,7 +5,7 @@ outputError() { echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" - echo -e "${BOLD}${MAGENTA} Script Failed. Exiting... ${RESET}" + echo -e "${BOLD}${MAGENTA}❌ Script Failed. Exiting... ${RESET}" echo -e "$2" echo -e "${BOLD}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" exit $1 @@ -32,15 +32,15 @@ REPO_BASE_NAME=$(basename -s .git "$PROJECT_REPOSITORY") # Check if repository folder is present. if [ "$PVE1" == "true" ]; then - if [ ! -z $CONTAINER_ID ] && pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then + if [ ! -z "$CONTAINER_ID" ] && pct exec $CONTAINER_ID -- test -f /root/container-updates.log; then exit 2; # Update Repository else exit 0; # Clone Repository fi else - if [ ! -z $CONTAINER_ID ] && ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log" ; then + if [ ! -z "$CONTAINER_ID" ] && ssh 10.15.0.5 "pct exec $CONTAINER_ID -- test -f /root/container-updates.log" ; then exit 2; # Update Repository else exit 0; # Clone Repository fi -fi +fi \ No newline at end of file diff --git a/ci-cd automation/update-container.sh b/ci-cd automation/update-container.sh index 562f72d1..0be5e739 100644 --- a/ci-cd automation/update-container.sh +++ b/ci-cd automation/update-container.sh @@ -148,9 +148,9 @@ writeEnvToFile() { component_path="$2" env_vars=$(cat "$env_file_path") if (( $CONTAINER_ID % 2 == 0 )); then - ssh 10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'if [ -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"'" + ssh 10.15.0.5 "pct exec $CONTAINER_ID -- bash -c 'if [ ! -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"'" else - pct exec $CONTAINER_ID -- bash -c "if [ -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"" + pct exec $CONTAINER_ID -- bash -c "if [ ! -f \"$component_path/.env\" ]; then touch \"$component_path/.env\"; fi; echo \"$env_vars\" >> \"$component_path/.env\"" fi } diff --git a/container creation/deployOnStart.sh b/container creation/deployOnStart.sh index aea15dc2..294dbd2c 100755 --- a/container creation/deployOnStart.sh +++ b/container creation/deployOnStart.sh @@ -42,13 +42,13 @@ if [ -d "$ENV_BASE_FOLDER" ]; then ENV_VARS=$(cat $ENV_BASE_FOLDER/$FILE_BASENAME) COMPONENT_PATH=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT/$ENV_ROUTE") - pct exec $CONTAINER_ID -- bash -c "if [ -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" > \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "if [ ! -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" >> \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 done else ENV_FOLDER_BASE_NAME=$(basename "$ENV_BASE_FOLDER") ENV_VARS=$(cat $ENV_BASE_FOLDER/$ENV_FOLDER_BASE_NAME.txt || true) COMPONENT_PATH=$(normalize_path "/root/$REPO_BASE_NAME/$PROJECT_ROOT") - pct exec $CONTAINER_ID -- bash -c "if [ -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" > \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 + pct exec $CONTAINER_ID -- bash -c "if [ ! -f \"$COMPONENT_PATH/.env\" ]; then touch \"$COMPONENT_PATH/.env\"; fi; echo \"$ENV_VARS\" >> \"$COMPONENT_PATH/.env\"" > /dev/null 2>&1 fi fi