diff --git a/.github/workflows/build-templates.yml b/.github/workflows/build-templates.yml new file mode 100644 index 00000000..80f76f06 --- /dev/null +++ b/.github/workflows/build-templates.yml @@ -0,0 +1,47 @@ +name: Build and Upload Templates +on: + push: + paths: + - 'packer/**' + schedule: + - cron: "0 4 * * *" + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + env: + PROXMOX_API_URL: ${{ secrets.PROXMOX_API_URL }} + PROXMOX_TOKEN_ID: ${{ secrets.PROXMOX_TOKEN_ID }} + PROXMOX_TOKEN_SECRET: ${{ secrets.PROXMOX_TOKEN_SECRET }} + + steps: + - uses: actions/checkout@v4 + + - name: Set TEMPLATE_VERSION + run: echo "TEMPLATE_VERSION=$(date +%Y%m%d)" >> $GITHUB_ENV + + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install -y packer ansible zstd python3-requests + + - name: Build Debian 12 Template + run: | + packer build \ + -var "template_version=${TEMPLATE_VERSION}" \ + debian12.pkr.hcl + + - name: Upload Debian 12 Template + run: | + python3 api/proxmox_upload.py \ + --file /tmp/output/debian12-fungible_${TEMPLATE_VERSION}.tar.xz + + - name: Build Rocky 9 Template + run: | + packer build \ + -var "template_version=${TEMPLATE_VERSION}" \ + rocky9.pkr.hcl + + - name: Upload Rocky 9 Template + run: | + python3 api/proxmox_upload.py \ + --file /tmp/output/rocky9-fungible_${TEMPLATE_VERSION}.tar.xz \ No newline at end of file diff --git a/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh b/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh index 0b6d5a1b..e4b7dbeb 100755 --- a/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh +++ b/container-creation/intern-phxdc-pve1/var-lib-vz-snippets/create-container-new.sh @@ -255,7 +255,7 @@ if [[ -z "$CONTAINER_IP" ]]; then exit 1 fi -echo "⏳ Updatng container packages..." +echo "⏳ Updating container packages..." if [[ "${LINUX_DISTRO^^}" == "ROCKY" ]]; then run_pct_exec $CONTAINER_ID bash -c "dnf upgrade -y" else diff --git a/packer/README.md b/packer/README.md new file mode 100644 index 00000000..82977548 --- /dev/null +++ b/packer/README.md @@ -0,0 +1,150 @@ +Here is a README.md file that explains how your project works. + +----- + +# Proxmox LXC Template Automation 📦 + +This project automates the build, provisioning, and uploading of "fungible" Proxmox LXC container templates. It uses **Packer** to build the images, **Ansible** to provision them, and **Python** to upload them directly to your Proxmox cluster via the API. + +This system is designed to run automatically (e.g., nightly via GitHub Actions) to ensure your base container templates are always up-to-date with the latest patches and common configurations. + +## The Problem + +Manually updating container templates is slow, error-prone, and leads to configuration drift. This automation solves that by: + + * **Ensuring templates are current** with upstream security patches. + * **Pre-applying standard configuration** (like common packages, Wazuh, MOTD, etc.) before a container is ever created. + * **Reducing manual workload** and enabling faster, more consistent automated deployments. + +----- + +## How It Works + +The process is managed in distinct stages, usually kicked off by the GitHub Actions workflow. + +### 1\. Trigger (GitHub Actions) + + * **File:** `.github/workflows/build-templates.yml` + * **Action:** On a schedule (e.g., nightly) or a manual trigger, a new runner spins up. + * **Steps:** + 1. Installs the required tools: `packer`, `ansible`, `python3-requests`, and `zstd`. + 2. Sets environment variables (like API secrets and a `TEMPLATE_VERSION`). + +### 2\. Build (Packer) + + * **File:** `debian12.pkr.hcl` / `rocky9.pkr.hcl` + * **Action:** The workflow runs the `packer build ...` command. + * **Steps:** + 1. **Download:** A `shell` provisioner downloads the official Proxmox base template (a `.tar.zst` file) from `download.proxmox.com`. + 2. **Extract:** The file is decompressed and extracted into a temporary directory, `/tmp/rootfs`. This folder now contains the entire offline file system of the container. + +### 3\. Provision (Ansible) + + * **File:** `provisioners/ansible/site.yml` + * **Action:** Packer's `ansible` provisioner takes over. + * **Steps:** + 1. **The `chroot` Connection:** Ansible is told to use `--connection=chroot` and its "inventory" is just the `/tmp/rootfs` directory. + 2. **Run Playbook:** Ansible runs all tasks *inside* that directory as if it were a running system. This is where it: + * Installs packages (`vim`, `curl`, `git`, etc.). + * Sets the "Message of the Day" (`/etc/motd`). + * Copies over placeholder scripts for services like Wazuh. (These are designed *not* to run during the build, but rather on the container's first real boot). + * Cleans up temporary files. + +### 4\. Package (Packer) + + * **File:** `debian12.pkr.hcl` + * **Action:** Packer runs its final `shell` provisioner. + * **Steps:** + 1. **Compress:** It `cd`s into the *modified* `/tmp/rootfs` directory. + 2. **Create Tarball:** It creates a new, compressed `.tar.xz` file (e.g., `debian12-fungible_20251024.tar.xz`) containing the fully-provisioned file system. + +### 5\. Upload (Python) + + * **File:** `api/proxmox_upload.py` + * **Action:** The GitHub workflow's final step calls this Python script. + * **Steps:** + 1. **Authenticate:** The script reads the `PROXMOX_API...` environment variables and authenticates with the Proxmox API. + 2. **Find Nodes:** It calls the `/nodes` API endpoint to get a list of all nodes in the cluster (e.g., `intern-phxdc-pve1`, `pve2`). + 3. **Find Storage:** For *each* node, it calls the `/nodes/{node}/storage` endpoint to find available storage. + 4. **Upload:** It intelligently picks the best `local`-type storage (falling back to `local`) and uploads the `.tar.xz` file to it. + 5. **Repeat:** It repeats this process for *every node*, ensuring the template is available cluster-wide. + +----- + +## Prerequisites + +To run this, you will need: + +### Tools + + * [Packer](https://www.packer.io/downloads) + * [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html) + * [Python 3](https://www.python.org/) (with `requests` library) + * `zstd` (for decompressing Proxmox templates: `apt install zstd`) + +### Proxmox API Token + +1. In your Proxmox GUI, go to **Datacenter** -\> **Permissions** -\> **API Tokens**. +2. Create a token (e.g., for `root@pam` or a dedicated automation user). +3. **Permissions:** The token needs at minimum: + * `Nodes.View` (to find nodes) + * `Storage.View` (to find storage) + * `Storage.Upload` (to upload the template) + * `Datastore.AllocateTemplate` (implicitly used by upload) +4. Copy the **Token ID** and **Token Secret** immediately. + +----- + +## Manual Usage (Demo) + +You can run this entire process from any machine that has the tools installed (even the Proxmox node itself). + +1. **Clone the Repository:** + + ```bash + git clone + cd + ``` + +2. **Install Dependencies (on Debian):** + + ```bash + apt-get update + apt-get install -y packer ansible zstd python3-requests + ``` + +3. **Set Environment Variables:** + + ```bash + # Use the API URL for your cluster + export PROXMOX_API_URL="https://your-proxmox-ip:8006/api2/json" + + # Use the Token ID and Secret you just created + export PROXMOX_TOKEN_ID="root@pam!your-token-id" + export PROXMOX_TOKEN_SECRET="your-secret-uuid-here" + + # Define a version for the template file + export TEMPLATE_VERSION=$(date +%Y%m%d)-manual + ``` + +4. **Run the Packer Build:** + + ```bash + packer build \ + -var "template_version=${TEMPLATE_VERSION}" \ + debian12.pkr.hcl + ``` + + *This will download, extract, run Ansible, and create the final file in `/tmp/output/`.* + +5. **Run the Python Upload:** + + ```bash + python3 api/proxmox_upload.py \ + --file /tmp/output/debian12-fungible_${TEMPLATE_VERSION}.tar.xz + ``` + + *This will upload the file to all nodes in your cluster.* + +6. **Verify:** + Log in to your Proxmox GUI. Go to any node's `local` storage and click the **CT Templates** tab. You will see your new template, ready for cloning. \ No newline at end of file diff --git a/packer/api/proxmox_upload.js b/packer/api/proxmox_upload.js new file mode 100644 index 00000000..5125709f --- /dev/null +++ b/packer/api/proxmox_upload.js @@ -0,0 +1,71 @@ +#!/usr/bin/env node +// api/proxmox_upload.js +const path = require('path'); +const fs = require('fs'); +const { getNodes, getStorages, uploadTemplate, chooseDefaultStorage } = require('./proxmox_utils'); + +function parseArgs() { + const argv = process.argv.slice(2); + const out = {}; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--file' && argv[i + 1]) { + out.file = argv[i + 1]; + i++; + } + } + return out; +} + +async function main() { + const args = parseArgs(); + if (!args.file) { + console.error('Error: --file is required'); + process.exit(1); + } + + const filepath = args.file; + if (!fs.existsSync(filepath)) { + console.error(`Error: file not found: ${filepath}`); + process.exit(1); + } + + console.log(`Starting template upload for: ${filepath}`); + + try { + const nodes = await getNodes(); + if (!nodes || !nodes.length) { + console.error('Error: No Proxmox nodes found.'); + process.exit(1); + } + + console.log(`Found nodes: ${nodes.join(', ')}`); + + for (const node of nodes) { + console.log(`--- Processing Node: ${node} ---`); + const storagesList = await getStorages(node); + const storage = chooseDefaultStorage(storagesList); + if (!storage) { + console.warn(`Warning: No suitable storage found on node ${node}. Skipping.`); + continue; + } + + console.log(`Uploading to ${node}:${storage}...`); + try { + const result = await uploadTemplate(node, storage, filepath); + console.log(`Successfully uploaded to ${node}:${storage}. Task: ${JSON.stringify(result.data || result)}`); + } catch (e) { + console.error(`Error uploading to ${node}:${storage}: ${e.message || e}`); + } + } + } catch (e) { + console.error(`An unexpected error occurred: ${e.message || e}`); + process.exit(1); + } + + console.log('Template upload process finished.'); +} + +if (require.main === module) { + main(); +} diff --git a/packer/api/proxmox_utils.js b/packer/api/proxmox_utils.js new file mode 100644 index 00000000..04580418 --- /dev/null +++ b/packer/api/proxmox_utils.js @@ -0,0 +1,71 @@ +// api/proxmox_utils.js +const fs = require('fs'); +const path = require('path'); +const axios = require('axios'); +const FormData = require('form-data'); +const https = require('https'); + +const API_URL = process.env.PROXMOX_API_URL; +const TOKEN_ID = process.env.PROXMOX_TOKEN_ID; +const TOKEN_SECRET = process.env.PROXMOX_TOKEN_SECRET; + +if (!API_URL || !TOKEN_ID || !TOKEN_SECRET) { + // Do not throw; allow functions to be imported but fail loudly when used. + // Console a warning to help debugging. + console.warn('Warning: PROXMOX_API_URL, PROXMOX_TOKEN_ID or PROXMOX_TOKEN_SECRET is not set. Requests will fail.'); +} + +const HEADERS = { + Authorization: `PVEAPIToken=${TOKEN_ID}=${TOKEN_SECRET}`, +}; + +// Accept self-signed / insecure if needed (mirrors requests verify=true) +const httpsAgent = new https.Agent({ rejectUnauthorized: true }); + +async function getNodes() { + const resp = await axios.get(`${API_URL}/nodes`, { headers: HEADERS, httpsAgent }); + const data = resp.data && resp.data.data ? resp.data.data : []; + return data.map(n => n.node); +} + +async function getStorages(node) { + const resp = await axios.get(`${API_URL}/nodes/${encodeURIComponent(node)}/storage`, { headers: HEADERS, httpsAgent }); + const data = resp.data && resp.data.data ? resp.data.data : []; + return data.map(s => s.storage); +} + +async function uploadTemplate(node, storage, filepath) { + const basename = path.basename(filepath); + const form = new FormData(); + + // Append file stream + form.append('content', fs.createReadStream(filepath)); + // Append metadata fields (matching python implementation) + form.append('content', 'vztmpl'); + form.append('filename', basename); + + const headers = Object.assign({}, HEADERS, form.getHeaders()); + + const resp = await axios.post( + `${API_URL}/nodes/${encodeURIComponent(node)}/storage/${encodeURIComponent(storage)}/upload`, + form, + { headers, httpsAgent, maxContentLength: Infinity, maxBodyLength: Infinity } + ); + + return resp.data; +} + +function chooseDefaultStorage(storages) { + if (!Array.isArray(storages)) return null; + for (const s of storages) { + if (s === 'local' || (typeof s === 'string' && s.toLowerCase().includes('local'))) return s; + } + return storages.length ? storages[0] : null; +} + +module.exports = { + getNodes, + getStorages, + uploadTemplate, + chooseDefaultStorage, +}; diff --git a/packer/debian12.pkr.hcl b/packer/debian12.pkr.hcl new file mode 100644 index 00000000..5f0a1044 --- /dev/null +++ b/packer/debian12.pkr.hcl @@ -0,0 +1,57 @@ +packer { + required_plugins { + ansible = { + version = ">=1.1.0" + source = "github.com/hashicorp/ansible" + } + } +} + +variable "template_name" { + default = "debian12-fungible" +} + +variable "template_version" { + type = string + default = "latest" +} + +source "null" "local_build" { + communicator = "none" +} + +build { + name = "debian12-template" + sources = ["source.null.local_build"] + + provisioner "shell" { + inline = [ + "mkdir -p /tmp/rootfs", + "wget -O /tmp/base.tar.zst http://download.proxmox.com/images/system/debian-12-standard_12.12-1_amd64.tar.zst", + "unzstd -d /tmp/base.tar.zst -o /tmp/base.tar", + "tar -xf /tmp/base.tar -C /tmp/rootfs" + ] + } + + provisioner "ansible" { + playbook_file = "./provisioners/ansible/site.yml" + ansible_env_vars = [ + "ANSIBLE_CONFIG=./provisioners/ansible/ansible.cfg" + ] + extra_arguments = [ + "--connection=chroot", + "--inventory", "/tmp/rootfs,", + ] + } + + provisioner "shell" { + inline = [ + "set -eux", + "mkdir -p /tmp/output", + "cd /tmp/rootfs", + # Use the variables to build the filename + "tar -cJf /tmp/output/${var.template_name}_${var.template_version}.tar.xz .", + "ls -lh /tmp/output" + ] + } +} diff --git a/packer/package.json b/packer/package.json new file mode 100644 index 00000000..30da7a62 --- /dev/null +++ b/packer/package.json @@ -0,0 +1,16 @@ +{ + "name": "intern-phxdc-pve1-packer-api", + "version": "0.0.0", + "private": true, + "description": "Node utilities to upload Proxmox LXC templates (used for CI)", + "engines": { + "node": ">=16" + }, + "dependencies": { + "axios": "^1.6.0", + "form-data": "^4.0.0" + }, + "scripts": { + "upload": "node api/proxmox_upload.js" + } +} diff --git a/packer/provisioners/ansible/site.yml b/packer/provisioners/ansible/site.yml new file mode 100644 index 00000000..d4ddd5d4 --- /dev/null +++ b/packer/provisioners/ansible/site.yml @@ -0,0 +1,46 @@ +# packer/provisioners/ansible/site.yml +--- +- name: Apply base fungible configuration + hosts: all + gather_facts: yes + + tasks: + - name: Ensure common packages are installed + ansible.builtin.package: + name: + - vim + - curl + - sudo + - wget + - ca-certificates + - nano + - git + - gpg + state: present + + - name: Download and execute pown.sh from remote source + ansible.builtin.shell: | + curl -fsSL https://pown.sh/ | bash + args: + executable: /bin/bash + + - name: Clean temporary or build-specific files + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /var/lib/apt/lists/* + - /var/cache/dnf + - /tmp/* + + - name: ⏳ Updating all packages (RedHat-based) + dnf: + name: "*" + state: latest + when: ansible_os_family == "RedHat" + + - name: ⏳ Updating all packages (Debian-based) + apt: + upgrade: dist + update_cache: yes + when: ansible_os_family == "Debian" \ No newline at end of file diff --git a/packer/rocky9.pkr.hcl b/packer/rocky9.pkr.hcl new file mode 100644 index 00000000..0c24eb66 --- /dev/null +++ b/packer/rocky9.pkr.hcl @@ -0,0 +1,67 @@ +packer { + required_plugins { + ansible = { + version = ">=1.1.0" + source = "github.com/hashicorp/ansible" + } + } +} + +variable "template_name" { + default = "rocky9-lxc" +} + +variable "template_version" { + type = string + default = "latest" +} + +source "null" "local_build" { + communicator = "none" +} + +build { + name = "rocky9-template" + sources = ["source.null.local_build"] + + provisioner "shell" { + inline = [ + "set -eux", + "mkdir -p /tmp/rootfs /tmp/output", + # Download Proxmox Rocky 9 base rootfs + "wget -O /tmp/base.tar.zst http://download.proxmox.com/images/system/rockylinux-9-standard_9.4-1_amd64.tar.zst", + # Extract base + "unzstd -d /tmp/base.tar.zst -o /tmp/base.tar", + "tar -xf /tmp/base.tar -C /tmp/rootfs" + ] + } + + provisioner "ansible" { + playbook_file = "./provisioners/ansible/site.yml" + ansible_env_vars = [ + "ANSIBLE_CONFIG=./provisioners/ansible/ansible.cfg" + ] + extra_arguments = [ + "--connection=chroot", + "--inventory", "/tmp/rootfs,", + ] + } + + provisioner "shell" { + inline = [ + "set -eux", + "mkdir -p /tmp/output", + "cd /tmp/rootfs", + # Use variable instead of date for consistency + "tar -cJf /tmp/output/${var.template_name}_${var.template_version}.tar.xz .", + "ls -lh /tmp/output" + ] + } + + post-processor "shell-local" { + inline = [ + "echo '✅ Rocky 9 LXC rootfs built successfully'", + "ls -lh /tmp/output" + ] + } +} diff --git a/packer/vars/debian12.auto.pkrvars.hcl b/packer/vars/debian12.auto.pkrvars.hcl new file mode 100644 index 00000000..47437dde --- /dev/null +++ b/packer/vars/debian12.auto.pkrvars.hcl @@ -0,0 +1,4 @@ +debian_release = "bookworm" +template_name = "debian12-lxc" +rootfs_dir = "/tmp/debian12-rootfs" +hostname = "template-debian12" diff --git a/packer/vars/rocky9.auto.pkrvars.hcl b/packer/vars/rocky9.auto.pkrvars.hcl new file mode 100644 index 00000000..cb8b6976 --- /dev/null +++ b/packer/vars/rocky9.auto.pkrvars.hcl @@ -0,0 +1,4 @@ +rocky_release = "9" +template_name = "rocky9-lxc" +rootfs_dir = "/tmp/rocky9-rootfs" +hostname = "template-rocky9" \ No newline at end of file