From 809b6870d952dd6e8778e126434adb1c0cda8c77 Mon Sep 17 00:00:00 2001 From: Jordan Frazier <122494242+jordanrfrazier@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:58:02 -0400 Subject: [PATCH 1/4] ci: allow releasing from branch (#9988) * Try skipping gha cache * simplify docker builds * remove workflow dispatch options from workflows that shouldn't be run manually * Update nightly docker build and add option to skip testing for nightly releases * simplify version fetching for both release workflows * revert docker-build v1 to original * remove removed input * Add push to registry flag for full workflow and add warning in old docker build workflow * Add uv setup * Add uv setup again * Try esbuild js native build * require a valid tag for release * Echo inputs * Reference inputs correctly * Try updating conditional * removes lfx temporarily for the 1.6 release * Fix boolean comparison * Add flag to allow creation of github release after pypi already exists * Add always to make it run * Use correct tag and remove commit since it's the tag for release * allow updates * Revert the lfx removals and flags to allow building without releasing * standardize conditional inputs * clean version check * temporary skip of jobs in release; revert me * Add arm64 to options * Try rustflags env var * echo the should_run_ci as well --- .github/workflows/ci.yml | 1 + .github/workflows/docker-build-v2.yml | 492 +++++++++++++++++++ .github/workflows/docker-build.yml | 19 + .github/workflows/docker-nightly-build.yml | 407 +++++++++++++++ .github/workflows/nightly_build.yml | 43 +- .github/workflows/release.yml | 128 +++-- .github/workflows/release_nightly.yml | 40 +- docker/build_and_push.Dockerfile | 2 +- docker/build_and_push_base.Dockerfile | 4 +- docker/build_and_push_ep.Dockerfile | 2 +- docker/build_and_push_with_extras.Dockerfile | 2 +- 11 files changed, 1076 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/docker-build-v2.yml create mode 100644 .github/workflows/docker-nightly-build.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b78ab2b916d9..5e1152455194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,7 @@ jobs: - run: echo "Labels -> ${{ join(github.event.pull_request.labels.*.name, ',') }}" - run: echo "IsDraft -> ${{ github.event.pull_request.draft }}" - run: echo "Event name -> ${{ github.event_name }}" + - run: echo "Should run ci -> ${{ (github.event.pull_request.draft == false) || (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'merge_group') }}" - run: echo "Should run tests -> ${{ !contains(github.event.pull_request.labels.*.name, 'fast-track') || github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' }}" path-filter: diff --git a/.github/workflows/docker-build-v2.yml b/.github/workflows/docker-build-v2.yml new file mode 100644 index 000000000000..e5a36c859f35 --- /dev/null +++ b/.github/workflows/docker-build-v2.yml @@ -0,0 +1,492 @@ +name: Docker Build and Push v2 +run-name: Docker Build and Push @${{ inputs.release_type }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + release_type: + required: true + type: string + description: "Release type. One of 'main', 'main-ep', 'base', 'main-all'." + pre_release: + required: false + type: boolean + default: false + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located." + push_to_registry: + required: false + type: boolean + default: true + description: "Whether to push images to registries. Set to false for testing builds without publishing." + + workflow_dispatch: + inputs: + release_type: + description: "Type of release. One of 'main', 'main-ep', 'base', 'main-all'." + required: true + type: choice + options: + - main + - main-ep + - base + - main-all + pre_release: + description: "Whether this is a pre-release." + required: false + type: boolean + default: false + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located." + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.13" + +jobs: + build-base: + name: Build Base Package + if: inputs.release_type == 'base' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting base version from pyproject.toml" + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main: + name: Build Main Package + if: inputs.release_type == 'main' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-ep: + name: Build Main EP Package + if: inputs.release_type == 'main-ep' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-ep:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-ep:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_ep.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_ep.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-all: + name: Build Main All Package + if: inputs.release_type == 'main-all' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-all:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-all:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + create-manifest: + name: Create Multi-Arch Manifest + needs: [build-base, build-main, build-main-ep, build-main-all] + runs-on: ubuntu-latest + if: always() && inputs.push_to_registry && (needs.build-base.result == 'success' || needs.build-main.result == 'success' || needs.build-main-ep.result == 'success' || needs.build-main-all.result == 'success') + steps: + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Get version + id: version + run: | + if [[ "${{ inputs.release_type }}" == "base" ]]; then + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + else + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + fi + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + case "${{ inputs.release_type }}" in + "base") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow:base-${version},ghcr.io/langflow-ai/langflow:base-${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow:base-${version},langflowai/langflow:base-latest,ghcr.io/langflow-ai/langflow:base-${version},ghcr.io/langflow-ai/langflow:base-latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow:base-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:base-${version}" >> $GITHUB_OUTPUT + ;; + "main") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow:${version},ghcr.io/langflow-ai/langflow:${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow:${version},langflowai/langflow:latest,ghcr.io/langflow-ai/langflow:${version},ghcr.io/langflow-ai/langflow:latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:${version}" >> $GITHUB_OUTPUT + ;; + "main-ep") + echo "final_tags=langflowai/langflow-ep:${version},langflowai/langflow-ep:latest,ghcr.io/langflow-ai/langflow-ep:${version},ghcr.io/langflow-ai/langflow-ep:latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow-ep:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-ep:${version}" >> $GITHUB_OUTPUT + ;; + "main-all") + echo "final_tags=langflowai/langflow-all:${version},langflowai/langflow-all:latest,ghcr.io/langflow-ai/langflow-all:${version},ghcr.io/langflow-ai/langflow-all:latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow-all:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-all:${version}" >> $GITHUB_OUTPUT + ;; + *) + echo "Error: Invalid release_type: ${{ inputs.release_type }}" >&2 + exit 1 + ;; + esac + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Create and push multi-arch manifests + run: | + # Split tags and create manifests for each + IFS=',' read -ra TAGS <<< "${{ steps.tags.outputs.final_tags }}" + for tag in "${TAGS[@]}"; do + echo "Creating manifest for $tag" + + # Determine architecture-specific tags + if [[ "$tag" == *"langflowai"* ]]; then + amd64_tag="${{ steps.tags.outputs.arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.arch_base }}-arm64" + else + amd64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-arm64" + fi + + docker buildx imagetools create \ + --tag "$tag" \ + "$amd64_tag" \ + "$arm64_tag" + done \ No newline at end of file diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9890be7519a4..193b15067438 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -23,6 +23,11 @@ on: required: false type: string description: "Ref to check out. If not specified, will default to the main version or current branch." + warning_check: + required: true + type: boolean + description: "Warning - use docker-build-v2 unless you have a very valid reason for using this deprecated workflow. By setting to True, you acknowledge all risks of using a possibly breaking workflow." + default: false workflow_dispatch: inputs: @@ -47,6 +52,11 @@ on: required: false type: string description: "Ref to check out. If not specified, will default to the main version or current branch." + warning_check: + description: "Warning - use docker-build-v2 unless you have a very valid reason for using this deprecated workflow. By setting to True, you acknowledge all risks of using a possibly breaking workflow." + required: true + type: boolean + default: false env: @@ -60,6 +70,15 @@ jobs: outputs: version: ${{ steps.get-version-input.outputs.version || steps.get-version-base.outputs.version || steps.get-version-main.outputs.version }} steps: + - name: Verify warning check + run: | + if [[ "${{ inputs.warning_check }}" == "false" ]]; then + echo "Warning - use docker-build-v2 unless you have a very valid reason for using this deprecated workflow. By setting to True, you acknowledge all risks of using a possibly breaking workflow." + exit 1 + else + echo "User has acknowledged the risks of using this deprecated workflow. Proceeding with build." + fi + - name: Verify a main version exists if: ${{ inputs.main_version == '' }} run: | diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml new file mode 100644 index 000000000000..dbcefdbcd3da --- /dev/null +++ b/.github/workflows/docker-nightly-build.yml @@ -0,0 +1,407 @@ +name: Docker Nightly Build and Push +run-name: Docker Nightly Build @${{ inputs.release_type }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + release_type: + required: true + type: string + description: "Nightly release type. One of 'nightly-main', 'nightly-base', 'nightly-main-all'." + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located." + push_to_registry: + required: false + type: boolean + default: true + description: "Whether to push images to registries. Set to false for testing builds without publishing." + + workflow_dispatch: + inputs: + release_type: + description: "Nightly release type. One of 'nightly-main', 'nightly-base', 'nightly-main-all'." + required: true + type: choice + options: + - nightly-main + - nightly-base + - nightly-main-all + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located. Note that if running via Github Actions, this tag (formatted as, e.g., v1.5.1.dev36) must be manually created and pushed to a branch prior to running the workflow." + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.13" + +jobs: + build-nightly-base: + name: Build Nightly Base Package + if: inputs.release_type == 'nightly-base' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version for tagging + id: version + run: | + echo "Extracting base version from pyproject.toml" + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + + # Verify nightly tag format + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then + echo "Base version format is incorrect. Must be in the format (e.g.) v1.5.1.dev36" + exit 1 + fi + + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:nightly-base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:nightly-base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-nightly-main: + name: Build Nightly Main Package + if: inputs.release_type == 'nightly-main' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version for tagging + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + + # Verify nightly tag format + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then + echo "Main version format is incorrect. Must be in the format (e.g.) v1.5.1.dev36" + exit 1 + fi + + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-nightly-main-all: + name: Build Nightly Main All Package + if: inputs.release_type == 'nightly-main-all' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version for tagging + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + + # Verify nightly tag format + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then + echo "Main version format is incorrect. Must be in the format (e.g.) v1.5.1.dev36" + exit 1 + fi + + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-all:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-all:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + create-nightly-manifest: + name: Create Multi-Arch Nightly Manifest + needs: [build-nightly-base, build-nightly-main, build-nightly-main-all] + runs-on: ubuntu-latest + if: always() && inputs.push_to_registry && (needs.build-nightly-base.result == 'success' || needs.build-nightly-main.result == 'success' || needs.build-nightly-main-all.result == 'success') + steps: + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Get version + id: version + run: | + if [[ "${{ inputs.release_type }}" == "nightly-base" ]]; then + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + else + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + fi + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + case "${{ inputs.release_type }}" in + "nightly-base") + echo "final_tags=langflowai/langflow:nightly-base-${version},langflowai/langflow:nightly-base-latest,ghcr.io/langflow-ai/langflow:nightly-base-${version},ghcr.io/langflow-ai/langflow:nightly-base-latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow:nightly-base-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:nightly-base-${version}" >> $GITHUB_OUTPUT + ;; + "nightly-main") + echo "final_tags=langflowai/langflow:nightly-${version},langflowai/langflow:nightly-latest,ghcr.io/langflow-ai/langflow:nightly-${version},ghcr.io/langflow-ai/langflow:nightly-latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow:nightly-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:nightly-${version}" >> $GITHUB_OUTPUT + ;; + "nightly-main-all") + echo "final_tags=langflowai/langflow-all:nightly-${version},langflowai/langflow-all:nightly-latest,ghcr.io/langflow-ai/langflow-all:nightly-${version},ghcr.io/langflow-ai/langflow-all:nightly-latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow-all:nightly-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-all:nightly-${version}" >> $GITHUB_OUTPUT + ;; + *) + echo "Error: Invalid release_type: ${{ inputs.release_type }}" >&2 + exit 1 + ;; + esac + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Create and push multi-arch nightly manifests + run: | + # Split tags and create manifests for each + IFS=',' read -ra TAGS <<< "${{ steps.tags.outputs.final_tags }}" + for tag in "${TAGS[@]}"; do + echo "Creating manifest for $tag" + + # Determine architecture-specific tags + if [[ "$tag" == *"langflowai"* ]]; then + amd64_tag="${{ steps.tags.outputs.arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.arch_base }}-arm64" + else + amd64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-arm64" + fi + + docker buildx imagetools create \ + --tag "$tag" \ + "$amd64_tag" \ + "$arm64_tag" + done \ No newline at end of file diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index e8fad2d1d3de..7692de6aef0c 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -2,6 +2,31 @@ name: Nightly Build on: workflow_dispatch: + inputs: + runs_on: + description: "Runner to use for tests (use self-hosted for safe/release code)" + required: false + type: choice + options: + - ubuntu-latest + - self-hosted + - "[self-hosted, linux, ARM64, langflow-ai-arm64-40gb]" + default: ubuntu-latest + skip_frontend_tests: + description: "Skip frontend tests. Only do this for testing purposes." + required: false + type: boolean + default: false + skip_backend_tests: + description: "Skip backend tests. Only do this for testing purposes." + required: false + type: boolean + default: false + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: false + type: boolean + default: true schedule: # Run job at 00:00 UTC (4:00 PM PST / 5:00 PM PDT) - cron: "0 0 * * *" @@ -11,6 +36,15 @@ env: PYTHON_VERSION: "3.13" jobs: + validate-inputs: + runs-on: ubuntu-latest + steps: + - name: Validate inputs + if: inputs.push_to_registry && (inputs.skip_frontend_tests || inputs.skip_backend_tests) + run: | + echo "Cannot skip tests while push_to_registry is true." + exit 1 + create-nightly-tag: if: github.repository == 'langflow-ai/langflow' runs-on: ubuntu-latest @@ -125,7 +159,7 @@ jobs: fi frontend-tests: - if: github.repository == 'langflow-ai/langflow' + if: github.repository == 'langflow-ai/langflow' && !inputs.skip_frontend_tests name: Run Frontend Tests needs: create-nightly-tag uses: ./.github/workflows/typescript_test.yml @@ -139,7 +173,7 @@ jobs: TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} backend-unit-tests: - if: github.repository == 'langflow-ai/langflow' + if: github.repository == 'langflow-ai/langflow' && !inputs.skip_backend_tests name: Run Backend Unit Tests needs: create-nightly-tag uses: ./.github/workflows/python_test.yml @@ -161,15 +195,16 @@ jobs: # ref: ${{ needs.create-nightly-tag.outputs.tag }} release-nightly-build: - if: github.repository == 'langflow-ai/langflow' + if: github.repository == 'langflow-ai/langflow' && (needs.frontend-tests.result == 'success' || needs.frontend-tests.result == 'skipped') && (needs.backend-unit-tests.result == 'success' || needs.backend-unit-tests.result == 'skipped') name: Run Nightly Langflow Build - needs: [frontend-tests, backend-unit-tests, create-nightly-tag] + needs: [create-nightly-tag, frontend-tests, backend-unit-tests] uses: ./.github/workflows/release_nightly.yml with: build_docker_base: true build_docker_main: true nightly_tag_main: ${{ needs.create-nightly-tag.outputs.main_tag }} nightly_tag_base: ${{ needs.create-nightly-tag.outputs.base_tag }} + push_to_registry: ${{ inputs.push_to_registry != false }} secrets: inherit # slack-notification: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b896747a548d..6d794a495675 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,8 @@ run-name: Langflow Release by @${{ github.actor }} on: workflow_dispatch: inputs: - release_branch: - description: "Branch to release from. This is the branch that contains the source code for the release." + release_tag: + description: "Tag to release from. This is the tag that contains the source code for the release." required: true type: string release_package_base: @@ -47,37 +47,76 @@ on: jobs: + echo-inputs: + name: Echo Workflow Inputs + runs-on: ubuntu-latest + steps: + - name: Echo workflow inputs + run: | + echo "release_tag: ${{ inputs.release_tag }}" + echo "release_package_base: ${{ inputs.release_package_base }}" + echo "release_package_main: ${{ inputs.release_package_main }}" + echo "release_lfx: ${{ inputs.release_lfx }}" + echo "build_docker_base: ${{ inputs.build_docker_base }}" + echo "build_docker_main: ${{ inputs.build_docker_main }}" + echo "build_docker_ep: ${{ inputs.build_docker_ep }}" + echo "pre_release: ${{ inputs.pre_release }}" + echo "create_release: ${{ inputs.create_release }}" + echo "runs_on: ${{ inputs.runs_on }}" + + validate-tag: + name: Validate Tag Input + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 # Fetch all history - required for tags (?) + - name: Validate that input is a tag, not a branch + run: | + # Check if the input exists as a tag + if ! git tag -l | grep -q "^${{ inputs.release_tag }}$"; then + echo "Error: '${{ inputs.release_tag }}' is not a valid tag." + echo "Available tags:" + git tag -l | head -20 + exit 1 + fi + + # Check if the input also exists as a branch (warn if so, but don't fail) + if git branch -r | grep -q "origin/${{ inputs.release_tag }}$"; then + echo "Tag '${{ inputs.release_tag }}' also exists as a branch. Exiting out of caution." + exit 1 + fi + + echo "Validated: '${{ inputs.release_tag }}' is a valid tag." + ci: - if: ${{ github.event.inputs.release_package_base == 'true' || github.event.inputs.release_package_main == 'true' }} + if: inputs.release_package_base || inputs.release_package_main || inputs.release_lfx name: CI + needs: [validate-tag] uses: ./.github/workflows/ci.yml with: - branch: ${{ inputs.release_branch }} + branch: ${{ inputs.release_tag }} python-versions: "['3.10', '3.11', '3.12', '3.13']" frontend-tests-folder: "tests" release: true + runs-on: ${{ (inputs['runs_on'] && startsWith(format('{0}', inputs['runs_on']), '[') && fromJSON(inputs['runs_on'])) || inputs['runs_on'] || inputs['runs_on'] || 'ubuntu-latest' }} secrets: inherit build-base: name: Build Langflow Base needs: [ci] - if: inputs.release_package_base == true + if: inputs.release_package_base runs-on: ubuntu-latest outputs: version: ${{ steps.check-version.outputs.version }} skipped: ${{ steps.check-version.outputs.skipped }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: ${{ inputs.release_branch }} - - name: Debug branch information - run: | - echo "Workflow triggered from: ${{ github.ref }}" - echo "Checking out branch: ${{ inputs.release_branch }}" - echo "Current branch after checkout: $(git branch --show-current)" - echo "Current commit: $(git rev-parse HEAD)" + ref: ${{ inputs.release_tag }} - name: Setup Environment uses: astral-sh/setup-uv@v6 with: @@ -138,22 +177,16 @@ jobs: build-main: name: Build Langflow Main - if: inputs.release_package_main == true + if: inputs.release_package_main needs: [build-base] runs-on: ubuntu-latest outputs: version: ${{ steps.check-version.outputs.version }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: ${{ inputs.release_branch }} - - name: Debug branch information - run: | - echo "Workflow triggered from: ${{ github.ref }}" - echo "Checking out branch: ${{ inputs.release_branch }}" - echo "Current branch after checkout: $(git branch --show-current)" - echo "Current commit: $(git rev-parse HEAD)" + ref: ${{ inputs.release_tag }} - name: Setup Environment uses: astral-sh/setup-uv@v6 with: @@ -169,7 +202,7 @@ jobs: # If pre-release is true, we need to check if ["a", "b", "rc", "dev", "post"] is in the version string # if the version string is incorrect, we need to exit the workflow - name: Check if pre-release - if: inputs.pre_release == 'true' + if: inputs.pre_release run: | version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//') if [[ "${version}" =~ ^([0-9]+\.)?([0-9]+\.)?[0-9]+((a|b|rc|dev|post)([0-9]+))$ ]]; then @@ -185,7 +218,8 @@ jobs: last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) if [ "$version" = "$last_released_version" ]; then echo "Version $version is already released. Skipping release." - exit 1 + echo skipped=true >> $GITHUB_OUTPUT + exit 0 else echo version=$version >> $GITHUB_OUTPUT fi @@ -223,7 +257,7 @@ jobs: test-cross-platform: name: Test Cross-Platform Installation - if: inputs.release_package_base == true || inputs.release_package_main == true + if: inputs.release_package_base || inputs.release_package_main needs: [build-base, build-main] uses: ./.github/workflows/cross-platform-test.yml with: @@ -232,7 +266,7 @@ jobs: publish-base: name: Publish Langflow Base to PyPI - if: inputs.release_package_base == true + if: inputs.release_package_base needs: [build-base, test-cross-platform] runs-on: ubuntu-latest steps: @@ -255,7 +289,7 @@ jobs: publish-main: name: Publish Langflow Main to PyPI - if: inputs.release_package_main == true + if: inputs.release_package_main needs: [build-main, test-cross-platform, publish-base] runs-on: ubuntu-latest steps: @@ -284,51 +318,58 @@ jobs: ref: ${{ inputs.release_branch }} base_version: ${{ needs.build-base.outputs.version }} main_version: ${{ needs.build-main.outputs.version }} + + call_docker_build_base: + name: Call Docker Build Workflow for Langflow Base + if: inputs.build_docker_base + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} release_type: base pre_release: ${{ inputs.pre_release }} secrets: inherit call_docker_build_main_ep: name: Call Docker Build Workflow for Langflow with Entrypoint - if: inputs.build_docker_ep == true - needs: [build-main, publish-main, call_docker_build_base] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_ep + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: main-ep pre_release: False secrets: inherit call_docker_build_main: name: Call Docker Build Workflow for Langflow - if: inputs.build_docker_main == true - needs: [build-main, publish-main, call_docker_build_main_ep] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_main + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: main pre_release: ${{ inputs.pre_release }} secrets: inherit call_docker_build_main_all: name: Call Docker Build Workflow for langflow-all - if: inputs.build_docker_main == true - needs: [build-main, publish-main] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_main + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: main-all pre_release: ${{ inputs.pre_release }} secrets: inherit + create_release: name: Create Release runs-on: ubuntu-latest needs: [build-main, publish-main] + if: always() && inputs.create_release && (needs.publish-main.result == 'success' ) steps: - uses: actions/download-artifact@v4 with: @@ -343,4 +384,5 @@ jobs: generateReleaseNotes: true prerelease: ${{ inputs.pre_release }} tag: ${{ needs.build-main.outputs.version }} - commit: ${{ inputs.release_branch }} + allowUpdates: true + updateOnlyUnreleased: false diff --git a/.github/workflows/release_nightly.yml b/.github/workflows/release_nightly.yml index 80e6df568423..48aa96628ae2 100644 --- a/.github/workflows/release_nightly.yml +++ b/.github/workflows/release_nightly.yml @@ -52,6 +52,11 @@ on: description: "Tag for the nightly base build" required: true type: string + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: true + type: boolean + default: false env: POETRY_VERSION: "1.8.3" @@ -222,7 +227,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + ref: ${{ inputs.nightly_tag_main }} + persist-credentials: true - name: Download base artifact uses: actions/download-artifact@v4 with: @@ -247,7 +255,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + ref: ${{ inputs.nightly_tag_main }} + persist-credentials: true - name: Download main artifact uses: actions/download-artifact@v4 with: @@ -267,23 +278,24 @@ jobs: call_docker_build_base: name: Call Docker Build Workflow for Langflow Base - if: always() && ${{ inputs.build_docker_base == 'true' }} + if: always() && inputs.build_docker_base needs: [build-nightly-base, build-nightly-main] - uses: ./.github/workflows/docker-build.yml + uses: ./.github/workflows/docker-nightly-build.yml with: + ref: ${{ inputs.nightly_tag_main }} release_type: nightly-base - base_version: ${{ inputs.nightly_tag_base }} - main_version: ${{ inputs.nightly_tag_main }} + push_to_registry: ${{ inputs.push_to_registry }} secrets: inherit call_docker_build_main: name: Call Docker Build Workflow for Langflow - if: always() && ${{ inputs.build_docker_main == 'true' }} + if: always() && inputs.build_docker_main needs: [build-nightly-main, call_docker_build_base] - uses: ./.github/workflows/docker-build.yml + uses: ./.github/workflows/docker-nightly-build.yml with: + ref: ${{ inputs.nightly_tag_main }} release_type: nightly-main - main_version: ${{ inputs.nightly_tag_main }} + push_to_registry: ${{ inputs.push_to_registry }} secrets: inherit # TODO: Uncomment this when our runner can fit the builds that contain pytorch (and other large dependencies) @@ -291,18 +303,20 @@ jobs: # name: Call Docker Build Workflow for langflow-all # if: always() && ${{ inputs.build_docker_main == 'true' }} # needs: [build-nightly-main] - # uses: ./.github/workflows/docker-build.yml + # uses: ./.github/workflows/docker-nightly-build.yml # with: + # ref: ${{ inputs.nightly_tag_main }} # release_type: nightly-main-all # main_version: ${{ inputs.nightly_tag_main }} # secrets: inherit call_docker_build_main_ep: name: Call Docker Build Workflow for Langflow with Entrypoint - if: always() && ${{ inputs.build_docker_ep == 'true' }} + if: always() && inputs.build_docker_ep needs: [build-nightly-main, call_docker_build_main] - uses: ./.github/workflows/docker-build.yml + uses: ./.github/workflows/docker-build-v2.yml with: + ref: ${{ inputs.nightly_tag_main }} release_type: main-ep - main_version: ${{ inputs.nightly_tag_main }} + push_to_registry: ${{ inputs.push_to_registry }} secrets: inherit diff --git a/docker/build_and_push.Dockerfile b/docker/build_and_push.Dockerfile index da964ca924ae..a95675437156 100644 --- a/docker/build_and_push.Dockerfile +++ b/docker/build_and_push.Dockerfile @@ -51,7 +51,7 @@ COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend RUN --mount=type=cache,target=/root/.npm \ npm ci \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/langflow/frontend \ && rm -rf /tmp/src/frontend diff --git a/docker/build_and_push_base.Dockerfile b/docker/build_and_push_base.Dockerfile index 8c1dd316afb5..72c1018a6037 100644 --- a/docker/build_and_push_base.Dockerfile +++ b/docker/build_and_push_base.Dockerfile @@ -52,8 +52,10 @@ COPY ./src /app/src COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend +# Increase memory and disable concurrent builds to avoid esbuild crashes on emulated architectures +# Force esbuild to use JS implementation on emulated architectures to avoid native binary crashes RUN npm install \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/base/langflow/frontend \ && rm -rf /tmp/src/frontend diff --git a/docker/build_and_push_ep.Dockerfile b/docker/build_and_push_ep.Dockerfile index 073b30004b53..d2337472b1d5 100644 --- a/docker/build_and_push_ep.Dockerfile +++ b/docker/build_and_push_ep.Dockerfile @@ -51,7 +51,7 @@ COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend RUN --mount=type=cache,target=/root/.npm \ npm ci \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/langflow/frontend \ && rm -rf /tmp/src/frontend diff --git a/docker/build_and_push_with_extras.Dockerfile b/docker/build_and_push_with_extras.Dockerfile index 5a3b59660bdd..cbb5208e2cad 100644 --- a/docker/build_and_push_with_extras.Dockerfile +++ b/docker/build_and_push_with_extras.Dockerfile @@ -51,7 +51,7 @@ COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend RUN --mount=type=cache,target=/root/.npm \ npm ci \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/langflow/frontend \ && rm -rf /tmp/src/frontend From 206c4c75b8e82f021c37622743b50a12391c7b78 Mon Sep 17 00:00:00 2001 From: Adam-Aghili <149833988+Adam-Aghili@users.noreply.github.com> Date: Mon, 3 Nov 2025 13:00:58 -0500 Subject: [PATCH 2/4] ci: publish v2 docker images (#10350) * ci: publish v2 docker images since or transition to v2 in 1.6.x we have not been publishing any of our docker images. * chore: try fixing lfx cleanup error lfx cleanup error * chore: address rabbitcode comments * chore: address rabit uv comment * chore: remove build-args to see if it passes * chore: skip ci * chore: add back in LANGFLOW_IMAGE * chore: debug inputs.ref and matrix.langflow_image * chore: add {} to fix InvalidDefaultArgInForm * chore: try adding quotes * chore: try ENV LANGFLOW_IMAGE=${LANGFLOW_IMAGE} * chore: add build args to GitHub Container Registry * chore: clean up * chore: seperate out frontend and backend images * chore: update create-manifest * chore: remove ci step * chore: update release * chore: clean up * chore: clean up langflow-image * chore: revert ci removal * fix: add back in main logic update back/frontend add main back in update back/frontend to match orginal man logic more * chore: more clean up to match main * chore: remove arch from lagnflowimage for backend * chore: add misisng - for ghcr.io * chore: skip ci * chore: build_main and build_and_push_backend fixes * chore: seperate ghcr and docker publishing with if * chore: add back CI step --------- Co-authored-by: Adam Aghili --- .github/workflows/docker-build-v2.yml | 247 ++++++++++++++++++++++- .github/workflows/python_test.yml | 3 - .github/workflows/release.yml | 42 +++- docker/build_and_push_backend.Dockerfile | 2 +- 4 files changed, 277 insertions(+), 17 deletions(-) diff --git a/.github/workflows/docker-build-v2.yml b/.github/workflows/docker-build-v2.yml index e5a36c859f35..79985e0fd497 100644 --- a/.github/workflows/docker-build-v2.yml +++ b/.github/workflows/docker-build-v2.yml @@ -7,7 +7,7 @@ on: release_type: required: true type: string - description: "Release type. One of 'main', 'main-ep', 'base', 'main-all'." + description: "Release type. One of 'main', 'main-backend', 'main-frontend', 'main-ep', 'base', 'main-all'." pre_release: required: false type: boolean @@ -25,11 +25,13 @@ on: workflow_dispatch: inputs: release_type: - description: "Type of release. One of 'main', 'main-ep', 'base', 'main-all'." + description: "Type of release. One of 'main', 'main-backend', 'main-frontend', 'main-ep', 'base', 'main-all'." required: true type: choice options: - main + - main-backend + - main-frotnend - main-ep - base - main-all @@ -64,7 +66,6 @@ jobs: runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] runs-on: ${{ matrix.runner }} steps: - - name: Check out the code uses: actions/checkout@v5 with: @@ -148,7 +149,6 @@ jobs: runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] runs-on: ${{ matrix.runner }} steps: - - name: Check out the code uses: actions/checkout@v5 with: @@ -220,6 +220,212 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + build-main-backend: + name: Build Main Backend Package + if: ${{ inputs.release_type == 'main-backend' }} + strategy: + matrix: + component: [docker-backend, ghcr-backend] + include: + - component: docker-backend + arch: amd64 + runner: Langflow-runner + + - component: docker-backend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + + - component: ghcr-backend + arch: amd64 + runner: Langflow-runner + + - component: ghcr-backend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree 2>/dev/null | grep '^langflow' | grep -v '^langflow-base' | cut -d' ' -f2 | sed 's/^v//') + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-backend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-backend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: ${{ matrix.component == 'docker-backend' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push to Docker Hub + if: ${{ matrix.component == 'docker-backend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + build-args: | + LANGFLOW_IMAGE=langflowai/langflow:${{ steps.version.outputs.version }}-${{ matrix.arch }} + file: ./docker/build_and_push_backend.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Login to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-backend' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-backend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + build-args: | + LANGFLOW_IMAGE=ghcr.io/langflow-ai/langflow:${{ steps.version.outputs.version }}-${{ matrix.arch }} + file: ./docker/build_and_push_backend.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-frontend: + name: Build Main Frontend Package + if: ${{ inputs.release_type == 'main-frontend' }} + strategy: + matrix: + component: [docker-frontend, ghcr-frontend] + include: + - component: docker-frontend + arch: amd64 + runner: Langflow-runner + + - component: docker-frontend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + + - component: ghcr-frontend + arch: amd64 + runner: Langflow-runner + + - component: ghcr-frontend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree 2>/dev/null | grep '^langflow' | grep -v '^langflow-base' | cut -d' ' -f2 | sed 's/^v//') + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-frontend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-frontend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: ${{ matrix.component == 'docker-frontend' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push to Docker Hub + if: ${{ matrix.component == 'docker-frontend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Login to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-frontend' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-frontend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + build-main-ep: name: Build Main EP Package if: inputs.release_type == 'main-ep' @@ -232,7 +438,6 @@ jobs: runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] runs-on: ${{ matrix.runner }} steps: - - name: Check out the code uses: actions/checkout@v5 with: @@ -389,9 +594,17 @@ jobs: create-manifest: name: Create Multi-Arch Manifest - needs: [build-base, build-main, build-main-ep, build-main-all] + needs: + [ + build-base, + build-main, + build-main-backend, + build-main-frontend, + build-main-ep, + build-main-all, + ] runs-on: ubuntu-latest - if: always() && inputs.push_to_registry && (needs.build-base.result == 'success' || needs.build-main.result == 'success' || needs.build-main-ep.result == 'success' || needs.build-main-all.result == 'success') + if: ${{ always() && inputs.push_to_registry && (needs.build-base.result == 'success' || needs.build-main.result == 'success' || needs.build-main-backend.result == 'success' || needs.build-main-frontend.result == 'success' || needs.build-main-ep.result == 'success' || needs.build-main-all.result == 'success') }} steps: - name: Setup Environment uses: astral-sh/setup-uv@v6 @@ -440,6 +653,24 @@ jobs: echo "arch_base=langflowai/langflow:${version}" >> $GITHUB_OUTPUT echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:${version}" >> $GITHUB_OUTPUT ;; + "main-backend") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow-backend:${version},ghcr.io/langflow-ai/langflow-backend:${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow-backend:${version},langflowai/langflow-backend:latest,ghcr.io/langflow-ai/langflow-backend:${version},ghcr.io/langflow-ai/langflow-backend:latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow-backend:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-backend:${version}" >> $GITHUB_OUTPUT + ;; + "main-frontend") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow-frontend:${version},ghcr.io/langflow-ai/langflow-frontend:${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow-frontend:${version},langflowai/langflow-frontend:latest,ghcr.io/langflow-ai/langflow-frontend:${version},ghcr.io/langflow-ai/langflow-frontend:latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow-frontend:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-frontend:${version}" >> $GITHUB_OUTPUT + ;; "main-ep") echo "final_tags=langflowai/langflow-ep:${version},langflowai/langflow-ep:latest,ghcr.io/langflow-ai/langflow-ep:${version},ghcr.io/langflow-ai/langflow-ep:latest" >> $GITHUB_OUTPUT echo "arch_base=langflowai/langflow-ep:${version}" >> $GITHUB_OUTPUT @@ -489,4 +720,4 @@ jobs: --tag "$tag" \ "$amd64_tag" \ "$arm64_tag" - done \ No newline at end of file + done diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index f9faf7625a46..d7f7e38b7822 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -40,7 +40,6 @@ env: jobs: build: - name: Unit Tests - Python ${{ matrix.python-version }} - Group ${{ matrix.group }} runs-on: ubuntu-latest strategy: @@ -105,7 +104,6 @@ jobs: htmlcov/ retention-days: 30 - integration-tests: name: Integration Tests - Python ${{ matrix.python-version }} runs-on: ubuntu-latest @@ -198,4 +196,3 @@ jobs: else echo "Server terminated successfully" fi - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d794a495675..c1ee13f54d5f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,8 +44,6 @@ on: type: boolean default: true - - jobs: echo-inputs: name: Echo Workflow Inputs @@ -103,7 +101,6 @@ jobs: runs-on: ${{ (inputs['runs_on'] && startsWith(format('{0}', inputs['runs_on']), '[') && fromJSON(inputs['runs_on'])) || inputs['runs_on'] || inputs['runs_on'] || 'ubuntu-latest' }} secrets: inherit - build-base: name: Build Langflow Base needs: [ci] @@ -350,6 +347,43 @@ jobs: ref: ${{ inputs.release_tag }} release_type: main pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} + secrets: inherit + + call_docker_build_main_backend: + name: Call Docker Build Workflow for Langflow Backend + if: ${{ inputs.build_docker_main }} + needs: [ci] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} + release_type: main-backend + pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} + secrets: inherit + + call_docker_build_main_frontend: + name: Call Docker Build Workflow for Langflow Frontend + if: ${{ inputs.build_docker_main }} + needs: [ci] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} + release_type: main-frontend + pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} + secrets: inherit + + call_docker_build_main_ep: + name: Call Docker Build Workflow for Langflow with Entrypoint + if: ${{ inputs.build_docker_main }} + needs: [ci] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} + release_type: main-ep + pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} secrets: inherit call_docker_build_main_all: @@ -363,8 +397,6 @@ jobs: pre_release: ${{ inputs.pre_release }} secrets: inherit - - create_release: name: Create Release runs-on: ubuntu-latest diff --git a/docker/build_and_push_backend.Dockerfile b/docker/build_and_push_backend.Dockerfile index 66cf65212548..894718c43871 100644 --- a/docker/build_and_push_backend.Dockerfile +++ b/docker/build_and_push_backend.Dockerfile @@ -2,7 +2,7 @@ # Keep this syntax directive! It's used to enable Docker BuildKit ARG LANGFLOW_IMAGE -FROM $LANGFLOW_IMAGE +FROM ${LANGFLOW_IMAGE} RUN rm -rf /app/.venv/langflow/frontend From 34a1368a670138b11311bac8bcdfdec81790ca5b Mon Sep 17 00:00:00 2001 From: Adam-Aghili <149833988+Adam-Aghili@users.noreply.github.com> Date: Thu, 6 Nov 2025 12:11:50 -0500 Subject: [PATCH 3/4] chore: fix main-frotnend typo (#10518) main-frotnend -> main-frontend Co-authored-by: Adam Aghili --- .github/workflows/docker-build-v2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-v2.yml b/.github/workflows/docker-build-v2.yml index 79985e0fd497..e49a931d73d0 100644 --- a/.github/workflows/docker-build-v2.yml +++ b/.github/workflows/docker-build-v2.yml @@ -31,7 +31,7 @@ on: options: - main - main-backend - - main-frotnend + - main-frontend - main-ep - base - main-all From 01df57e5ad04b40b32122d0ff5f287788aafa7b1 Mon Sep 17 00:00:00 2001 From: Adam Aghili Date: Thu, 6 Nov 2025 14:32:53 -0500 Subject: [PATCH 4/4] chore: fix merge dup --- .github/workflows/release.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c1ee13f54d5f..846eceb51bda 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -306,16 +306,6 @@ jobs: run: | uv publish dist/*.whl - call_docker_build_base: - name: Call Docker Build Workflow for Langflow Base - if: inputs.build_docker_base == true - needs: [build-base, build-main, publish-base, publish-main] - uses: ./.github/workflows/docker-build.yml - with: - ref: ${{ inputs.release_branch }} - base_version: ${{ needs.build-base.outputs.version }} - main_version: ${{ needs.build-main.outputs.version }} - call_docker_build_base: name: Call Docker Build Workflow for Langflow Base if: inputs.build_docker_base