diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b78ab2b916d9..5e1152455194 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,7 @@ jobs: - run: echo "Labels -> ${{ join(github.event.pull_request.labels.*.name, ',') }}" - run: echo "IsDraft -> ${{ github.event.pull_request.draft }}" - run: echo "Event name -> ${{ github.event_name }}" + - run: echo "Should run ci -> ${{ (github.event.pull_request.draft == false) || (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call' || github.event_name == 'merge_group') }}" - run: echo "Should run tests -> ${{ !contains(github.event.pull_request.labels.*.name, 'fast-track') || github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch' || github.event_name == 'merge_group' }}" path-filter: diff --git a/.github/workflows/docker-build-v2.yml b/.github/workflows/docker-build-v2.yml new file mode 100644 index 000000000000..e49a931d73d0 --- /dev/null +++ b/.github/workflows/docker-build-v2.yml @@ -0,0 +1,723 @@ +name: Docker Build and Push v2 +run-name: Docker Build and Push @${{ inputs.release_type }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + release_type: + required: true + type: string + description: "Release type. One of 'main', 'main-backend', 'main-frontend', 'main-ep', 'base', 'main-all'." + pre_release: + required: false + type: boolean + default: false + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located." + push_to_registry: + required: false + type: boolean + default: true + description: "Whether to push images to registries. Set to false for testing builds without publishing." + + workflow_dispatch: + inputs: + release_type: + description: "Type of release. One of 'main', 'main-backend', 'main-frontend', 'main-ep', 'base', 'main-all'." + required: true + type: choice + options: + - main + - main-backend + - main-frontend + - main-ep + - base + - main-all + pre_release: + description: "Whether this is a pre-release." + required: false + type: boolean + default: false + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located." + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.13" + +jobs: + build-base: + name: Build Base Package + if: inputs.release_type == 'base' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting base version from pyproject.toml" + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main: + name: Build Main Package + if: inputs.release_type == 'main' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-backend: + name: Build Main Backend Package + if: ${{ inputs.release_type == 'main-backend' }} + strategy: + matrix: + component: [docker-backend, ghcr-backend] + include: + - component: docker-backend + arch: amd64 + runner: Langflow-runner + + - component: docker-backend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + + - component: ghcr-backend + arch: amd64 + runner: Langflow-runner + + - component: ghcr-backend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree 2>/dev/null | grep '^langflow' | grep -v '^langflow-base' | cut -d' ' -f2 | sed 's/^v//') + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-backend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-backend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: ${{ matrix.component == 'docker-backend' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push to Docker Hub + if: ${{ matrix.component == 'docker-backend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + build-args: | + LANGFLOW_IMAGE=langflowai/langflow:${{ steps.version.outputs.version }}-${{ matrix.arch }} + file: ./docker/build_and_push_backend.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Login to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-backend' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-backend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + build-args: | + LANGFLOW_IMAGE=ghcr.io/langflow-ai/langflow:${{ steps.version.outputs.version }}-${{ matrix.arch }} + file: ./docker/build_and_push_backend.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-frontend: + name: Build Main Frontend Package + if: ${{ inputs.release_type == 'main-frontend' }} + strategy: + matrix: + component: [docker-frontend, ghcr-frontend] + include: + - component: docker-frontend + arch: amd64 + runner: Langflow-runner + + - component: docker-frontend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + + - component: ghcr-frontend + arch: amd64 + runner: Langflow-runner + + - component: ghcr-frontend + arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree 2>/dev/null | grep '^langflow' | grep -v '^langflow-base' | cut -d' ' -f2 | sed 's/^v//') + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-frontend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-frontend:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: ${{ matrix.component == 'docker-frontend' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push to Docker Hub + if: ${{ matrix.component == 'docker-frontend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Login to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-frontend' }} + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to GitHub Container Registry + if: ${{ matrix.component == 'ghcr-frontend' }} + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/frontend/build_and_push_frontend.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + provenance: false + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-ep: + name: Build Main EP Package + if: inputs.release_type == 'main-ep' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-ep:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-ep:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_ep.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_ep.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-main-all: + name: Build Main All Package + if: inputs.release_type == 'main-all' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-all:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-all:${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + create-manifest: + name: Create Multi-Arch Manifest + needs: + [ + build-base, + build-main, + build-main-backend, + build-main-frontend, + build-main-ep, + build-main-all, + ] + runs-on: ubuntu-latest + if: ${{ always() && inputs.push_to_registry && (needs.build-base.result == 'success' || needs.build-main.result == 'success' || needs.build-main-backend.result == 'success' || needs.build-main-frontend.result == 'success' || needs.build-main-ep.result == 'success' || needs.build-main-all.result == 'success') }} + steps: + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Get version + id: version + run: | + if [[ "${{ inputs.release_type }}" == "base" ]]; then + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + else + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + fi + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + case "${{ inputs.release_type }}" in + "base") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow:base-${version},ghcr.io/langflow-ai/langflow:base-${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow:base-${version},langflowai/langflow:base-latest,ghcr.io/langflow-ai/langflow:base-${version},ghcr.io/langflow-ai/langflow:base-latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow:base-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:base-${version}" >> $GITHUB_OUTPUT + ;; + "main") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow:${version},ghcr.io/langflow-ai/langflow:${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow:${version},langflowai/langflow:latest,ghcr.io/langflow-ai/langflow:${version},ghcr.io/langflow-ai/langflow:latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:${version}" >> $GITHUB_OUTPUT + ;; + "main-backend") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow-backend:${version},ghcr.io/langflow-ai/langflow-backend:${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow-backend:${version},langflowai/langflow-backend:latest,ghcr.io/langflow-ai/langflow-backend:${version},ghcr.io/langflow-ai/langflow-backend:latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow-backend:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-backend:${version}" >> $GITHUB_OUTPUT + ;; + "main-frontend") + if [[ "${{ inputs.pre_release }}" == "true" ]]; then + echo "final_tags=langflowai/langflow-frontend:${version},ghcr.io/langflow-ai/langflow-frontend:${version}" >> $GITHUB_OUTPUT + else + echo "final_tags=langflowai/langflow-frontend:${version},langflowai/langflow-frontend:latest,ghcr.io/langflow-ai/langflow-frontend:${version},ghcr.io/langflow-ai/langflow-frontend:latest" >> $GITHUB_OUTPUT + fi + echo "arch_base=langflowai/langflow-frontend:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-frontend:${version}" >> $GITHUB_OUTPUT + ;; + "main-ep") + echo "final_tags=langflowai/langflow-ep:${version},langflowai/langflow-ep:latest,ghcr.io/langflow-ai/langflow-ep:${version},ghcr.io/langflow-ai/langflow-ep:latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow-ep:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-ep:${version}" >> $GITHUB_OUTPUT + ;; + "main-all") + echo "final_tags=langflowai/langflow-all:${version},langflowai/langflow-all:latest,ghcr.io/langflow-ai/langflow-all:${version},ghcr.io/langflow-ai/langflow-all:latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow-all:${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-all:${version}" >> $GITHUB_OUTPUT + ;; + *) + echo "Error: Invalid release_type: ${{ inputs.release_type }}" >&2 + exit 1 + ;; + esac + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Create and push multi-arch manifests + run: | + # Split tags and create manifests for each + IFS=',' read -ra TAGS <<< "${{ steps.tags.outputs.final_tags }}" + for tag in "${TAGS[@]}"; do + echo "Creating manifest for $tag" + + # Determine architecture-specific tags + if [[ "$tag" == *"langflowai"* ]]; then + amd64_tag="${{ steps.tags.outputs.arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.arch_base }}-arm64" + else + amd64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-arm64" + fi + + docker buildx imagetools create \ + --tag "$tag" \ + "$amd64_tag" \ + "$arm64_tag" + done diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 9890be7519a4..193b15067438 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -23,6 +23,11 @@ on: required: false type: string description: "Ref to check out. If not specified, will default to the main version or current branch." + warning_check: + required: true + type: boolean + description: "Warning - use docker-build-v2 unless you have a very valid reason for using this deprecated workflow. By setting to True, you acknowledge all risks of using a possibly breaking workflow." + default: false workflow_dispatch: inputs: @@ -47,6 +52,11 @@ on: required: false type: string description: "Ref to check out. If not specified, will default to the main version or current branch." + warning_check: + description: "Warning - use docker-build-v2 unless you have a very valid reason for using this deprecated workflow. By setting to True, you acknowledge all risks of using a possibly breaking workflow." + required: true + type: boolean + default: false env: @@ -60,6 +70,15 @@ jobs: outputs: version: ${{ steps.get-version-input.outputs.version || steps.get-version-base.outputs.version || steps.get-version-main.outputs.version }} steps: + - name: Verify warning check + run: | + if [[ "${{ inputs.warning_check }}" == "false" ]]; then + echo "Warning - use docker-build-v2 unless you have a very valid reason for using this deprecated workflow. By setting to True, you acknowledge all risks of using a possibly breaking workflow." + exit 1 + else + echo "User has acknowledged the risks of using this deprecated workflow. Proceeding with build." + fi + - name: Verify a main version exists if: ${{ inputs.main_version == '' }} run: | diff --git a/.github/workflows/docker-nightly-build.yml b/.github/workflows/docker-nightly-build.yml new file mode 100644 index 000000000000..dbcefdbcd3da --- /dev/null +++ b/.github/workflows/docker-nightly-build.yml @@ -0,0 +1,407 @@ +name: Docker Nightly Build and Push +run-name: Docker Nightly Build @${{ inputs.release_type }} by @${{ github.actor }} + +on: + workflow_call: + inputs: + release_type: + required: true + type: string + description: "Nightly release type. One of 'nightly-main', 'nightly-base', 'nightly-main-all'." + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located." + push_to_registry: + required: false + type: boolean + default: true + description: "Whether to push images to registries. Set to false for testing builds without publishing." + + workflow_dispatch: + inputs: + release_type: + description: "Nightly release type. One of 'nightly-main', 'nightly-base', 'nightly-main-all'." + required: true + type: choice + options: + - nightly-main + - nightly-base + - nightly-main-all + ref: + required: true + type: string + description: "Ref to check out (branch, tag, or commit). This is required -- it specifies where the source code for the release is located. Note that if running via Github Actions, this tag (formatted as, e.g., v1.5.1.dev36) must be manually created and pushed to a branch prior to running the workflow." + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: false + type: boolean + default: false + +env: + PYTHON_VERSION: "3.13" + +jobs: + build-nightly-base: + name: Build Nightly Base Package + if: inputs.release_type == 'nightly-base' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version for tagging + id: version + run: | + echo "Extracting base version from pyproject.toml" + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + + # Verify nightly tag format + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then + echo "Base version format is incorrect. Must be in the format (e.g.) v1.5.1.dev36" + exit 1 + fi + + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:nightly-base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:nightly-base-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_base.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-nightly-main: + name: Build Nightly Main Package + if: inputs.release_type == 'nightly-main' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version for tagging + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + + # Verify nightly tag format + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then + echo "Main version format is incorrect. Must be in the format (e.g.) v1.5.1.dev36" + exit 1 + fi + + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + build-nightly-main-all: + name: Build Nightly Main All Package + if: inputs.release_type == 'nightly-main-all' + strategy: + matrix: + include: + - arch: amd64 + runner: [self-hosted, linux, X64, Langflow-runners] + - arch: arm64 + runner: [self-hosted, linux, ARM64, langflow-ai-arm64-40gb] + runs-on: ${{ matrix.runner }} + steps: + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Get version for tagging + id: version + run: | + echo "Extracting main version from pyproject.toml" + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + + # Verify nightly tag format + if [[ ! "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.dev[0-9]+$ ]]; then + echo "Main version format is incorrect. Must be in the format (e.g.) v1.5.1.dev36" + exit 1 + fi + + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + echo "docker_tags=langflowai/langflow-all:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + echo "ghcr_tags=ghcr.io/langflow-ai/langflow-all:nightly-${version}-${{ matrix.arch }}" >> $GITHUB_OUTPUT + + - name: Docker cleanup + run: | + docker system prune -af --volumes || true + docker buildx prune -af || true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Build and push to Docker Hub + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.docker_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Build and push to GitHub Container Registry + uses: docker/build-push-action@v6 + with: + context: . + push: ${{ inputs.push_to_registry }} + file: ./docker/build_and_push_with_extras.Dockerfile + tags: ${{ steps.tags.outputs.ghcr_tags }} + platforms: linux/${{ matrix.arch }} + cache-from: type=gha + cache-to: type=gha,mode=max + + create-nightly-manifest: + name: Create Multi-Arch Nightly Manifest + needs: [build-nightly-base, build-nightly-main, build-nightly-main-all] + runs-on: ubuntu-latest + if: always() && inputs.push_to_registry && (needs.build-nightly-base.result == 'success' || needs.build-nightly-main.result == 'success' || needs.build-nightly-main-all.result == 'success') + steps: + - name: Setup Environment + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + cache-dependency-glob: "uv.lock" + python-version: "3.13" + prune-cache: false + + - name: Check out the code + uses: actions/checkout@v5 + with: + ref: ${{ inputs.ref }} + + - name: Get version + id: version + run: | + if [[ "${{ inputs.release_type }}" == "nightly-base" ]]; then + version=$(uv tree | grep 'langflow-base' | awk '{print $3}' | sed 's/^v//' | head -n 1) + else + version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//' | head -n 1) + fi + echo "Using version: $version" + echo version=$version >> $GITHUB_OUTPUT + + - name: Set nightly tags + id: tags + run: | + version="${{ steps.version.outputs.version }}" + case "${{ inputs.release_type }}" in + "nightly-base") + echo "final_tags=langflowai/langflow:nightly-base-${version},langflowai/langflow:nightly-base-latest,ghcr.io/langflow-ai/langflow:nightly-base-${version},ghcr.io/langflow-ai/langflow:nightly-base-latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow:nightly-base-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:nightly-base-${version}" >> $GITHUB_OUTPUT + ;; + "nightly-main") + echo "final_tags=langflowai/langflow:nightly-${version},langflowai/langflow:nightly-latest,ghcr.io/langflow-ai/langflow:nightly-${version},ghcr.io/langflow-ai/langflow:nightly-latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow:nightly-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow:nightly-${version}" >> $GITHUB_OUTPUT + ;; + "nightly-main-all") + echo "final_tags=langflowai/langflow-all:nightly-${version},langflowai/langflow-all:nightly-latest,ghcr.io/langflow-ai/langflow-all:nightly-${version},ghcr.io/langflow-ai/langflow-all:nightly-latest" >> $GITHUB_OUTPUT + echo "arch_base=langflowai/langflow-all:nightly-${version}" >> $GITHUB_OUTPUT + echo "ghcr_arch_base=ghcr.io/langflow-ai/langflow-all:nightly-${version}" >> $GITHUB_OUTPUT + ;; + *) + echo "Error: Invalid release_type: ${{ inputs.release_type }}" >&2 + exit 1 + ;; + esac + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.TEMP_GHCR_TOKEN}} + + - name: Create and push multi-arch nightly manifests + run: | + # Split tags and create manifests for each + IFS=',' read -ra TAGS <<< "${{ steps.tags.outputs.final_tags }}" + for tag in "${TAGS[@]}"; do + echo "Creating manifest for $tag" + + # Determine architecture-specific tags + if [[ "$tag" == *"langflowai"* ]]; then + amd64_tag="${{ steps.tags.outputs.arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.arch_base }}-arm64" + else + amd64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-amd64" + arm64_tag="${{ steps.tags.outputs.ghcr_arch_base }}-arm64" + fi + + docker buildx imagetools create \ + --tag "$tag" \ + "$amd64_tag" \ + "$arm64_tag" + done \ No newline at end of file diff --git a/.github/workflows/nightly_build.yml b/.github/workflows/nightly_build.yml index e8fad2d1d3de..7692de6aef0c 100644 --- a/.github/workflows/nightly_build.yml +++ b/.github/workflows/nightly_build.yml @@ -2,6 +2,31 @@ name: Nightly Build on: workflow_dispatch: + inputs: + runs_on: + description: "Runner to use for tests (use self-hosted for safe/release code)" + required: false + type: choice + options: + - ubuntu-latest + - self-hosted + - "[self-hosted, linux, ARM64, langflow-ai-arm64-40gb]" + default: ubuntu-latest + skip_frontend_tests: + description: "Skip frontend tests. Only do this for testing purposes." + required: false + type: boolean + default: false + skip_backend_tests: + description: "Skip backend tests. Only do this for testing purposes." + required: false + type: boolean + default: false + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: false + type: boolean + default: true schedule: # Run job at 00:00 UTC (4:00 PM PST / 5:00 PM PDT) - cron: "0 0 * * *" @@ -11,6 +36,15 @@ env: PYTHON_VERSION: "3.13" jobs: + validate-inputs: + runs-on: ubuntu-latest + steps: + - name: Validate inputs + if: inputs.push_to_registry && (inputs.skip_frontend_tests || inputs.skip_backend_tests) + run: | + echo "Cannot skip tests while push_to_registry is true." + exit 1 + create-nightly-tag: if: github.repository == 'langflow-ai/langflow' runs-on: ubuntu-latest @@ -125,7 +159,7 @@ jobs: fi frontend-tests: - if: github.repository == 'langflow-ai/langflow' + if: github.repository == 'langflow-ai/langflow' && !inputs.skip_frontend_tests name: Run Frontend Tests needs: create-nightly-tag uses: ./.github/workflows/typescript_test.yml @@ -139,7 +173,7 @@ jobs: TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }} backend-unit-tests: - if: github.repository == 'langflow-ai/langflow' + if: github.repository == 'langflow-ai/langflow' && !inputs.skip_backend_tests name: Run Backend Unit Tests needs: create-nightly-tag uses: ./.github/workflows/python_test.yml @@ -161,15 +195,16 @@ jobs: # ref: ${{ needs.create-nightly-tag.outputs.tag }} release-nightly-build: - if: github.repository == 'langflow-ai/langflow' + if: github.repository == 'langflow-ai/langflow' && (needs.frontend-tests.result == 'success' || needs.frontend-tests.result == 'skipped') && (needs.backend-unit-tests.result == 'success' || needs.backend-unit-tests.result == 'skipped') name: Run Nightly Langflow Build - needs: [frontend-tests, backend-unit-tests, create-nightly-tag] + needs: [create-nightly-tag, frontend-tests, backend-unit-tests] uses: ./.github/workflows/release_nightly.yml with: build_docker_base: true build_docker_main: true nightly_tag_main: ${{ needs.create-nightly-tag.outputs.main_tag }} nightly_tag_base: ${{ needs.create-nightly-tag.outputs.base_tag }} + push_to_registry: ${{ inputs.push_to_registry != false }} secrets: inherit # slack-notification: diff --git a/.github/workflows/python_test.yml b/.github/workflows/python_test.yml index f9faf7625a46..d7f7e38b7822 100644 --- a/.github/workflows/python_test.yml +++ b/.github/workflows/python_test.yml @@ -40,7 +40,6 @@ env: jobs: build: - name: Unit Tests - Python ${{ matrix.python-version }} - Group ${{ matrix.group }} runs-on: ubuntu-latest strategy: @@ -105,7 +104,6 @@ jobs: htmlcov/ retention-days: 30 - integration-tests: name: Integration Tests - Python ${{ matrix.python-version }} runs-on: ubuntu-latest @@ -198,4 +196,3 @@ jobs: else echo "Server terminated successfully" fi - diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b896747a548d..846eceb51bda 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,8 @@ run-name: Langflow Release by @${{ github.actor }} on: workflow_dispatch: inputs: - release_branch: - description: "Branch to release from. This is the branch that contains the source code for the release." + release_tag: + description: "Tag to release from. This is the tag that contains the source code for the release." required: true type: string release_package_base: @@ -44,40 +44,76 @@ on: type: boolean default: true +jobs: + echo-inputs: + name: Echo Workflow Inputs + runs-on: ubuntu-latest + steps: + - name: Echo workflow inputs + run: | + echo "release_tag: ${{ inputs.release_tag }}" + echo "release_package_base: ${{ inputs.release_package_base }}" + echo "release_package_main: ${{ inputs.release_package_main }}" + echo "release_lfx: ${{ inputs.release_lfx }}" + echo "build_docker_base: ${{ inputs.build_docker_base }}" + echo "build_docker_main: ${{ inputs.build_docker_main }}" + echo "build_docker_ep: ${{ inputs.build_docker_ep }}" + echo "pre_release: ${{ inputs.pre_release }}" + echo "create_release: ${{ inputs.create_release }}" + echo "runs_on: ${{ inputs.runs_on }}" + validate-tag: + name: Validate Tag Input + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + with: + fetch-depth: 0 # Fetch all history - required for tags (?) + - name: Validate that input is a tag, not a branch + run: | + # Check if the input exists as a tag + if ! git tag -l | grep -q "^${{ inputs.release_tag }}$"; then + echo "Error: '${{ inputs.release_tag }}' is not a valid tag." + echo "Available tags:" + git tag -l | head -20 + exit 1 + fi + + # Check if the input also exists as a branch (warn if so, but don't fail) + if git branch -r | grep -q "origin/${{ inputs.release_tag }}$"; then + echo "Tag '${{ inputs.release_tag }}' also exists as a branch. Exiting out of caution." + exit 1 + fi + + echo "Validated: '${{ inputs.release_tag }}' is a valid tag." -jobs: ci: - if: ${{ github.event.inputs.release_package_base == 'true' || github.event.inputs.release_package_main == 'true' }} + if: inputs.release_package_base || inputs.release_package_main || inputs.release_lfx name: CI + needs: [validate-tag] uses: ./.github/workflows/ci.yml with: - branch: ${{ inputs.release_branch }} + branch: ${{ inputs.release_tag }} python-versions: "['3.10', '3.11', '3.12', '3.13']" frontend-tests-folder: "tests" release: true + runs-on: ${{ (inputs['runs_on'] && startsWith(format('{0}', inputs['runs_on']), '[') && fromJSON(inputs['runs_on'])) || inputs['runs_on'] || inputs['runs_on'] || 'ubuntu-latest' }} secrets: inherit - build-base: name: Build Langflow Base needs: [ci] - if: inputs.release_package_base == true + if: inputs.release_package_base runs-on: ubuntu-latest outputs: version: ${{ steps.check-version.outputs.version }} skipped: ${{ steps.check-version.outputs.skipped }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: ${{ inputs.release_branch }} - - name: Debug branch information - run: | - echo "Workflow triggered from: ${{ github.ref }}" - echo "Checking out branch: ${{ inputs.release_branch }}" - echo "Current branch after checkout: $(git branch --show-current)" - echo "Current commit: $(git rev-parse HEAD)" + ref: ${{ inputs.release_tag }} - name: Setup Environment uses: astral-sh/setup-uv@v6 with: @@ -138,22 +174,16 @@ jobs: build-main: name: Build Langflow Main - if: inputs.release_package_main == true + if: inputs.release_package_main needs: [build-base] runs-on: ubuntu-latest outputs: version: ${{ steps.check-version.outputs.version }} steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - ref: ${{ inputs.release_branch }} - - name: Debug branch information - run: | - echo "Workflow triggered from: ${{ github.ref }}" - echo "Checking out branch: ${{ inputs.release_branch }}" - echo "Current branch after checkout: $(git branch --show-current)" - echo "Current commit: $(git rev-parse HEAD)" + ref: ${{ inputs.release_tag }} - name: Setup Environment uses: astral-sh/setup-uv@v6 with: @@ -169,7 +199,7 @@ jobs: # If pre-release is true, we need to check if ["a", "b", "rc", "dev", "post"] is in the version string # if the version string is incorrect, we need to exit the workflow - name: Check if pre-release - if: inputs.pre_release == 'true' + if: inputs.pre_release run: | version=$(uv tree | grep 'langflow' | grep -v 'langflow-base' | awk '{print $2}' | sed 's/^v//') if [[ "${version}" =~ ^([0-9]+\.)?([0-9]+\.)?[0-9]+((a|b|rc|dev|post)([0-9]+))$ ]]; then @@ -185,7 +215,8 @@ jobs: last_released_version=$(curl -s "https://pypi.org/pypi/langflow/json" | jq -r '.releases | keys | .[]' | sort -V | tail -n 1) if [ "$version" = "$last_released_version" ]; then echo "Version $version is already released. Skipping release." - exit 1 + echo skipped=true >> $GITHUB_OUTPUT + exit 0 else echo version=$version >> $GITHUB_OUTPUT fi @@ -223,7 +254,7 @@ jobs: test-cross-platform: name: Test Cross-Platform Installation - if: inputs.release_package_base == true || inputs.release_package_main == true + if: inputs.release_package_base || inputs.release_package_main needs: [build-base, build-main] uses: ./.github/workflows/cross-platform-test.yml with: @@ -232,7 +263,7 @@ jobs: publish-base: name: Publish Langflow Base to PyPI - if: inputs.release_package_base == true + if: inputs.release_package_base needs: [build-base, test-cross-platform] runs-on: ubuntu-latest steps: @@ -255,7 +286,7 @@ jobs: publish-main: name: Publish Langflow Main to PyPI - if: inputs.release_package_main == true + if: inputs.release_package_main needs: [build-main, test-cross-platform, publish-base] runs-on: ubuntu-latest steps: @@ -277,58 +308,90 @@ jobs: call_docker_build_base: name: Call Docker Build Workflow for Langflow Base - if: inputs.build_docker_base == true - needs: [build-base, build-main, publish-base, publish-main] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_base + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - base_version: ${{ needs.build-base.outputs.version }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: base pre_release: ${{ inputs.pre_release }} secrets: inherit call_docker_build_main_ep: name: Call Docker Build Workflow for Langflow with Entrypoint - if: inputs.build_docker_ep == true - needs: [build-main, publish-main, call_docker_build_base] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_ep + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: main-ep pre_release: False secrets: inherit call_docker_build_main: name: Call Docker Build Workflow for Langflow - if: inputs.build_docker_main == true - needs: [build-main, publish-main, call_docker_build_main_ep] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_main + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: main pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} + secrets: inherit + + call_docker_build_main_backend: + name: Call Docker Build Workflow for Langflow Backend + if: ${{ inputs.build_docker_main }} + needs: [ci] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} + release_type: main-backend + pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} + secrets: inherit + + call_docker_build_main_frontend: + name: Call Docker Build Workflow for Langflow Frontend + if: ${{ inputs.build_docker_main }} + needs: [ci] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} + release_type: main-frontend + pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} + secrets: inherit + + call_docker_build_main_ep: + name: Call Docker Build Workflow for Langflow with Entrypoint + if: ${{ inputs.build_docker_main }} + needs: [ci] + uses: ./.github/workflows/docker-build-v2.yml + with: + ref: ${{ inputs.release_tag }} + release_type: main-ep + pre_release: ${{ inputs.pre_release }} + push_to_registry: ${{ !inputs.dry_run }} secrets: inherit call_docker_build_main_all: name: Call Docker Build Workflow for langflow-all - if: inputs.build_docker_main == true - needs: [build-main, publish-main] - uses: ./.github/workflows/docker-build.yml + if: inputs.build_docker_main + needs: [validate-tag] + uses: ./.github/workflows/docker-build-v2.yml with: - ref: ${{ inputs.release_branch }} - main_version: ${{ needs.build-main.outputs.version }} + ref: ${{ inputs.release_tag }} release_type: main-all pre_release: ${{ inputs.pre_release }} secrets: inherit - create_release: name: Create Release runs-on: ubuntu-latest needs: [build-main, publish-main] + if: always() && inputs.create_release && (needs.publish-main.result == 'success' ) steps: - uses: actions/download-artifact@v4 with: @@ -343,4 +406,5 @@ jobs: generateReleaseNotes: true prerelease: ${{ inputs.pre_release }} tag: ${{ needs.build-main.outputs.version }} - commit: ${{ inputs.release_branch }} + allowUpdates: true + updateOnlyUnreleased: false diff --git a/.github/workflows/release_nightly.yml b/.github/workflows/release_nightly.yml index 80e6df568423..48aa96628ae2 100644 --- a/.github/workflows/release_nightly.yml +++ b/.github/workflows/release_nightly.yml @@ -52,6 +52,11 @@ on: description: "Tag for the nightly base build" required: true type: string + push_to_registry: + description: "Whether to push images to registries. Set to false for testing builds without publishing." + required: true + type: boolean + default: false env: POETRY_VERSION: "1.8.3" @@ -222,7 +227,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + ref: ${{ inputs.nightly_tag_main }} + persist-credentials: true - name: Download base artifact uses: actions/download-artifact@v4 with: @@ -247,7 +255,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 + with: + ref: ${{ inputs.nightly_tag_main }} + persist-credentials: true - name: Download main artifact uses: actions/download-artifact@v4 with: @@ -267,23 +278,24 @@ jobs: call_docker_build_base: name: Call Docker Build Workflow for Langflow Base - if: always() && ${{ inputs.build_docker_base == 'true' }} + if: always() && inputs.build_docker_base needs: [build-nightly-base, build-nightly-main] - uses: ./.github/workflows/docker-build.yml + uses: ./.github/workflows/docker-nightly-build.yml with: + ref: ${{ inputs.nightly_tag_main }} release_type: nightly-base - base_version: ${{ inputs.nightly_tag_base }} - main_version: ${{ inputs.nightly_tag_main }} + push_to_registry: ${{ inputs.push_to_registry }} secrets: inherit call_docker_build_main: name: Call Docker Build Workflow for Langflow - if: always() && ${{ inputs.build_docker_main == 'true' }} + if: always() && inputs.build_docker_main needs: [build-nightly-main, call_docker_build_base] - uses: ./.github/workflows/docker-build.yml + uses: ./.github/workflows/docker-nightly-build.yml with: + ref: ${{ inputs.nightly_tag_main }} release_type: nightly-main - main_version: ${{ inputs.nightly_tag_main }} + push_to_registry: ${{ inputs.push_to_registry }} secrets: inherit # TODO: Uncomment this when our runner can fit the builds that contain pytorch (and other large dependencies) @@ -291,18 +303,20 @@ jobs: # name: Call Docker Build Workflow for langflow-all # if: always() && ${{ inputs.build_docker_main == 'true' }} # needs: [build-nightly-main] - # uses: ./.github/workflows/docker-build.yml + # uses: ./.github/workflows/docker-nightly-build.yml # with: + # ref: ${{ inputs.nightly_tag_main }} # release_type: nightly-main-all # main_version: ${{ inputs.nightly_tag_main }} # secrets: inherit call_docker_build_main_ep: name: Call Docker Build Workflow for Langflow with Entrypoint - if: always() && ${{ inputs.build_docker_ep == 'true' }} + if: always() && inputs.build_docker_ep needs: [build-nightly-main, call_docker_build_main] - uses: ./.github/workflows/docker-build.yml + uses: ./.github/workflows/docker-build-v2.yml with: + ref: ${{ inputs.nightly_tag_main }} release_type: main-ep - main_version: ${{ inputs.nightly_tag_main }} + push_to_registry: ${{ inputs.push_to_registry }} secrets: inherit diff --git a/docker/build_and_push.Dockerfile b/docker/build_and_push.Dockerfile index da964ca924ae..a95675437156 100644 --- a/docker/build_and_push.Dockerfile +++ b/docker/build_and_push.Dockerfile @@ -51,7 +51,7 @@ COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend RUN --mount=type=cache,target=/root/.npm \ npm ci \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/langflow/frontend \ && rm -rf /tmp/src/frontend diff --git a/docker/build_and_push_backend.Dockerfile b/docker/build_and_push_backend.Dockerfile index 66cf65212548..894718c43871 100644 --- a/docker/build_and_push_backend.Dockerfile +++ b/docker/build_and_push_backend.Dockerfile @@ -2,7 +2,7 @@ # Keep this syntax directive! It's used to enable Docker BuildKit ARG LANGFLOW_IMAGE -FROM $LANGFLOW_IMAGE +FROM ${LANGFLOW_IMAGE} RUN rm -rf /app/.venv/langflow/frontend diff --git a/docker/build_and_push_base.Dockerfile b/docker/build_and_push_base.Dockerfile index 8c1dd316afb5..72c1018a6037 100644 --- a/docker/build_and_push_base.Dockerfile +++ b/docker/build_and_push_base.Dockerfile @@ -52,8 +52,10 @@ COPY ./src /app/src COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend +# Increase memory and disable concurrent builds to avoid esbuild crashes on emulated architectures +# Force esbuild to use JS implementation on emulated architectures to avoid native binary crashes RUN npm install \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/base/langflow/frontend \ && rm -rf /tmp/src/frontend diff --git a/docker/build_and_push_ep.Dockerfile b/docker/build_and_push_ep.Dockerfile index 073b30004b53..d2337472b1d5 100644 --- a/docker/build_and_push_ep.Dockerfile +++ b/docker/build_and_push_ep.Dockerfile @@ -51,7 +51,7 @@ COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend RUN --mount=type=cache,target=/root/.npm \ npm ci \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/langflow/frontend \ && rm -rf /tmp/src/frontend diff --git a/docker/build_and_push_with_extras.Dockerfile b/docker/build_and_push_with_extras.Dockerfile index 5a3b59660bdd..cbb5208e2cad 100644 --- a/docker/build_and_push_with_extras.Dockerfile +++ b/docker/build_and_push_with_extras.Dockerfile @@ -51,7 +51,7 @@ COPY src/frontend /tmp/src/frontend WORKDIR /tmp/src/frontend RUN --mount=type=cache,target=/root/.npm \ npm ci \ - && npm run build \ + && ESBUILD_BINARY_PATH="" NODE_OPTIONS="--max-old-space-size=12288" JOBS=1 npm run build \ && cp -r build /app/src/backend/langflow/frontend \ && rm -rf /tmp/src/frontend