Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions .github/workflows/build-and-push-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,6 @@ on:
- both
- amd64
- arm64
with_r_support:
description: 'Enable R support for workflow-execution-coordinator'
required: false
default: false
type: boolean
schedule:
# Run nightly at 2:00 AM UTC
- cron: '0 2 * * *'
Expand All @@ -75,7 +70,6 @@ jobs:
docker_registry: ${{ steps.set-params.outputs.docker_registry }}
services: ${{ steps.set-params.outputs.services }}
platforms: ${{ steps.set-params.outputs.platforms }}
with_r_support: ${{ steps.set-params.outputs.with_r_support }}
steps:
- name: Set build parameters
id: set-params
Expand All @@ -88,15 +82,13 @@ jobs:
echo "docker_registry=apache" >> $GITHUB_OUTPUT
echo "services=*" >> $GITHUB_OUTPUT
echo "platforms=both" >> $GITHUB_OUTPUT
echo "with_r_support=false" >> $GITHUB_OUTPUT
else
echo "Manual workflow_dispatch - using user inputs"
echo "branch=${{ github.event.inputs.branch || 'main' }}" >> $GITHUB_OUTPUT
echo "image_tag=${{ github.event.inputs.image_tag }}" >> $GITHUB_OUTPUT
echo "docker_registry=${{ github.event.inputs.docker_registry || 'apache' }}" >> $GITHUB_OUTPUT
echo "services=${{ github.event.inputs.services || '*' }}" >> $GITHUB_OUTPUT
echo "platforms=${{ github.event.inputs.platforms || 'both' }}" >> $GITHUB_OUTPUT
echo "with_r_support=${{ github.event.inputs.with_r_support || 'false' }}" >> $GITHUB_OUTPUT
fi

# Step 1: Generate JOOQ code once and share it
Expand Down Expand Up @@ -358,8 +350,6 @@ jobs:
tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64
cache-from: type=gha,scope=${{ matrix.image_name }}-amd64
cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-amd64
build-args: |
${{ matrix.service == 'computing-unit-master' && needs.set-parameters.outputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }}
labels: |
org.opencontainers.image.title=${{ matrix.image_name }}
org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (AMD64)
Expand Down Expand Up @@ -437,8 +427,6 @@ jobs:
tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64
cache-from: type=gha,scope=${{ matrix.image_name }}-arm64
cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-arm64
build-args: |
${{ matrix.service == 'computing-unit-master' && needs.set-parameters.outputs.with_r_support == 'true' && 'WITH_R_SUPPORT=true' || '' }}
labels: |
org.opencontainers.image.title=${{ matrix.image_name }}
org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (ARM64)
Expand Down Expand Up @@ -499,7 +487,6 @@ jobs:
echo "- **Tag:** \`${{ needs.set-parameters.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY
echo "- **Services:** ${{ needs.set-parameters.outputs.services }}" >> $GITHUB_STEP_SUMMARY
echo "- **Platforms:** ${{ needs.set-parameters.outputs.platforms }}" >> $GITHUB_STEP_SUMMARY
echo "- **R Support:** ${{ needs.set-parameters.outputs.with_r_support }}" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "## Build Method" >> $GITHUB_STEP_SUMMARY
echo "**Parallel platform builds** (faster)" >> $GITHUB_STEP_SUMMARY
Expand Down
58 changes: 5 additions & 53 deletions bin/computing-unit-master.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,72 +43,24 @@ RUN unzip amber/target/universal/amber-*.zip -d amber/target/

FROM eclipse-temurin:11-jdk-jammy AS runtime

# Build argument to enable/disable R support (default: true for backward compatibility)
ARG WITH_R_SUPPORT=false

WORKDIR /texera/amber

COPY --from=build /texera/amber/r-requirements.txt /tmp/r-requirements.txt
COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt
COPY --from=build /texera/amber/operator-requirements.txt /tmp/operator-requirements.txt

# Install Python runtime dependencies (always) and R runtime dependencies (conditional)
# Install Python runtime dependencies
RUN apt-get update && apt-get install -y \
python3-pip \
python3-dev \
libpq-dev \
curl \
unzip \
$(if [ "$WITH_R_SUPPORT" = "true" ]; then echo "\
gfortran \
build-essential \
libreadline-dev \
libncurses-dev \
libssl-dev \
libxml2-dev \
xorg-dev \
libbz2-dev \
liblzma-dev \
libpcre++-dev \
libpango1.0-dev \
libcurl4-openssl-dev"; fi) \
&& apt-get clean

# Install R and needed libraries (conditional)
ENV R_VERSION=4.3.3
RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
curl -O https://cran.r-project.org/src/base/R-4/R-${R_VERSION}.tar.gz && \
tar -xf R-${R_VERSION}.tar.gz && \
cd R-${R_VERSION} && \
./configure --prefix=/usr/local \
--enable-R-shlib \
--with-blas \
--with-lapack && \
make -j 4 && \
make install && \
cd .. && \
rm -rf R-${R_VERSION}* && R --version && pip3 install --upgrade pip setuptools wheel && \
pip3 install -r /tmp/requirements.txt && \
pip3 install -r /tmp/operator-requirements.txt && \
pip3 install -r /tmp/r-requirements.txt; \
else \
pip3 install --upgrade pip setuptools wheel && \
pip3 install -r /tmp/requirements.txt && \
pip3 install -r /tmp/operator-requirements.txt; \
fi
# Install R packages, pinning arrow to 14.0.2.1 explicitly (conditional)
RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
install.packages(c('coro', 'dplyr'), \
Ncpus = parallel::detectCores())" && \
Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
if (!requireNamespace('remotes', quietly=TRUE)) \
install.packages('remotes'); \
remotes::install_version('arrow', version='14.0.2.1', \
repos='https://cran.r-project.org', upgrade='never'); \
cat('R arrow version: ', as.character(packageVersion('arrow')), '\n')"; \
fi
ENV LD_LIBRARY_PATH=/usr/local/lib/R/lib:$LD_LIBRARY_PATH
# Install Python packages
RUN pip3 install --upgrade pip setuptools wheel && \
pip3 install -r /tmp/requirements.txt && \
pip3 install -r /tmp/operator-requirements.txt

# Copy the built texera binary from the build phase
COPY --from=build /texera/.git /texera/amber/.git
Expand Down
66 changes: 7 additions & 59 deletions bin/computing-unit-worker.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -43,76 +43,24 @@ RUN unzip amber/target/universal/amber-*.zip -d amber/target/

FROM eclipse-temurin:11-jre-jammy AS runtime

# Build argument to enable/disable R support (default: false for backward compatibility)
ARG WITH_R_SUPPORT=false

WORKDIR /texera/amber

COPY --from=build /texera/amber/r-requirements.txt /tmp/r-requirements.txt
COPY --from=build /texera/amber/requirements.txt /tmp/requirements.txt
COPY --from=build /texera/amber/operator-requirements.txt /tmp/operator-requirements.txt

# Install Python runtime dependencies (always) and R runtime dependencies (conditional)
# Install Python runtime dependencies
RUN apt-get update && apt-get install -y \
python3-pip \
python3-dev \
libpq-dev \
$(if [ "$WITH_R_SUPPORT" = "true" ]; then echo "\
gfortran \
curl \
build-essential \
libreadline-dev \
libncurses-dev \
libssl-dev \
libxml2-dev \
xorg-dev \
libbz2-dev \
liblzma-dev \
libpcre++-dev \
libpango1.0-dev \
libcurl4-openssl-dev"; fi) \
&& apt-get clean

# Install R and needed libraries (conditional)
ENV R_VERSION=4.3.3
RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
curl -O https://cran.r-project.org/src/base/R-4/R-${R_VERSION}.tar.gz && \
tar -xf R-${R_VERSION}.tar.gz && \
cd R-${R_VERSION} && \
./configure --prefix=/usr/local \
--enable-R-shlib \
--with-blas \
--with-lapack && \
make -j 4 && \
make install && \
cd .. && \
rm -rf R-${R_VERSION}* && R --version && pip3 install --upgrade pip setuptools wheel && \
pip3 install python-lsp-server python-lsp-server[websockets] && \
pip3 install -r /tmp/requirements.txt && \
pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r /tmp/operator-requirements.txt || \
pip3 install --no-cache-dir wordcloud==1.9.2 && \
pip3 install -r /tmp/r-requirements.txt; \
else \
pip3 install --upgrade pip setuptools wheel && \
pip3 install python-lsp-server python-lsp-server[websockets] && \
pip3 install -r /tmp/requirements.txt && \
pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r /tmp/operator-requirements.txt || \
pip3 install --no-cache-dir wordcloud==1.9.2; \
fi

# Install R packages, pinning arrow to 14.0.2.1 explicitly (conditional)
RUN if [ "$WITH_R_SUPPORT" = "true" ]; then \
Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
install.packages(c('coro', 'dplyr'), \
Ncpus = parallel::detectCores())" && \
Rscript -e "options(repos = c(CRAN = 'https://cran.r-project.org')); \
if (!requireNamespace('remotes', quietly=TRUE)) \
install.packages('remotes'); \
remotes::install_version('arrow', version='14.0.2.1', \
repos='https://cran.r-project.org', upgrade='never'); \
cat('R arrow version: ', as.character(packageVersion('arrow')), '\n')"; \
fi
ENV LD_LIBRARY_PATH=/usr/local/lib/R/lib:$LD_LIBRARY_PATH
# Install Python packages
RUN pip3 install --upgrade pip setuptools wheel && \
pip3 install python-lsp-server python-lsp-server[websockets] && \
pip3 install -r /tmp/requirements.txt && \
pip3 install --no-cache-dir --find-links https://pypi.org/simple/ -r /tmp/operator-requirements.txt || \
pip3 install --no-cache-dir wordcloud==1.9.2

# Copy the built texera binary from the build phase
COPY --from=build /texera/amber/target/amber-* /texera/amber/
Expand Down
Loading