From ab3e09ac3f9a6d74ed8e7b5ec56bbb67a73caa9f Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Tue, 25 Nov 2025 01:30:59 -0800 Subject: [PATCH 1/5] finish fix --- bin/single-node/.env | 2 -- bin/single-node/docker-compose.yml | 36 ++++++++++++++++++- bin/single-node/nginx.conf | 12 +++++++ .../sql/iceberg_postgres_catalog.sql | 1 + bin/single-node/sql/texera_ddl.sql | 1 + bin/single-node/sql/texera_lakefs.sql | 1 + 6 files changed, 50 insertions(+), 3 deletions(-) create mode 120000 bin/single-node/sql/iceberg_postgres_catalog.sql create mode 120000 bin/single-node/sql/texera_ddl.sql create mode 120000 bin/single-node/sql/texera_lakefs.sql diff --git a/bin/single-node/.env b/bin/single-node/.env index 4b5d1d58364..d7845ec297a 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -44,8 +44,6 @@ STORAGE_JDBC_USERNAME=texera STORAGE_JDBC_PASSWORD=password FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT=http://texera-file-service:9092/api/dataset/presign-download FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT=http://texera-file-service:9092/api/dataset/did/upload -USER_SYS_ENABLED=true -STORAGE_ICEBERG_CATALOG_TYPE=postgres STORAGE_ICEBERG_CATALOG_POSTGRES_URI_WITHOUT_SCHEME=texera-postgres:5432/texera_iceberg_catalog STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME=texera STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index ce95d64c708..2be279afe97 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -46,7 +46,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data # mount the sql files for initializing the postgres - - ../../sql:/docker-entrypoint-initdb.d + - ./sql:/docker-entrypoint-initdb.d # lakeFS is the underlying storage of Texera's dataset service lakefs: @@ -94,6 +94,38 @@ services: timeout: 3s retries: 10 + # ConfigService provides endpoints for configuration management + config-service: + image: apache/texera-config-service:latest + container_name: texera-config-service + restart: always + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:9094/api/healthcheck"] + interval: 5s + timeout: 3s + retries: 10 + + # AccessControlService handles user permissions and access control + access-control-service: + image: apache/texera-access-control-service:latest + container_name: texera-access-control-service + restart: always + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:9096/api/healthcheck"] + interval: 5s + timeout: 3s + retries: 10 + # WorkflowCompilingService provides endpoints for sanity check and schema propagation while workflows are being edited workflow-compiling-service: image: apache/texera-workflow-compiling-service:latest @@ -191,6 +223,8 @@ services: - file-service - texera-web-application - computing-unit-master + - config-service + - access-control-service volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro ports: diff --git a/bin/single-node/nginx.conf b/bin/single-node/nginx.conf index f703fb027ff..d4d98082641 100644 --- a/bin/single-node/nginx.conf +++ b/bin/single-node/nginx.conf @@ -42,6 +42,18 @@ http { proxy_set_header X-Real-IP $remote_addr; } + location /api/config { + proxy_pass http://texera-config-service:9094; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + location /api/auth { + proxy_pass http://texera-access-control-service:9096; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location /api/ { proxy_pass http://texera-web-application:8080; proxy_set_header Host $host; diff --git a/bin/single-node/sql/iceberg_postgres_catalog.sql b/bin/single-node/sql/iceberg_postgres_catalog.sql new file mode 120000 index 00000000000..7def191e8ee --- /dev/null +++ b/bin/single-node/sql/iceberg_postgres_catalog.sql @@ -0,0 +1 @@ +../../../sql/iceberg_postgres_catalog.sql \ No newline at end of file diff --git a/bin/single-node/sql/texera_ddl.sql b/bin/single-node/sql/texera_ddl.sql new file mode 120000 index 00000000000..5653a9a01e1 --- /dev/null +++ b/bin/single-node/sql/texera_ddl.sql @@ -0,0 +1 @@ +../../../sql/texera_ddl.sql \ No newline at end of file diff --git a/bin/single-node/sql/texera_lakefs.sql b/bin/single-node/sql/texera_lakefs.sql new file mode 120000 index 00000000000..6d58e92106a --- /dev/null +++ b/bin/single-node/sql/texera_lakefs.sql @@ -0,0 +1 @@ +../../../sql/texera_lakefs.sql \ No newline at end of file From 79ed56c3e0e968594c5f423fab51fb19aac2d257 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Tue, 25 Nov 2025 09:20:44 -0800 Subject: [PATCH 2/5] add workflow-computing-unit-managing-service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- bin/single-node/docker-compose.yml | 87 ++++++++++++++++++------------ bin/single-node/nginx.conf | 4 +- 2 files changed, 54 insertions(+), 37 deletions(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 2be279afe97..887c44a3d9e 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -126,6 +126,22 @@ services: timeout: 3s retries: 10 + # WorkflowComputingUnitManagingService provides endpoints for managing computing units + workflow-computing-unit-managing-service: + image: apache/texera-workflow-computing-unit-managing-service:latest + container_name: texera-workflow-computing-unit-managing-service + restart: always + depends_on: + postgres: + condition: service_healthy + env_file: + - .env + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8888/api/healthcheck"] + interval: 5s + timeout: 3s + retries: 10 + # WorkflowCompilingService provides endpoints for sanity check and schema propagation while workflows are being edited workflow-compiling-service: image: apache/texera-workflow-compiling-service:latest @@ -145,7 +161,7 @@ services: # ComputingUnitMaster provides endpoints for executing workflows and interactions during executions. computing-unit-master: # to enable R operators, change the image tag to single-node-release-1-0-0-R - image: apache/texera-workflow-execution-coordinator:latest + image: apache/texera-workflow-execution-coordinator:latest-arm64 container_name: texera-computing-unit-master restart: always depends_on: @@ -178,41 +194,41 @@ services: timeout: 3s retries: 10 - texera-example-data-loader: - image: apache/texera-example-data-loader:latest - container_name: texera-example-data-loader - depends_on: - texera-web-application: - condition: service_healthy - environment: - TEXERA_WEB_APPLICATION_URL: http://texera-web-application:8080/api - TEXERA_FILE_SERVICE_URL: http://texera-file-service:9092/api - restart: "no" +# texera-example-data-loader: +# image: apache/texera-example-data-loader:latest +# container_name: texera-example-data-loader +# depends_on: +# texera-web-application: +# condition: service_healthy +# environment: +# TEXERA_WEB_APPLICATION_URL: http://texera-web-application:8080/api +# TEXERA_FILE_SERVICE_URL: http://texera-file-service:9092/api +# restart: "no" - texera-access-message: - image: alpine:latest - container_name: texera-access-message - depends_on: - texera-example-data-loader: - condition: service_completed_successfully - env_file: - - .env - command: > - /bin/sh -c " - echo '==============================================='; - echo 'Texera is ready!'; - echo '==============================================='; - echo ''; - echo 'To access Texera, open your browser and navigate to:'; - echo ' ${TEXERA_HOST}:8080'; - echo ''; - echo 'The following account has been created for you:'; - echo ' Username: texera'; - echo ' Password: texera'; - echo ''; - echo '===============================================' - " - restart: "no" +# texera-access-message: +# image: alpine:latest +# container_name: texera-access-message +# depends_on: +# texera-example-data-loader: +# condition: service_completed_successfully +# env_file: +# - .env +# command: > +# /bin/sh -c " +# echo '==============================================='; +# echo 'Texera is ready!'; +# echo '==============================================='; +# echo ''; +# echo 'To access Texera, open your browser and navigate to:'; +# echo ' ${TEXERA_HOST}:8080'; +# echo ''; +# echo 'The following account has been created for you:'; +# echo ' Username: texera'; +# echo ' Password: texera'; +# echo ''; +# echo '===============================================' +# " +# restart: "no" # Part 3: reverse proxy service for Texera's micro services nginx: @@ -225,6 +241,7 @@ services: - computing-unit-master - config-service - access-control-service + - workflow-computing-unit-managing-service volumes: - ./nginx.conf:/etc/nginx/nginx.conf:ro ports: diff --git a/bin/single-node/nginx.conf b/bin/single-node/nginx.conf index d4d98082641..4bdf8baf56b 100644 --- a/bin/single-node/nginx.conf +++ b/bin/single-node/nginx.conf @@ -48,8 +48,8 @@ http { proxy_set_header X-Real-IP $remote_addr; } - location /api/auth { - proxy_pass http://texera-access-control-service:9096; + location /api/computing-unit { + proxy_pass http://texera-workflow-computing-unit-managing-service:8888; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; } From 7bf80e8a9fc2f4d3813cffbead85d867ceade4c6 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Tue, 25 Nov 2025 09:26:37 -0800 Subject: [PATCH 3/5] revert changes --- bin/single-node/docker-compose.yml | 70 +++++++++++++++--------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 887c44a3d9e..ef6f8354552 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -161,7 +161,7 @@ services: # ComputingUnitMaster provides endpoints for executing workflows and interactions during executions. computing-unit-master: # to enable R operators, change the image tag to single-node-release-1-0-0-R - image: apache/texera-workflow-execution-coordinator:latest-arm64 + image: apache/texera-workflow-execution-coordinator:latest container_name: texera-computing-unit-master restart: always depends_on: @@ -194,41 +194,41 @@ services: timeout: 3s retries: 10 -# texera-example-data-loader: -# image: apache/texera-example-data-loader:latest -# container_name: texera-example-data-loader -# depends_on: -# texera-web-application: -# condition: service_healthy -# environment: -# TEXERA_WEB_APPLICATION_URL: http://texera-web-application:8080/api -# TEXERA_FILE_SERVICE_URL: http://texera-file-service:9092/api -# restart: "no" + texera-example-data-loader: + image: apache/texera-example-data-loader:latest + container_name: texera-example-data-loader + depends_on: + texera-web-application: + condition: service_healthy + environment: + TEXERA_WEB_APPLICATION_URL: http://texera-web-application:8080/api + TEXERA_FILE_SERVICE_URL: http://texera-file-service:9092/api + restart: "no" -# texera-access-message: -# image: alpine:latest -# container_name: texera-access-message -# depends_on: -# texera-example-data-loader: -# condition: service_completed_successfully -# env_file: -# - .env -# command: > -# /bin/sh -c " -# echo '==============================================='; -# echo 'Texera is ready!'; -# echo '==============================================='; -# echo ''; -# echo 'To access Texera, open your browser and navigate to:'; -# echo ' ${TEXERA_HOST}:8080'; -# echo ''; -# echo 'The following account has been created for you:'; -# echo ' Username: texera'; -# echo ' Password: texera'; -# echo ''; -# echo '===============================================' -# " -# restart: "no" + texera-access-message: + image: alpine:latest + container_name: texera-access-message + depends_on: + texera-example-data-loader: + condition: service_completed_successfully + env_file: + - .env + command: > + /bin/sh -c " + echo '==============================================='; + echo 'Texera is ready!'; + echo '==============================================='; + echo ''; + echo 'To access Texera, open your browser and navigate to:'; + echo ' ${TEXERA_HOST}:8080'; + echo ''; + echo 'The following account has been created for you:'; + echo ' Username: texera'; + echo ' Password: texera'; + echo ''; + echo '===============================================' + " + restart: "no" # Part 3: reverse proxy service for Texera's micro services nginx: From 58857e5adb00ac844bcc8bcc26f8b9d869bf40ae Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Tue, 25 Nov 2025 10:46:56 -0800 Subject: [PATCH 4/5] rollback the sql mounting --- bin/single-node/docker-compose.yml | 2 +- bin/single-node/sql/iceberg_postgres_catalog.sql | 1 - bin/single-node/sql/texera_ddl.sql | 1 - bin/single-node/sql/texera_lakefs.sql | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) delete mode 120000 bin/single-node/sql/iceberg_postgres_catalog.sql delete mode 120000 bin/single-node/sql/texera_ddl.sql delete mode 120000 bin/single-node/sql/texera_lakefs.sql diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index ef6f8354552..5c4766e82a4 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -46,7 +46,7 @@ services: volumes: - postgres_data:/var/lib/postgresql/data # mount the sql files for initializing the postgres - - ./sql:/docker-entrypoint-initdb.d + - ../../sql:/docker-entrypoint-initdb.d # lakeFS is the underlying storage of Texera's dataset service lakefs: diff --git a/bin/single-node/sql/iceberg_postgres_catalog.sql b/bin/single-node/sql/iceberg_postgres_catalog.sql deleted file mode 120000 index 7def191e8ee..00000000000 --- a/bin/single-node/sql/iceberg_postgres_catalog.sql +++ /dev/null @@ -1 +0,0 @@ -../../../sql/iceberg_postgres_catalog.sql \ No newline at end of file diff --git a/bin/single-node/sql/texera_ddl.sql b/bin/single-node/sql/texera_ddl.sql deleted file mode 120000 index 5653a9a01e1..00000000000 --- a/bin/single-node/sql/texera_ddl.sql +++ /dev/null @@ -1 +0,0 @@ -../../../sql/texera_ddl.sql \ No newline at end of file diff --git a/bin/single-node/sql/texera_lakefs.sql b/bin/single-node/sql/texera_lakefs.sql deleted file mode 120000 index 6d58e92106a..00000000000 --- a/bin/single-node/sql/texera_lakefs.sql +++ /dev/null @@ -1 +0,0 @@ -../../../sql/texera_lakefs.sql \ No newline at end of file From f1073b6b455af3f507aa0877ef7e57178d3682c0 Mon Sep 17 00:00:00 2001 From: Jiadong Bai Date: Tue, 25 Nov 2025 11:14:16 -0800 Subject: [PATCH 5/5] remove the example data loader --- bin/single-node/.env | 9 +------- bin/single-node/docker-compose.yml | 36 ------------------------------ 2 files changed, 1 insertion(+), 44 deletions(-) diff --git a/bin/single-node/.env b/bin/single-node/.env index d7845ec297a..f7a0d349bfc 100644 --- a/bin/single-node/.env +++ b/bin/single-node/.env @@ -46,11 +46,4 @@ FILE_SERVICE_GET_PRESIGNED_URL_ENDPOINT=http://texera-file-service:9092/api/data FILE_SERVICE_UPLOAD_ONE_FILE_TO_DATASET_ENDPOINT=http://texera-file-service:9092/api/dataset/did/upload STORAGE_ICEBERG_CATALOG_POSTGRES_URI_WITHOUT_SCHEME=texera-postgres:5432/texera_iceberg_catalog STORAGE_ICEBERG_CATALOG_POSTGRES_USERNAME=texera -STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password - -# variables for example user, datasets and workflows -TEXERA_EXAMPLE_USERNAME=texera -TEXERA_EXAMPLE_PASSWORD=texera -TEXERA_EXAMPLE_IS_PUBLIC=true -TEXERA_EXAMPLE_DATASET_DIR=/example_data/data -TEXERA_EXAMPLE_WORKFLOW_DIR=/example_data/workflows \ No newline at end of file +STORAGE_ICEBERG_CATALOG_POSTGRES_PASSWORD=password \ No newline at end of file diff --git a/bin/single-node/docker-compose.yml b/bin/single-node/docker-compose.yml index 5c4766e82a4..a113b008f63 100644 --- a/bin/single-node/docker-compose.yml +++ b/bin/single-node/docker-compose.yml @@ -194,42 +194,6 @@ services: timeout: 3s retries: 10 - texera-example-data-loader: - image: apache/texera-example-data-loader:latest - container_name: texera-example-data-loader - depends_on: - texera-web-application: - condition: service_healthy - environment: - TEXERA_WEB_APPLICATION_URL: http://texera-web-application:8080/api - TEXERA_FILE_SERVICE_URL: http://texera-file-service:9092/api - restart: "no" - - texera-access-message: - image: alpine:latest - container_name: texera-access-message - depends_on: - texera-example-data-loader: - condition: service_completed_successfully - env_file: - - .env - command: > - /bin/sh -c " - echo '==============================================='; - echo 'Texera is ready!'; - echo '==============================================='; - echo ''; - echo 'To access Texera, open your browser and navigate to:'; - echo ' ${TEXERA_HOST}:8080'; - echo ''; - echo 'The following account has been created for you:'; - echo ' Username: texera'; - echo ' Password: texera'; - echo ''; - echo '===============================================' - " - restart: "no" - # Part 3: reverse proxy service for Texera's micro services nginx: image: nginx:alpine