From 9b56f6fc5322ee135972fd43bfafeab7c928526b Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Mon, 15 Apr 2024 09:05:14 -0700 Subject: [PATCH 1/2] fix: backtracking on adding docker/.env-local for docker-compose As reported here https://github.com/apache/superset/issues/28026, in https://github.com/apache/superset/pull/27953 I added support for a supplemental `docker/.env-local` override file, not thinking it was going to be required upon starting docker-compose. I thought I had tested that while switching branches but somehow didn't test that docker-compose would simply boot up in the absence of that file. Since this is confusing to most, and we don't really have a script I can latch onto to `touch docker/.env-local` for now, i decided to backtrack and simply remove the convenience "feature". People should find an alternate way to alter their environment somehow before starting the service. I added a bit of documentation to point to an existing mechanism around providing a `docker/pythonpath_dev/superset_config_docker.py` file, which existed in the logic but wasn't clearly documented. This solves for my use case where I was working a PR adding features around Slack, where I wanted to provide an API key configuration without risks of commiting/pushing it back to the repo. --- .gitignore | 1 - docker-compose-image-tag.yml | 15 ------------- docker-compose-non-dev.yml | 15 ------------- docker-compose.yml | 21 ------------------- ...stalling-superset-using-docker-compose.mdx | 16 ++++++++++---- 5 files changed, 12 insertions(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index bbfc640c9acd..02657eb0fa98 100644 --- a/.gitignore +++ b/.gitignore @@ -110,7 +110,6 @@ release.json messages.mo docker/requirements-local.txt -docker/.env-local cache/ docker/*local* diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml index 6c017ef75e75..c3bef372f1fb 100644 --- a/docker-compose-image-tag.yml +++ b/docker-compose-image-tag.yml @@ -34,10 +34,7 @@ services: db: env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local image: postgres:15 container_name: superset_db restart: unless-stopped @@ -47,10 +44,7 @@ services: superset: env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local image: *superset-image container_name: superset_app command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] @@ -66,10 +60,7 @@ services: container_name: superset_init command: ["/app/docker/docker-init.sh"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local depends_on: *superset-depends-on user: "root" volumes: *superset-volumes @@ -81,10 +72,7 @@ services: container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" @@ -101,10 +89,7 @@ services: container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index 78d8cb9c3513..1462a8654efe 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -39,10 +39,7 @@ services: db: env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local image: postgres:15 container_name: superset_db restart: unless-stopped @@ -52,10 +49,7 @@ services: superset: env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local build: <<: *common-build container_name: superset_app @@ -73,10 +67,7 @@ services: <<: *common-build command: ["/app/docker/docker-init.sh"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local depends_on: *superset-depends-on user: "root" volumes: *superset-volumes @@ -89,10 +80,7 @@ services: container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" @@ -110,10 +98,7 @@ services: container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: "root" diff --git a/docker-compose.yml b/docker-compose.yml index 6822a90e05bb..b467013e303a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,10 +55,7 @@ services: db: env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local image: postgres:15 container_name: superset_db restart: unless-stopped @@ -70,10 +67,7 @@ services: superset: env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local build: <<: *common-build container_name: superset_app @@ -125,10 +119,7 @@ services: container_name: superset_init command: ["/app/docker/docker-init.sh"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local depends_on: *superset-depends-on user: *superset-user volumes: *superset-volumes @@ -146,10 +137,7 @@ services: container_name: superset_node command: ["/app/docker/docker-frontend.sh"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local depends_on: *superset-depends-on volumes: *superset-volumes @@ -159,10 +147,7 @@ services: container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -181,10 +166,7 @@ services: container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -198,10 +180,7 @@ services: container_name: superset_tests_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - # defaults - docker/.env - # gitignored overrides - - docker/.env-local environment: DATABASE_HOST: localhost DATABASE_DB: test diff --git a/docs/docs/installation/installing-superset-using-docker-compose.mdx b/docs/docs/installation/installing-superset-using-docker-compose.mdx index 4193a1f0fd47..f372b902412e 100644 --- a/docs/docs/installation/installing-superset-using-docker-compose.mdx +++ b/docs/docs/installation/installing-superset-using-docker-compose.mdx @@ -112,6 +112,7 @@ with docker compose will store that data in a PostgreSQL database contained in a [volume](https://docs.docker.com/storage/volumes/), which is not backed up. Again **DO NOT USE THIS FOR PRODUCTION** + ::: You should see a wall of logging output from the containers being launched on your machine. Once @@ -126,10 +127,10 @@ can skip to the next section. You can install additional python packages and apply config overrides by following the steps mentioned in [docker/README.md](https://github.com/apache/superset/tree/master/docker#configuration) -Note that `docker/.env` sets the default environment variables for all the docker images -used by `docker-compose`, and that `docker/.env-local` can be used to override those defaults. -Also note that `docker/.env-local` is referenced in our `.gitignore`, -preventing developers from risking committing potentially sensitive configuration to the repository. +You can configure the Docker Compose environment variables for dev and non-dev mode with +`docker/.env`. This environment file sets the environment +for most containers in the Docker Compose setup, and some variables affect multiple containers and +others only single ones. One important variable is `SUPERSET_LOAD_EXAMPLES` which determines whether the `superset_init` container will populate example data and visualizations into the metadata database. These examples @@ -137,6 +138,13 @@ are helpful for learning and testing out Superset but unnecessary for experience production deployments. The loading process can sometimes take a few minutes and a good amount of CPU, so you may want to disable it on a resource-constrained device. +For more advanced or dynamic configurations that are typically managed in a `superset_config.py` file +located in your `PYTHONPATH`, note that it can be done by providing a +`docker/pythonpath_dev/superset_config_docker.py` that will be ignored by git +(preventing you to commit/push your local configuration back to the repository). +The mechanics of this are in `docker/pythonpath_dev/superset_config.py` where you can see +that the logic runs a `from superset_config_docker import *` + :::note Users often want to connect to other databases from Superset. Currently, the easiest way to From e721b30c0b0d8d1ab9dd89e51b3eaed548424161 Mon Sep 17 00:00:00 2001 From: Maxime Beauchemin Date: Mon, 15 Apr 2024 09:43:49 -0700 Subject: [PATCH 2/2] backtracking the backtrack --- docker-compose-image-tag.yml | 25 ++++++++++--- docker-compose-non-dev.yml | 25 ++++++++++--- docker-compose.yml | 35 +++++++++++++++---- ...stalling-superset-using-docker-compose.mdx | 8 ++--- 4 files changed, 72 insertions(+), 21 deletions(-) diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml index c3bef372f1fb..4a939c779082 100644 --- a/docker-compose-image-tag.yml +++ b/docker-compose-image-tag.yml @@ -34,7 +34,10 @@ services: db: env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false image: postgres:15 container_name: superset_db restart: unless-stopped @@ -44,7 +47,10 @@ services: superset: env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false image: *superset-image container_name: superset_app command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"] @@ -60,7 +66,10 @@ services: container_name: superset_init command: ["/app/docker/docker-init.sh"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false depends_on: *superset-depends-on user: "root" volumes: *superset-volumes @@ -72,7 +81,10 @@ services: container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false restart: unless-stopped depends_on: *superset-depends-on user: "root" @@ -89,7 +101,10 @@ services: container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false restart: unless-stopped depends_on: *superset-depends-on user: "root" diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index 1462a8654efe..f0b4cccd3233 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -39,7 +39,10 @@ services: db: env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false image: postgres:15 container_name: superset_db restart: unless-stopped @@ -49,7 +52,10 @@ services: superset: env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false build: <<: *common-build container_name: superset_app @@ -67,7 +73,10 @@ services: <<: *common-build command: ["/app/docker/docker-init.sh"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false depends_on: *superset-depends-on user: "root" volumes: *superset-volumes @@ -80,7 +89,10 @@ services: container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false restart: unless-stopped depends_on: *superset-depends-on user: "root" @@ -98,7 +110,10 @@ services: container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false restart: unless-stopped depends_on: *superset-depends-on user: "root" diff --git a/docker-compose.yml b/docker-compose.yml index b467013e303a..8c897ef00240 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,10 @@ services: db: env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false image: postgres:15 container_name: superset_db restart: unless-stopped @@ -67,7 +70,10 @@ services: superset: env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false build: <<: *common-build container_name: superset_app @@ -119,7 +125,10 @@ services: container_name: superset_init command: ["/app/docker/docker-init.sh"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false depends_on: *superset-depends-on user: *superset-user volumes: *superset-volumes @@ -137,7 +146,10 @@ services: container_name: superset_node command: ["/app/docker/docker-frontend.sh"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false depends_on: *superset-depends-on volumes: *superset-volumes @@ -147,7 +159,10 @@ services: container_name: superset_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -166,7 +181,10 @@ services: container_name: superset_worker_beat command: ["/app/docker/docker-bootstrap.sh", "beat"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -180,7 +198,10 @@ services: container_name: superset_tests_worker command: ["/app/docker/docker-bootstrap.sh", "worker"] env_file: - - docker/.env + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false environment: DATABASE_HOST: localhost DATABASE_DB: test diff --git a/docs/docs/installation/installing-superset-using-docker-compose.mdx b/docs/docs/installation/installing-superset-using-docker-compose.mdx index f372b902412e..a6516f378f71 100644 --- a/docs/docs/installation/installing-superset-using-docker-compose.mdx +++ b/docs/docs/installation/installing-superset-using-docker-compose.mdx @@ -127,10 +127,10 @@ can skip to the next section. You can install additional python packages and apply config overrides by following the steps mentioned in [docker/README.md](https://github.com/apache/superset/tree/master/docker#configuration) -You can configure the Docker Compose environment variables for dev and non-dev mode with -`docker/.env`. This environment file sets the environment -for most containers in the Docker Compose setup, and some variables affect multiple containers and -others only single ones. +Note that `docker/.env` sets the default environment variables for all the docker images +used by `docker-compose`, and that `docker/.env-local` can be used to override those defaults. +Also note that `docker/.env-local` is referenced in our `.gitignore`, +preventing developers from risking committing potentially sensitive configuration to the repository. One important variable is `SUPERSET_LOAD_EXAMPLES` which determines whether the `superset_init` container will populate example data and visualizations into the metadata database. These examples