diff --git a/.gitignore b/.gitignore index 33ceade84..5bf94c072 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,8 @@ yarn-error.log* #deployment /deployment/payara/target + +#dev environment +/dev-env/docker-dev-volumes +/dev-env/dataverse +/dev-env/dataverse-sample-data diff --git a/README.md b/README.md index 61018fd38..357f9bd14 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,43 @@ Launches the prettier formatter. We recommend you to configure your IDE to run p Runs the Storybook in the development mode. Open [http://localhost:6006](http://localhost:6006) to view it in your browser. +## Local development environment + +A containerized environment, oriented to local development, is available to be run from the repository. + +This environment contains a dockerized instance of the Dataverse backend with its dependent services (database, mailserver, etc), as well as an npm development server running the SPA frontend (With code autoupdating). + +This environment is intended for locally testing any functionality that requires access to the Dataverse API from the SPA frontend. + +There is an Nginx reverse proxy container on top of the frontend and backend containers to avoid CORS issues while testing the application. + +### Run the environment + +Inside the `dev-env` folder, run the following command: + +``` +./run-env +``` + +As the script argument, add the name of the Dataverse backend branch you want to deploy. + +Note that both the branch and the associated tag in the docker registry must to be pre pushed, otherwise the script will fail. + +If you are running the script for the first time, it may take a while, since `npm install` has to install all the dependencies. This can also happen if you added new dependencies to package.json. + +Once the script has finished, you will be able to access Dataverse via: + +- [http://localhost:8000/spa](http://localhost:8000/spa): SPA Frontend +- [http://localhost:8000](http://localhost:8000): Dataverse Backend and JSF Frontend + +### Remove the environment + +To clean up your environment of any running environment containers, as well as any associated data volumes, run this script inside the `dev-env` folder: + +``` +./rm-env +``` + ## Deployment Once the site is built through the `npm run build` command, it can be deployed in different ways to different types of infrastructure, depending on installation needs. diff --git a/dev-env/.env b/dev-env/.env new file mode 100644 index 000000000..5440ffd9a --- /dev/null +++ b/dev-env/.env @@ -0,0 +1,4 @@ +POSTGRES_VERSION=13 +DATAVERSE_DB_USER=dataverse +SOLR_VERSION=8.11.1 +REGISTRY=ghcr.io diff --git a/dev-env/docker-compose-dev.yml b/dev-env/docker-compose-dev.yml new file mode 100644 index 000000000..c0f10de89 --- /dev/null +++ b/dev-env/docker-compose-dev.yml @@ -0,0 +1,137 @@ +version: '2.4' + +services: + dev_nginx: + container_name: 'dev_nginx_proxy' + image: nginx:stable + ports: + - '8000:80' + networks: + - dataverse + depends_on: + - dev_dataverse + - dev_frontend + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf + - ./docker-dev-volumes/nginx/logs:/var/log/nginx/ + + dev_frontend: + container_name: 'dev_frontend' + hostname: frontend + build: + context: ../ + dockerfile: ./dev.Dockerfile + network: host + expose: + - '5173' + stdin_open: true + networks: + - dataverse + depends_on: + - dev_dataverse + environment: + - VITE_DATAVERSE_BACKEND_URL=http://localhost:8000 + volumes: + - ../:/usr/src/app + - /usr/src/app/dev-env + - /usr/src/app/node_modules + + dev_dataverse: + container_name: 'dev_dataverse' + hostname: dataverse + image: ${REGISTRY}/gdcc/dataverse:${DATAVERSE_BRANCH_NAME} + restart: on-failure + user: payara + environment: + - DATAVERSE_DB_HOST=postgres + - DATAVERSE_DB_PASSWORD=secret + - DATAVERSE_DB_USER=${DATAVERSE_DB_USER} + - DATAVERSE_FEATURE_API_SESSION_AUTH=1 + # We open 8080, rather than just expose, so that the docker-final-setup.sh script works from the run-env.sh script + ports: + - '8080:8080' + networks: + - dataverse + depends_on: + - dev_postgres + - dev_solr + volumes: + - ./docker-dev-volumes/app/data:/dv + - ./docker-dev-volumes/app/secrets:/secrets + tmpfs: + - /dumps:mode=770,size=2052M,uid=1000,gid=1000 + - /tmp:mode=770,size=2052M,uid=1000,gid=1000 + mem_limit: 2147483648 # 2 GiB + mem_reservation: 1024m + privileged: false + + dev_postgres: + container_name: 'dev_postgres' + hostname: postgres + image: postgres:${POSTGRES_VERSION} + restart: on-failure + environment: + - POSTGRES_USER=${DATAVERSE_DB_USER} + - POSTGRES_PASSWORD=secret + ports: + - '5432:5432' + networks: + - dataverse + volumes: + - ./docker-dev-volumes/postgresql/data:/var/lib/postgresql/data + + dev_solr_initializer: + container_name: 'dev_solr_initializer' + image: alpine + restart: 'no' + command: + - sh + - -c + - 'chown 8983:8983 /conf /var/solr && cp *.xml /conf' + volumes: + - ./docker-dev-volumes/solr/data:/var/solr + - ./docker-dev-volumes/solr/conf:/conf + - ./dataverse/conf/solr/8.11.1/schema.xml:/schema.xml + - ./dataverse/conf/solr/8.11.1/solrconfig.xml:/solrconfig.xml + + dev_solr: + container_name: 'dev_solr' + hostname: 'solr' + image: solr:${SOLR_VERSION} + depends_on: + - dev_solr_initializer + restart: on-failure + expose: + - '8983' + networks: + - dataverse + command: + - bash + - -c + - 'cd /opt/solr-${SOLR_VERSION}/server/solr/configsets/_default/conf && cp -R -n . /template && solr-precreate collection1 /template' + volumes: + - ./docker-dev-volumes/solr/data:/var/solr + - ./docker-dev-volumes/solr/conf:/template + + dev_smtp: + container_name: 'dev_smtp' + hostname: 'smtp' + image: maildev/maildev:2.0.5 + restart: on-failure + expose: + - '25' # smtp server + ports: + - '1080:1080' # web ui + environment: + - MAILDEV_SMTP_PORT=25 + - MAILDEV_MAIL_DIRECTORY=/mail + networks: + - dataverse + #volumes: + # - ./docker-dev-volumes/smtp/data:/mail + tmpfs: + - /mail:mode=770,size=128M,uid=1000,gid=1000 + +networks: + dataverse: + driver: bridge diff --git a/dev-env/dvconfig.py b/dev-env/dvconfig.py new file mode 100644 index 000000000..1ef2af74f --- /dev/null +++ b/dev-env/dvconfig.py @@ -0,0 +1,16 @@ +base_url = 'http://localhost:8000' +api_token = '' +sample_data = [ +'data/dataverses/pums/pums.json', +'data/dataverses/pums/datasets/2000pums5/2000pums5.json', +'data/dataverses/dataverseno/dataverseno.json', +'data/dataverses/open-source-at-harvard/open-source-at-harvard.json', +'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/dataverse-project.json', +'data/dataverses/open-source-at-harvard/dataverses/dataverse-project/datasets/dataverse-irc-metrics/dataverse-irc-metrics.json', +'data/dataverses/ubiquity-press/ubiquity-press.json', +'data/dataverses/ubiquity-press/dataverses/jopd/jopd.json', +'data/dataverses/ubiquity-press/dataverses/jopd/datasets/flynn-effect-in-estonia/flynn-effect-in-estonia.json', +'data/dataverses/ubiquity-press/dataverses/jopd/datasets/bafacalo/bafacalo.json', +'data/dataverses/king/king.json', +'data/dataverses/king/datasets/cause-of-death/cause-of-death.json', +] diff --git a/dev-env/nginx.conf b/dev-env/nginx.conf new file mode 100644 index 000000000..e74dfe423 --- /dev/null +++ b/dev-env/nginx.conf @@ -0,0 +1,18 @@ +events {} +http { + server { + listen 80; + server_name localhost; + + location / { + proxy_pass http://dataverse:8080; + } + + location /spa { + proxy_pass http://frontend:5173; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + } + } +} diff --git a/dev-env/rm-env.sh b/dev-env/rm-env.sh new file mode 100755 index 000000000..1bf0a3764 --- /dev/null +++ b/dev-env/rm-env.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +docker-compose -f "./docker-compose-dev.yml" down +rm -rf ./docker-dev-volumes diff --git a/dev-env/run-env.sh b/dev-env/run-env.sh new file mode 100755 index 000000000..997841b79 --- /dev/null +++ b/dev-env/run-env.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +export DATAVERSE_BRANCH_NAME=$1 + +# To avoid timeout issues on frontend container startup +export COMPOSE_HTTP_TIMEOUT=200 + +DATAVERSE_API_BASE_URL=http://localhost:8000/api + +echo "INFO - Setting up Dataverse on branch ${DATAVERSE_BRANCH_NAME}..." + +echo "INFO - Removing current environment if exists..." +./rm-env.sh + +echo "INFO - Cloning Dataverse backend repository..." +git clone -b ${DATAVERSE_BRANCH_NAME} git@github.com:IQSS/dataverse.git + +echo "INFO - Running docker containers..." +docker-compose -f "./docker-compose-dev.yml" up -d --build + +echo "INFO - Waiting for containers to be ready..." +# Up to ~5 minutes +max_attempts=30 +n_attempts=0 +until $(curl --output /dev/null --silent --head --fail ${DATAVERSE_API_BASE_URL}/info/version); do + if [ ${n_attempts} -eq ${max_attempts} ];then + echo "ERROR - Timeout reached while waiting for containers to be ready" + ./rm-env.sh + rm -rf dataverse + exit 1 + fi + n_attempts=$(($n_attempts+1)) + sleep 10 +done + +echo "INFO - Bootstrapping dataverse..." +cd dataverse +./scripts/dev/docker-final-setup.sh + +echo "INFO - Cleaning up repository..." +cd .. +rm -rf dataverse + +echo "INFO - Cloning Dataverse sample data repository..." +git clone git@github.com:IQSS/dataverse-sample-data.git + +echo "INFO - Configuring Dataverse sample data repository..." +cd dataverse-sample-data +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +cp ../dvconfig.py ./dvconfig.py +curl -X PUT -d 'true' ${DATAVERSE_API_BASE_URL}/admin/settings/:AllowApiTokenLookupViaApi +dataverse_api_token=$(python3 get_api_token.py) +sed -i '' "s//${dataverse_api_token}/g" dvconfig.py + +echo "INFO - Creating sample data..." +python3 create_sample_data.py + +echo "INFO - Cleaning up repository..." +cd .. +rm -rf dataverse-sample-data diff --git a/dev.Dockerfile b/dev.Dockerfile new file mode 100644 index 000000000..a00e33c5a --- /dev/null +++ b/dev.Dockerfile @@ -0,0 +1,6 @@ +FROM node:19.6.1 +WORKDIR /usr/src/app +COPY package.json ./ +RUN npm install +EXPOSE 5173 +CMD ["npm", "start"] diff --git a/package.json b/package.json index f1caed6f5..69bd89f1d 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "web-vitals": "^2.1.4" }, "scripts": { - "start": "vite", + "start": "vite --base=/spa", "build": "tsc && vite build", "preview": "vite preview", "test": "vitest watch", diff --git a/vite.config.ts b/vite.config.ts index d443ec029..2a2041a9a 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -5,5 +5,13 @@ export default defineConfig({ plugins: [react()], preview: { port: 5173 + }, + server: { + //https://github.com/vitejs/vite/discussions/3396 + host: true, + port: 5173, + hmr: { + clientPort: 8000 // nginx reverse proxy port + } } })