From 06f90644d454f41555d77da82ef2f477031fe73e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Placzy=C5=84ski?= Date: Fri, 21 Feb 2025 12:01:16 +0100 Subject: [PATCH 01/23] [#3054] Remove leftover infrastructure configuration This commit removes the remaining configuration and automation related to Byron's infrastructure, following the decision to withdraw from Byron's instances for SanchoNet. These instances were shut down on February 19, 2025. --- .github/workflows/build-and-deploy-beta.yml | 95 -- .github/workflows/build-and-deploy-dev.yml | 95 -- .../workflows/build-and-deploy-staging.yml | 98 -- .github/workflows/build-and-deploy-test.yml | 98 -- .github/workflows/deploy-dev-from-tag.yml | 67 -- .../resync-cardano-node-and-db-sync.yml | 112 --- .github/workflows/toggle-maintenance.yml | 68 -- README.md | 5 - docs/operations/DEPLOY.md | 70 -- docs/operations/README.md | 69 -- govtool/status-service/.dockerignore | 3 - govtool/status-service/.gitignore | 3 - govtool/status-service/Dockerfile | 9 - govtool/status-service/app.py | 42 - govtool/status-service/requirements.txt | 14 - govtool/status-service/status-service.mk | 21 - govtool/status-service/templates/status.html | 14 - infra/terraform/.envrc | 3 - infra/terraform/README.md | 99 -- infra/terraform/bootstrap-aws-account.sh | 19 - infra/terraform/default.nix | 12 - infra/terraform/main.tf | 162 --- infra/terraform/modules/ecr/main.tf | 11 - infra/terraform/modules/ecr/outputs.tf | 7 - infra/terraform/modules/ecr/variables.tf | 4 - infra/terraform/modules/govtool-ec2/main.tf | 204 ---- .../terraform/modules/govtool-ec2/outputs.tf | 7 - .../modules/govtool-ec2/user_data.sh | 28 - .../modules/govtool-ec2/variables.tf | 26 - .../vva_to_govtool_migration_20240212.tf | 15 - infra/terraform/versions.tf | 16 - .../vva_to_govtool_migration_20240212.tf | 35 - scripts/govtool/.envrc | 16 - scripts/govtool/Makefile | 76 -- scripts/govtool/README.md | 183 ---- scripts/govtool/analytics-dashboard.mk | 22 - scripts/govtool/backend.mk | 31 - scripts/govtool/common.mk | 65 -- scripts/govtool/config.mk | 129 --- .../config/templates/backend-config.json.tpl | 14 - .../config/templates/docker-compose.yml.tpl | 308 ------ .../alerting/alerting.yml | 379 ------- .../dashboards/dashboard.yml | 21 - .../dashboards/govtool.json | 921 ------------------ .../dashboards/traefik_rev4.json | 693 ------------- .../datasources/datasource.yml | 84 -- scripts/govtool/config/templates/loki.yml | 56 -- .../govtool/config/templates/prometheus.yml | 23 - scripts/govtool/config/templates/promtail.yml | 18 - .../config/templates/ssh-client-config | 4 - .../govtool/custom-cardano-db-sync.Dockerfile | 6 - .../custom-cardano-db-sync.entrypoint.sh | 39 - scripts/govtool/custom-cardano-db-sync.mk | 22 - scripts/govtool/default.nix | 29 - .../govtool/docker-compose.node+dbsync.yml | 99 -- scripts/govtool/frontend.mk | 35 - scripts/govtool/info.mk | 44 - scripts/govtool/metadata-validation.mk | 23 - scripts/govtool/status-service.mk | 21 - scripts/govtool/utils.mk | 30 - 60 files changed, 4922 deletions(-) delete mode 100644 .github/workflows/build-and-deploy-beta.yml delete mode 100644 .github/workflows/build-and-deploy-dev.yml delete mode 100644 .github/workflows/build-and-deploy-staging.yml delete mode 100644 .github/workflows/build-and-deploy-test.yml delete mode 100644 .github/workflows/deploy-dev-from-tag.yml delete mode 100644 .github/workflows/resync-cardano-node-and-db-sync.yml delete mode 100644 .github/workflows/toggle-maintenance.yml delete mode 100644 docs/operations/DEPLOY.md delete mode 100644 docs/operations/README.md delete mode 100644 govtool/status-service/.dockerignore delete mode 100644 govtool/status-service/.gitignore delete mode 100644 govtool/status-service/Dockerfile delete mode 100644 govtool/status-service/app.py delete mode 100644 govtool/status-service/requirements.txt delete mode 100644 govtool/status-service/status-service.mk delete mode 100644 govtool/status-service/templates/status.html delete mode 100644 infra/terraform/.envrc delete mode 100644 infra/terraform/README.md delete mode 100755 infra/terraform/bootstrap-aws-account.sh delete mode 100644 infra/terraform/default.nix delete mode 100644 infra/terraform/main.tf delete mode 100644 infra/terraform/modules/ecr/main.tf delete mode 100644 infra/terraform/modules/ecr/outputs.tf delete mode 100644 infra/terraform/modules/ecr/variables.tf delete mode 100644 infra/terraform/modules/govtool-ec2/main.tf delete mode 100644 infra/terraform/modules/govtool-ec2/outputs.tf delete mode 100644 infra/terraform/modules/govtool-ec2/user_data.sh delete mode 100644 infra/terraform/modules/govtool-ec2/variables.tf delete mode 100644 infra/terraform/modules/govtool-ec2/vva_to_govtool_migration_20240212.tf delete mode 100644 infra/terraform/versions.tf delete mode 100644 infra/terraform/vva_to_govtool_migration_20240212.tf delete mode 100644 scripts/govtool/.envrc delete mode 100644 scripts/govtool/Makefile delete mode 100644 scripts/govtool/README.md delete mode 100644 scripts/govtool/analytics-dashboard.mk delete mode 100644 scripts/govtool/backend.mk delete mode 100644 scripts/govtool/common.mk delete mode 100644 scripts/govtool/config.mk delete mode 100644 scripts/govtool/config/templates/backend-config.json.tpl delete mode 100644 scripts/govtool/config/templates/docker-compose.yml.tpl delete mode 100644 scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml delete mode 100644 scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml delete mode 100644 scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json delete mode 100644 scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json delete mode 100644 scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml delete mode 100644 scripts/govtool/config/templates/loki.yml delete mode 100644 scripts/govtool/config/templates/prometheus.yml delete mode 100644 scripts/govtool/config/templates/promtail.yml delete mode 100644 scripts/govtool/config/templates/ssh-client-config delete mode 100644 scripts/govtool/custom-cardano-db-sync.Dockerfile delete mode 100644 scripts/govtool/custom-cardano-db-sync.entrypoint.sh delete mode 100644 scripts/govtool/custom-cardano-db-sync.mk delete mode 100644 scripts/govtool/default.nix delete mode 100644 scripts/govtool/docker-compose.node+dbsync.yml delete mode 100644 scripts/govtool/frontend.mk delete mode 100644 scripts/govtool/info.mk delete mode 100644 scripts/govtool/metadata-validation.mk delete mode 100644 scripts/govtool/status-service.mk delete mode 100644 scripts/govtool/utils.mk diff --git a/.github/workflows/build-and-deploy-beta.yml b/.github/workflows/build-and-deploy-beta.yml deleted file mode 100644 index eb9e05763..000000000 --- a/.github/workflows/build-and-deploy-beta.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Build and deploy GovTool to BETA server -run-name: Deploy by @${{ github.actor }} - -# That should be executed on create: tag event -on: - workflow_dispatch: - inputs: - isProposalDiscussionForumEnabled: - description: "Enable proposal discussion forum" - required: true - type: choice - default: "enabled" - options: - - "enabled" - - "disabled" - forceRebuildDockerImages: - description: "Force rebuild the docker images" - required: false - type: choice - default: "false" - options: - - "true" - - "false" - -env: - ENVIRONMENT: "beta" - CARDANO_NETWORK: "sanchonet" - DOMAIN: "sanchogov.tools" - FORCE_REBUILD: ${{inputs.forceRebuildDockerImages == 'true'}} - -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - env: - DBSYNC_POSTGRES_DB: ${{ secrets.DBSYNC_POSTGRES_DB || 'cexplorer' }} - DBSYNC_POSTGRES_PASSWORD: ${{ secrets.DBSYNC_POSTGRES_PASSWORD || 'pSa8JCpQOACMUdGb' }} - DBSYNC_POSTGRES_USER: ${{ secrets.DBSYNC_POSTGRES_USER || 'postgres' }} - DBSYNC_POSTGRES_HOST: ${{ secrets.DBSYNC_POSTGRES_HOST || 'postgres' }} - DBSYNC_POSTGRES_PORT: ${{ secrets.DBSYNC_POSTGRES_PORT || '5432' }} - GA_CLIENT_EMAIL: ${{ secrets.GA_CLIENT_EMAIL }} - GA_PRIVATE_KEY: ${{ secrets.GA_PRIVATE_KEY }} - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }} - IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }} - NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools" - NEXT_PUBLIC_GA4_PROPERTY_ID: ${{ secrets.NEXT_PUBLIC_GA4_PROPERTY_ID }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - SENTRY_IGNORE_API_RESOLUTION_ERROR: "1" - TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" - USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} - IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }} - PDF_API_URL: ${{ secrets.PDF_API_URL}} - IPFS_GATEWAY: ${{ secrets.IPFS_GATEWAY }} - IPFS_PROJECT_ID: ${{ secrets.IPFS_PROJECT_ID }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Deploy app - run: | - make --debug=b all - - - name: Reprovision Grafana - run: | - sleep 30 # give grafana time to start up - make --debug=b reload-grafana diff --git a/.github/workflows/build-and-deploy-dev.yml b/.github/workflows/build-and-deploy-dev.yml deleted file mode 100644 index f80a3b91b..000000000 --- a/.github/workflows/build-and-deploy-dev.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Build and deploy GovTool to DEV server -run-name: Deploy by @${{ github.actor }} - -on: - workflow_dispatch: - inputs: - isProposalDiscussionForumEnabled: - description: "Enable proposal discussion forum" - required: true - type: choice - default: "enabled" - options: - - "enabled" - - "disabled" - forceRebuildDockerImages: - description: "Force rebuild the docker images" - required: false - type: choice - default: "false" - options: - - "true" - - "false" - -env: - ENVIRONMENT: "dev" - CARDANO_NETWORK: "sanchonet" - DOMAIN: "dev-sanchonet.govtool.byron.network" - FORCE_REBUILD: ${{inputs.forceRebuildDockerImages == 'true'}} - -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - env: - DBSYNC_POSTGRES_DB: ${{ secrets.DBSYNC_POSTGRES_DB || 'cexplorer' }} - DBSYNC_POSTGRES_PASSWORD: ${{ secrets.DBSYNC_POSTGRES_PASSWORD || 'pSa8JCpQOACMUdGb' }} - DBSYNC_POSTGRES_USER: ${{ secrets.DBSYNC_POSTGRES_USER || 'postgres' }} - DBSYNC_POSTGRES_HOST: ${{ secrets.DBSYNC_POSTGRES_HOST || 'postgres' }} - DBSYNC_POSTGRES_PORT: ${{ secrets.DBSYNC_POSTGRES_PORT || '5432' }} - GA_CLIENT_EMAIL: ${{ secrets.GA_CLIENT_EMAIL }} - GA_PRIVATE_KEY: ${{ secrets.GA_PRIVATE_KEY }} - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }} - IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }} - NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools" - NEXT_PUBLIC_GA4_PROPERTY_ID: ${{ secrets.NEXT_PUBLIC_GA4_PROPERTY_ID }} - DEV_NGINX_BASIC_AUTH: ${{ secrets.DEV_NGINX_BASIC_AUTH }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - SENTRY_IGNORE_API_RESOLUTION_ERROR: "1" - TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" - USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} - IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }} - PDF_API_URL: ${{ secrets.PDF_API_URL }} - IPFS_GATEWAY: ${{ secrets.IPFS_GATEWAY }} - IPFS_PROJECT_ID: ${{ secrets.IPFS_PROJECT_ID }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Deploy app - run: | - make --debug=b all - - - name: Reprovision Grafana - run: | - sleep 30 # give grafana time to start up - make --debug=b reload-grafana diff --git a/.github/workflows/build-and-deploy-staging.yml b/.github/workflows/build-and-deploy-staging.yml deleted file mode 100644 index bbd68afc3..000000000 --- a/.github/workflows/build-and-deploy-staging.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Build and deploy GovTool to STAGING server -run-name: Deploy by @${{ github.actor }} - -on: - push: - branches: - - staging - workflow_dispatch: - inputs: - isProposalDiscussionForumEnabled: - description: "Enable proposal discussion forum" - required: true - type: choice - default: "enabled" - options: - - "enabled" - - "disabled" - forceRebuildDockerImages: - description: "Force rebuild the docker images" - required: false - type: choice - default: "false" - options: - - "true" - - "false" - -env: - ENVIRONMENT: "staging" - CARDANO_NETWORK: "sanchonet" - DOMAIN: "staging.govtool.byron.network" - FORCE_REBUILD: ${{inputs.forceRebuildDockerImages == 'true'}} - -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - env: - DBSYNC_POSTGRES_DB: ${{ secrets.DBSYNC_POSTGRES_DB || 'cexplorer' }} - DBSYNC_POSTGRES_PASSWORD: ${{ secrets.DBSYNC_POSTGRES_PASSWORD || 'pSa8JCpQOACMUdGb' }} - DBSYNC_POSTGRES_USER: ${{ secrets.DBSYNC_POSTGRES_USER || 'postgres' }} - DBSYNC_POSTGRES_HOST: ${{ secrets.DBSYNC_POSTGRES_HOST || 'postgres' }} - DBSYNC_POSTGRES_PORT: ${{ secrets.DBSYNC_POSTGRES_PORT || '5432' }} - GA_CLIENT_EMAIL: ${{ secrets.GA_CLIENT_EMAIL }} - GA_PRIVATE_KEY: ${{ secrets.GA_PRIVATE_KEY }} - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }} - IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }} - NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools" - NEXT_PUBLIC_GA4_PROPERTY_ID: ${{ secrets.NEXT_PUBLIC_GA4_PROPERTY_ID }} - STAGING_NGINX_BASIC_AUTH: ${{ secrets.STAGING_NGINX_BASIC_AUTH }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - SENTRY_IGNORE_API_RESOLUTION_ERROR: "1" - TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" - USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} - IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{github.event_name == 'push' && 'true' || inputs.isProposalDiscussionForumEnabled == 'enabled'}} - PDF_API_URL: ${{ secrets.PDF_API_URL}} - IPFS_GATEWAY: ${{ secrets.IPFS_GATEWAY }} - IPFS_PROJECT_ID: ${{ secrets.IPFS_PROJECT_ID }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Deploy app - run: | - make --debug=b all - - - name: Reprovision Grafana - run: | - sleep 30 # give grafana time to start up - make --debug=b reload-grafana diff --git a/.github/workflows/build-and-deploy-test.yml b/.github/workflows/build-and-deploy-test.yml deleted file mode 100644 index 201f18799..000000000 --- a/.github/workflows/build-and-deploy-test.yml +++ /dev/null @@ -1,98 +0,0 @@ -name: Build and deploy GovTool to TEST server -run-name: Deploy by @${{ github.actor }} - -on: - push: - branches: - - test - workflow_dispatch: - inputs: - isProposalDiscussionForumEnabled: - description: "Enable proposal discussion forum" - required: true - type: choice - default: "enabled" - options: - - "enabled" - - "disabled" - forceRebuildDockerImages: - description: "Force rebuild the docker images" - required: false - type: choice - default: "false" - options: - - "true" - - "false" - -env: - ENVIRONMENT: "test" - CARDANO_NETWORK: "sanchonet" - DOMAIN: "test-sanchonet.govtool.byron.network" - FORCE_REBUILD: ${{inputs.forceRebuildDockerImages == 'true'}} - -jobs: - deploy: - name: Deploy app - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - env: - DBSYNC_POSTGRES_DB: ${{ secrets.DBSYNC_POSTGRES_DB || 'cexplorer' }} - DBSYNC_POSTGRES_PASSWORD: ${{ secrets.DBSYNC_POSTGRES_PASSWORD || 'pSa8JCpQOACMUdGb' }} - DBSYNC_POSTGRES_USER: ${{ secrets.DBSYNC_POSTGRES_USER || 'postgres' }} - DBSYNC_POSTGRES_HOST: ${{ secrets.DBSYNC_POSTGRES_HOST || 'postgres' }} - DBSYNC_POSTGRES_PORT: ${{ secrets.DBSYNC_POSTGRES_PORT || '5432' }} - GA_CLIENT_EMAIL: ${{ secrets.GA_CLIENT_EMAIL }} - GA_PRIVATE_KEY: ${{ secrets.GA_PRIVATE_KEY }} - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }} - IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }} - NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools" - NEXT_PUBLIC_GA4_PROPERTY_ID: ${{ secrets.NEXT_PUBLIC_GA4_PROPERTY_ID }} - TEST_NGINX_BASIC_AUTH: ${{ secrets.TEST_NGINX_BASIC_AUTH }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - SENTRY_IGNORE_API_RESOLUTION_ERROR: "1" - TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" - USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} - IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{github.event_name == 'push' && 'true' || inputs.isProposalDiscussionForumEnabled == 'enabled'}} - PDF_API_URL: ${{ secrets.PDF_API_URL}} - IPFS_GATEWAY: ${{ secrets.IPFS_GATEWAY }} - IPFS_PROJECT_ID: ${{ secrets.IPFS_PROJECT_ID }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Deploy app - run: | - make --debug=b all - - - name: Reprovision Grafana - run: | - sleep 30 # give grafana time to start up - make --debug=b reload-grafana diff --git a/.github/workflows/deploy-dev-from-tag.yml b/.github/workflows/deploy-dev-from-tag.yml deleted file mode 100644 index 9c0d94ec8..000000000 --- a/.github/workflows/deploy-dev-from-tag.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Build and deploy GovTool to DEV server -run-name: Deploy by @${{ github.actor }} - -on: - create: - -env: - ENVIRONMENT: "dev" - CARDANO_NETWORK: "sanchonet" - DOMAIN: "dev-sanchonet.govtool.byron.network" - -jobs: - deploy: - name: Deploy app - if: github.ref_type == 'tag' && startsWith(github.ref, 'refs/tags/dev-') - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - env: - DBSYNC_POSTGRES_DB: "cexplorer" - DBSYNC_POSTGRES_USER: "postgres" - DBSYNC_POSTGRES_PASSWORD: "pSa8JCpQOACMUdGb" - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - NGINX_BASIC_AUTH: ${{ secrets.NGINX_BASIC_AUTH }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }} - IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }} - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - ref: ${{ github.ref }} - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Deploy app - run: | - make --debug=b all - - - name: Reprovision Grafana - run: | - sleep 30 # give grafana time to start up - make --debug=b reload-grafana diff --git a/.github/workflows/resync-cardano-node-and-db-sync.yml b/.github/workflows/resync-cardano-node-and-db-sync.yml deleted file mode 100644 index f4ffc8fac..000000000 --- a/.github/workflows/resync-cardano-node-and-db-sync.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Re-sync Cardano Node and Cardano DB Sync -run-name: Cardano re-sync on ${{ inputs.environment }} by @${{ github.actor }} - -on: - workflow_dispatch: - inputs: - environment: - required: true - type: choice - default: "dev" - options: - - "dev" - - "test" - - "staging" - - "beta" - isProposalDiscussionForumEnabled: - description: "Enable proposal discussion forum" - required: true - type: choice - default: "enabled" - options: - - "enabled" - - "disabled" - forceRebuildDockerImages: - description: "Force rebuild the docker images" - required: false - type: choice - default: "false" - options: - - "true" - - "false" - -env: - ENVIRONMENT: ${{ inputs.environment || 'dev' }} - CARDANO_NETWORK: "sanchonet" - FORCE_REBUILD: ${{inputs.forceRebuildDockerImages == 'true'}} - -jobs: - deploy: - name: Re-sync Cardano state - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - env: - DBSYNC_POSTGRES_DB: ${{ secrets.DBSYNC_POSTGRES_DB || 'cexplorer' }} - DBSYNC_POSTGRES_PASSWORD: ${{ secrets.DBSYNC_POSTGRES_PASSWORD || 'pSa8JCpQOACMUdGb' }} - DBSYNC_POSTGRES_USER: ${{ secrets.DBSYNC_POSTGRES_USER || 'postgres' }} - DBSYNC_POSTGRES_HOST: ${{ secrets.DBSYNC_POSTGRES_HOST || 'postgres' }} - DBSYNC_POSTGRES_PORT: ${{ secrets.DBSYNC_POSTGRES_PORT || '5432' }} - GA_CLIENT_EMAIL: ${{ secrets.GA_CLIENT_EMAIL }} - GA_PRIVATE_KEY: ${{ secrets.GA_PRIVATE_KEY }} - GRAFANA_ADMIN_PASSWORD: ${{ secrets.GRAFANA_ADMIN_PASSWORD }} - GRAFANA_SLACK_OAUTH_TOKEN: ${{ secrets.GRAFANA_SLACK_OAUTH_TOKEN }} - GRAFANA_SLACK_RECIPIENT: ${{ secrets.GRAFANA_SLACK_RECIPIENT }} - GTM_ID: ${{ secrets.GTM_ID }} - NPMRC_TOKEN: ${{ secrets.NPMRC_TOKEN }} - IP_ADDRESS_BYPASSING_BASIC_AUTH1: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH1 }} - IP_ADDRESS_BYPASSING_BASIC_AUTH2: ${{ secrets.IP_ADDRESS_BYPASSING_BASIC_AUTH2 }} - NEXT_PUBLIC_API_URL: "https://participation.sanchogov.tools" - NEXT_PUBLIC_GA4_PROPERTY_ID: ${{ secrets.NEXT_PUBLIC_GA4_PROPERTY_ID }} - DEV_NGINX_BASIC_AUTH: ${{ secrets.DEV_NGINX_BASIC_AUTH }} - TEST_NGINX_BASIC_AUTH: ${{ secrets.TEST_NGINX_BASIC_AUTH }} - STAGING_NGINX_BASIC_AUTH: ${{ secrets.STAGING_NGINX_BASIC_AUTH }} - PIPELINE_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - SENTRY_DSN: ${{ secrets.SENTRY_DSN_FRONTEND }} - SENTRY_DSN_BACKEND: ${{ secrets.SENTRY_DSN_BACKEND }} - SENTRY_IGNORE_API_RESOLUTION_ERROR: "1" - TRAEFIK_LE_EMAIL: "admin+govtool@binarapps.com" - USERSNAP_SPACE_API_KEY: ${{ secrets.USERSNAP_SPACE_API_KEY }} - IS_PROPOSAL_DISCUSSION_FORUM_ENABLED: ${{ inputs.isProposalDiscussionForumEnabled == 'enabled' }} - PDF_API_URL: ${{ secrets.PDF_API_URL }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Set domain - run: | - if [[ "${{ inputs.environment }}" == "staging" ]]; then - echo "DOMAIN=staging.govtool.byron.network" >> $GITHUB_ENV - elif [[ "${{ inputs.environment }}" == "beta" ]]; then - echo "DOMAIN=sanchogov.tools" >> $GITHUB_ENV - else - echo "DOMAIN=${DOMAIN:-$ENVIRONMENT-$CARDANO_NETWORK.govtool.byron.network}" >> $GITHUB_ENV - fi - - - name: Destroy Cardano Node, DB sync and Postgres database - run: | - make --debug=b destroy-cardano-node-and-dbsync - - - name: Deploy app - run: | - make --debug=b all diff --git a/.github/workflows/toggle-maintenance.yml b/.github/workflows/toggle-maintenance.yml deleted file mode 100644 index bdb5f5434..000000000 --- a/.github/workflows/toggle-maintenance.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Toggle maintenance page -run-name: Maintenance mode set to ${{ inputs.maintenance }} on ${{ inputs.environment }} by @${{ github.actor }} - -on: - workflow_dispatch: - inputs: - environment: - required: true - type: choice - default: "dev" - options: - - "dev" - - "test" - - "staging" - - "beta" - maintenance: - required: true - type: choice - default: "enabled" - options: - - "enabled" - - "disabled" - -env: - ENVIRONMENT: ${{ inputs.environment || 'dev' }} - CARDANO_NETWORK: "sanchonet" - -jobs: - toggle_maintenance: - name: Toggl maintenance state - runs-on: ubuntu-latest - defaults: - run: - working-directory: ./scripts/govtool - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }} - aws-region: eu-west-1 - - - name: Login to AWS ECR - uses: aws-actions/configure-aws-credentials@v2 - with: - aws-region: eu-west-1 - - - name: Setup SSH agent - uses: webfactory/ssh-agent@v0.8.0 - with: - ssh-private-key: ${{ secrets.GHA_SSH_PRIVATE_KEY }} - - - name: Set domain - run: | - if [[ "${{ inputs.environment }}" == "staging" ]]; then - echo "DOMAIN=staging.govtool.byron.network" >> $GITHUB_ENV - elif [[ "${{ inputs.environment }}" == "beta" ]]; then - echo "DOMAIN=sanchogov.tools" >> $GITHUB_ENV - else - echo "DOMAIN=${DOMAIN:-$ENVIRONMENT-$CARDANO_NETWORK.govtool.byron.network}" >> $GITHUB_ENV - fi - - - name: Toggle maintenance - run: | - make toggle-maintenance maintenance=${{ inputs.maintenance || 'disabled' }} diff --git a/README.md b/README.md index 27388fa33..5fb7b8912 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,6 @@ The Cardano GovTool enables ada holders to experience the governance features de - [gov.tools](https://gov.tools/) -#### SanchoNet - -- [sanchogov.tools](https://sanchogov.tools/) - #### Preview Testnet - [preview.gov.tools](https://preview.gov.tools/) @@ -42,7 +38,6 @@ Learn more; [docs.gov.tools](https://docs.gov.tools/cardano-govtool/using-govtoo - [Backend](./govtool/backend/README.md) - [Frontend](./govtool/frontend/README.md) -- [Infrastructure](./infra/terraform/README.md) - [In repo documentation](./docs/) - [Tests](./tests/) diff --git a/docs/operations/DEPLOY.md b/docs/operations/DEPLOY.md deleted file mode 100644 index 2f75566f3..000000000 --- a/docs/operations/DEPLOY.md +++ /dev/null @@ -1,70 +0,0 @@ -# Deployment - -## AWS Setup - -Requirements: - -- aws-cli v2+ -- Terraform v1.5.6+ -- AWS IAM account CLI keys - -### Using NIX configuration - -Change directory to `src` and type `nix-shell`. The required tools will be -downloaded and installed in their required versions. - -### Bootstrap - -AWS is configured with Terraform, but the account needs to be bootstrapped beforehand. - -This has to be done only once per AWS account. - -1. Configure AWS CLI with `aws configure` (if you are using a configuration profile, use `aws --profile MYPROFILE configure` and then `export AWS_PROFILE=MYPROFILE`). -1. Execute `./infra/terraform/bootstrap-aws-account.sh` script. - -### Infrastructure setup - -1. Change directory to `src/terraform`. -1. Run `terraform init` to initialize Terraform. -1. Run `terraform plan` to view changes that would be performed to infrastructure. -1. Take note of the outputs - they contain ECR repo URLs and app domains. - -Note: the Terraform code configures the EC2 instance using `infra/terraform/modules/govtool-ec2/user_data.sh`. This script is only executed on instance creation. - -## Application deployment - -1. Set environment variables in your shell (see `src/.env.example` file for list of variables and sample values). -1. Prepare configuration files for target environment: - 1. Run `make prepare-config` - this will create `config/target` directory on your machine with necessary files. - 1. Run `make upload-config instance=$INSTANCE cardano_network=$CARDANO_NETWORK env=$ENVIRONMENT` - this will upload the `config/target` directory to appropriate EC2 instance. -1. Build the application Docker images: - 1. Run `make build-backend`. - 1. Run `make build-frontend instance=$INSTANCE cardano_network=$CARDANO_NETWORK env=$ENVIRONMENT`. -1. Push the images to ECR: - 1. Run `make docker-login`. - 1. Run `make push-backend`. - 1. Run `make push-frontend instance=$INSTANCE cardano_network=$CARDANO_NETWORK env=$ENVIRONMENT`. -1. Deploy the application: run `make deploy-stack instance=$INSTANCE cardano_network=$CARDANO_NETWORK env=$ENVIRONMENT`. - -On first deploy, the last command will take a long while. On subsequent deployments it should perform faster. - -Alternatively you can type `make all instance=$INSTANCE cardano_network=$CARDANO_NETWORK env=$ENVIRONMENT` to deploy all at once. - -View the app at `https://${ENVIRONMENT}-${INSTANCE}.govtool.byron.network`. -Keep in mind that after initial deployment on a new environment, it will take some time for the Cardano node to get in sync. - -## Aftermatch - -After performing a deploy from a local machine, it is crucial to carefully track -the application's accessibility and functionality to ensure no unintended -changes, such as the accidental activation of BasicAuth[^1], have occurred. - -Additionally, verifying that all configuration files and environment variables, -particularly those related to environment-specific settings like database -connections or external service endpoints, are correctly generated and applied -is essential for maintaining the intended behavior of the application across -different environments. This includes a thorough check of environment variables -to confirm they are correctly set and aligned with the specific needs of the -deployed environment. - -[^1]: https://github.com/IntersectMBO/govtool/discussions/174 diff --git a/docs/operations/README.md b/docs/operations/README.md deleted file mode 100644 index f14769304..000000000 --- a/docs/operations/README.md +++ /dev/null @@ -1,69 +0,0 @@ -# Overview - -The application is setup with the following tools: - -- Terraform - for creating infrastructure in AWS for each environment -- Docker - to build and run application components -- Docker Compose - to connect the application components together and deploy them as a stack -- make - to simplify operations tasks -- Prometheus - to gather metrics from application host and Docker containers -- Grafana - to visualize metrics gathered from Prometheus and handle alerting - -# Environments - -The application is hosted on AWS, there are several application environments, each of them is described with Terraform in `src/terraform/main.tf` file. Terraform is executed manually, in order to add/modify/delete an environment, modify the code and run `terraform plan` to see the changes and `terraform apply` to execute them. - -Each environment consists of: - -- VPC network (with subnets, route tables, IGW) -- Security Groups -- EC2 instance -- Elastic IPs associated with EC2 instance -- Route 53 record (only for environments using `govtool.byron.network` domain) - -For each environment, the frontend is hosted at root and the backend is at `/api`. - -## List of public environments - -### beta - -A beta environment connected to `preview` Cardano network. - -Available at https://preview.gov.tools/. The DNS record for this domain is created manually. - -# Deployment - -Deployment is performed via GitHub Actions workflow (`.github/workflows/build-and-deploy.yml`). - -The workflow performs the following steps: - -- check if the environment is defined in Terraform (to avoid deployment attempt to inexistant environment) -- build of frontend app -- build of backend app -- generate configuration files and upload them (over SSH) to the target environment -- setup the application compoments with Docker Compose on the target environment - -The workflow can be triggered directly from GitHub Actions panel. When ruuning the workflow, you need to specify: - -- Cardano network to be used -- environment name -- optionally skip the build process (frontend and backend) - useful when there are plain configuration changes that do not require the application to be rebuild - -# Monitoring - -Monitoring is achieved with Prometheus and Grafana, which are deployed together with each environment. Grafana is available at `/grafana`, the `admin` password is defined in a GitHub Actions secret `GRAFANA_ADMIN_PASSWORD`. - -Each Grafana instance is managed as code and provisioned with YAML files located at `src/config/grafana-provisioning`. This includes a default datasource, dashboard and alerting. The alerts are configured to be sent to Slack via Slack bot. - -The Slack bot OAuth token and recipient are defined with GitHub Actions secrets `GRAFANA_SLACK_OAUTH_TOKEN` and `GRAFANA_SLACK_RECIPIENT`, respectively. - -### Frontend - -Deploying new versions of the application is done using Github actions - -1. Open [`repository`](https://github.com/IntersectMBO/govtool) -2. Select "Actions" tab -3. From left menu choose "Build and deploy app" -4. From the droping options - "Run workflow", select the branch, Cardano network and type of environment for your deployment -5. Press "Run workflow" -6. Wait for the final effect. It's done. diff --git a/govtool/status-service/.dockerignore b/govtool/status-service/.dockerignore deleted file mode 100644 index 63c47ef33..000000000 --- a/govtool/status-service/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -__pycache__ -Dockerfile -venv diff --git a/govtool/status-service/.gitignore b/govtool/status-service/.gitignore deleted file mode 100644 index 2552d9aac..000000000 --- a/govtool/status-service/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.swp -__pycache__ -venv diff --git a/govtool/status-service/Dockerfile b/govtool/status-service/Dockerfile deleted file mode 100644 index b4a2d94d3..000000000 --- a/govtool/status-service/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM python:3.11-alpine -WORKDIR /app -CMD ["python3", "-m", "gunicorn", "--bind=0.0.0.0", "--workers=4", "--access-logfile=-", "app:app"] -ENV GRAFANA_USERNAME admin -ENV GRAFANA_PASSWORD set-me-at-runtime - -COPY requirements.txt ./requirements.txt -RUN pip install -r requirements.txt -COPY . ./ diff --git a/govtool/status-service/app.py b/govtool/status-service/app.py deleted file mode 100644 index a453218f4..000000000 --- a/govtool/status-service/app.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -import json -from os import environ -from flask import Flask, render_template, request, json, Response - -GRAFANA_USERNAME = environ['GRAFANA_USERNAME'] -GRAFANA_PASSWORD = environ['GRAFANA_PASSWORD'] - -alert_health_mapping = { - 'inactive': 'healthy', - 'pending': 'warning', - 'firing': 'not healthy' -} -app = Flask(__name__) - - -def fetch_grafana_alert_rules(grafana_url): - rules_endpoint = '/api/prometheus/grafana/api/v1/rules?limit_alerts=10' - res = requests.get(grafana_url + rules_endpoint, auth=(GRAFANA_USERNAME, GRAFANA_PASSWORD)) - return res.text - - -@app.route('/status') -def status(): - my_host_header = request.headers['Host'] - grafana_url = 'https://' + my_host_header + '/grafana' - app_health = {} - rules = json.loads(fetch_grafana_alert_rules(grafana_url)) - if rules['status'] == 'success': - for group in rules['data']['groups']: - if group['name'] == 'status': - for rule in group['rules']: - component_name = rule['labels']['app_component'] - component_health = alert_health_mapping[rule['state']] - app_health[component_name] = component_health - return render_template( - 'status.html', - cardano_node_state=app_health['cardano-node'], - cardano_db_sync_state=app_health['cardano-db-sync'], - backend_state=app_health['backend']) - else: - return Response('failed to fetch rules from grafana', 500) diff --git a/govtool/status-service/requirements.txt b/govtool/status-service/requirements.txt deleted file mode 100644 index 5d471b3d8..000000000 --- a/govtool/status-service/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -blinker==1.7.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -Flask==3.0.0 -gunicorn==21.2.0 -idna==3.4 -itsdangerous==2.1.2 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -packaging==23.2 -requests==2.31.0 -urllib3==2.1.0 -Werkzeug==3.0.1 diff --git a/govtool/status-service/status-service.mk b/govtool/status-service/status-service.mk deleted file mode 100644 index 1337f776a..000000000 --- a/govtool/status-service/status-service.mk +++ /dev/null @@ -1,21 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-status-service - -# image tags -status_service_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/status-service) - -.PHONY: build-status-service -build-status-service: docker-login - $(call check_image_on_ecr,status-service,$(status_service_image_tag)) || \ - $(docker) build --tag "$(repo_url)/status-service:$(status_service_image_tag)" \ - $(root_dir)/govtool/status-service - -.PHONY: push-status-service -push-status-service: build-status-service - $(call check_image_on_ecr,status-service,$(status_service_image_tag)) || \ - $(docker) push $(repo_url)/status-service:$(status_service_image_tag) diff --git a/govtool/status-service/templates/status.html b/govtool/status-service/templates/status.html deleted file mode 100644 index fb33dff10..000000000 --- a/govtool/status-service/templates/status.html +++ /dev/null @@ -1,14 +0,0 @@ - - - - - VVA System Status - - -
-

cardano-node: {{ cardano_node_state }}

-

cardano-db-sync: {{ cardano_db_sync_state }}

-

backend: {{ backend_state }}

-
- - diff --git a/infra/terraform/.envrc b/infra/terraform/.envrc deleted file mode 100644 index a5c07fb88..000000000 --- a/infra/terraform/.envrc +++ /dev/null @@ -1,3 +0,0 @@ -source_up -env_vars_required AWS_PROFILE -use flake --extra-experimental-features nix-command --extra-experimental-features flakes ../..#infra diff --git a/infra/terraform/README.md b/infra/terraform/README.md deleted file mode 100644 index 0cfd562c8..000000000 --- a/infra/terraform/README.md +++ /dev/null @@ -1,99 +0,0 @@ -# Terraform Infrastructure for GovTool - -This repository contains Terraform configurations for managing cloud resources -for the GovTool project, focusing on AWS. - -## Architecture Overview - -The infrastructure is designed to support multiple environments and components -across the development lifecycle. - -### Components - -- **IAM Groups and Policies**: Manage access and permissions for CI/CD - processes. -- **DNS Management**: Route 53 is used to handle DNS configurations for various - services. -- **ECR Repositories**: Host Docker images for backend, frontend, and services. -- **EC2 Instances**: Support development, testing, staging, and beta - environments. - -## Managed Resources - -This project configures and manages several AWS resources essential for the -operation and deployment of the GovTool application across various environments. - -### IAM Configuration - -- **IAM Group (`CICD`)**: Manages CI/CD permissions. -- **IAM Policy (`CICD_ECR`)**: Attached to the `CICD` group to manage ECR - access. - -### DNS Management - -- **Route 53 Zone (`govtool.byron.network`)**: Handles DNS configurations and - routing for the application. - -### Container Registry - -- **ECR Repositories**: - - `backend`: Stores backend service images. - - `backend-base`: Stores base images for backend development. - - `frontend`: Stores frontend service images. - - `status-service`: Stores images for the status monitoring service. - - `metadata-validation`: Stores images for metadata validation service. - - `analytics-dashboard`: Stores images for analytics dashboard. - -### Compute Resources - -- **EC2 Instances**: - - `dev`: Instance for development. - - `test`: Instance for testing. - - `staging`: Instance for staging. - - `beta`: Instance for beta testing. - -## Configuration Files - -- `main.tf`: Defines the provider, resources, and modules for infrastructure - management. -- `versions.tf`: Specifies the required versions for Terraform and providers, - ensuring compatibility. - -## Managing Infrastructure - -### Prerequisites - -- AWS CLI -- Configured AWS credentials -- Terraform CLI (version specified in `versions.tf`) - -### Initialization - -Initialize the Terraform environment to begin managing the infrastructure: - -```bash -terraform init -terraform plan -terraform apply -``` - -### Adding or Modifying Components - -When adding new resources or modifying existing configurations, ensure to follow -the modular structure and update the respective environment settings as -required. - -## Continuous Integration and Deployment - -IAM configurations and ECR policies are tailored to support CI/CD pipelines, -allowing automated build and deployment processes. - -## Outputs - -Terraform outputs are configured to provide URLs and domain names for the -deployed services, facilitating easy access and management. - -## Contributions - -Contributions are welcome. Please ensure to maintain the modular architecture -and follow best practices for cloud resource management. diff --git a/infra/terraform/bootstrap-aws-account.sh b/infra/terraform/bootstrap-aws-account.sh deleted file mode 100755 index 0e9307c07..000000000 --- a/infra/terraform/bootstrap-aws-account.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -state_bucket="vva-terraform-state" -lock_table="vva-terraform-locks" -region="eu-west-1" - -# aws s3api create-bucket --bucket "$state_bucket" --region "$region" -aws s3 mb s3://"$state_bucket" --region "$region" -aws s3api put-bucket-versioning --bucket "$state_bucket" --versioning-configuration Status=Enabled -aws s3api get-bucket-versioning --bucket "$state_bucket" -aws dynamodb describe-table --table-name "$lock_table" -if [ $? -ne 0 ]; then - aws dynamodb create-table \ - --region "$region" \ - --table-name "$lock_table" \ - --attribute-definitions AttributeName=LockID,AttributeType=S \ - --key-schema AttributeName=LockID,KeyType=HASH \ - --billing-mode PAY_PER_REQUEST -fi diff --git a/infra/terraform/default.nix b/infra/terraform/default.nix deleted file mode 100644 index b69f64e6b..000000000 --- a/infra/terraform/default.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ pkgs ? import { } }: -pkgs.mkShell { - packages = with pkgs; [ awscli2 terraform ]; - shellHook = '' - function warn() { tput setaf $2; echo "$1"; tput sgr0; } - - tput bold - warn "Welcome to GovTool!" 4 - warn "This is an infrastructure shell." 4 - warn "Read the infra/terraform/README.md to get more info about the deployment processes." 8 - ''; -} diff --git a/infra/terraform/main.tf b/infra/terraform/main.tf deleted file mode 100644 index 6349816bf..000000000 --- a/infra/terraform/main.tf +++ /dev/null @@ -1,162 +0,0 @@ -provider "aws" { - region = "eu-west-1" -} - -resource "aws_iam_group" "cicd" { - name = "CICD" - path = "/" -} - -resource "aws_route53_zone" "govtool" { - name = "govtool.byron.network." -} - -module "govtool-ecr-backend" { - source = "./modules/ecr" - repo_name = "backend" -} - -module "govtool-ecr-backend-base" { - source = "./modules/ecr" - repo_name = "backend-base" -} - -module "govtool-ecr-frontend" { - source = "./modules/ecr" - repo_name = "frontend" -} - -module "govtool-ecr-status-service" { - source = "./modules/ecr" - repo_name = "status-service" -} - -module "govtool-ecr-metadata-validation" { - source = "./modules/ecr" - repo_name = "metadata-validation" -} - -module "govtool-ecr-analytics-dashboard" { - source = "./modules/ecr" - repo_name = "analytics-dashboard" -} - -module "govtool-ecr-custom-cardano-db-sync" { - source = "./modules/ecr" - repo_name = "custom-cardano-db-sync" -} - -resource "aws_iam_policy" "cicd_ecr" { - name = "CICD_ECR" - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Action = [ - "ecr:*" - ] - Effect = "Allow" - Resource = [ - module.govtool-ecr-backend.repo_arn, - module.govtool-ecr-backend-base.repo_arn, - module.govtool-ecr-frontend.repo_arn, - module.govtool-ecr-status-service.repo_arn, - module.govtool-ecr-metadata-validation.repo_arn, - module.govtool-ecr-analytics-dashboard.repo_arn, - module.govtool-ecr-custom-cardano-db-sync.repo_arn - ] - }, - { - Action = "ecr:GetAuthorizationToken" - Effect = "Allow" - Resource = [ - "*" - ] - } - ] - }) -} - -resource "aws_iam_group_policy_attachment" "cicd" { - group = aws_iam_group.cicd.name - policy_arn = aws_iam_policy.cicd_ecr.arn -} - -# duplicate the following block in order to prepare a new environment -# make sure that app_env/cardano_network variable pair is unique - -module "govtool-dev-sanchonet" { - source = "./modules/govtool-ec2" - app_env = "dev" - cardano_network = "sanchonet" - instance_type = "t3.large" - dns_zone_id = aws_route53_zone.govtool.id -} - -module "govtool-test-sanchonet" { - source = "./modules/govtool-ec2" - app_env = "test" - cardano_network = "sanchonet" - instance_type = "t3.large" - dns_zone_id = aws_route53_zone.govtool.id -} - -module "govtool-staging-sanchonet" { - source = "./modules/govtool-ec2" - app_env = "staging" - cardano_network = "sanchonet" - instance_type = "t3.large" - dns_zone_id = aws_route53_zone.govtool.id - custom_subdomain = "staging" -} - -module "govtool-beta-sanchonet" { - source = "./modules/govtool-ec2" - app_env = "beta" - cardano_network = "sanchonet" - instance_type = "t3.large" -} - -output "govtool-ecr-backend-url" { - value = module.govtool-ecr-backend.repo_url -} - -output "govtool-ecr-backend-base-url" { - value = module.govtool-ecr-backend-base.repo_url -} - -output "govtool-ecr-frontend-url" { - value = module.govtool-ecr-frontend.repo_url -} - -output "govtool-ecr-status-service-url" { - value = module.govtool-ecr-status-service.repo_url -} - -output "govtool-ecr-metadata-validation-url" { - value = module.govtool-ecr-metadata-validation.repo_url -} - -output "govtool-ecr-analytics-dashboard-url" { - value = module.govtool-ecr-analytics-dashboard.repo_url -} - -output "govtool-ecr-custom-cardano-db-sync-url" { - value = module.govtool-ecr-custom-cardano-db-sync.repo_url -} - -output "govtool-dev-sanchonet-frontend-domain" { - value = module.govtool-dev-sanchonet.frontend_domain -} - -output "govtool-test-sanchonet-frontend-domain" { - value = module.govtool-test-sanchonet.frontend_domain -} - -output "govtool-staging-sanchonet-frontend-domain" { - value = module.govtool-staging-sanchonet.frontend_domain -} - -output "govtool-beta-sanchonet-frontend-domain" { - value = module.govtool-beta-sanchonet.frontend_domain -} diff --git a/infra/terraform/modules/ecr/main.tf b/infra/terraform/modules/ecr/main.tf deleted file mode 100644 index 69940b0f4..000000000 --- a/infra/terraform/modules/ecr/main.tf +++ /dev/null @@ -1,11 +0,0 @@ -resource "aws_ecr_repository" "ecr_repo" { - name = var.repo_name - - encryption_configuration { - encryption_type = "KMS" - } - - tags = { - app_name = var.app_name - } -} diff --git a/infra/terraform/modules/ecr/outputs.tf b/infra/terraform/modules/ecr/outputs.tf deleted file mode 100644 index 50d112224..000000000 --- a/infra/terraform/modules/ecr/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "repo_arn" { - value = aws_ecr_repository.ecr_repo.arn -} - -output "repo_url" { - value = aws_ecr_repository.ecr_repo.repository_url -} diff --git a/infra/terraform/modules/ecr/variables.tf b/infra/terraform/modules/ecr/variables.tf deleted file mode 100644 index 33fcb170f..000000000 --- a/infra/terraform/modules/ecr/variables.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "app_name" { - default = "govtool" -} -variable "repo_name" {} diff --git a/infra/terraform/modules/govtool-ec2/main.tf b/infra/terraform/modules/govtool-ec2/main.tf deleted file mode 100644 index bddf37ce7..000000000 --- a/infra/terraform/modules/govtool-ec2/main.tf +++ /dev/null @@ -1,204 +0,0 @@ -resource "aws_vpc" "vpc" { - cidr_block = "172.30.0.0/16" - enable_dns_hostnames = "true" - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_internet_gateway" "igw" { - vpc_id = aws_vpc.vpc.id - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_route_table" "rt" { - vpc_id = aws_vpc.vpc.id - - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.igw.id - } - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = "${var.app_name}" - app_env = "${var.app_env}" - cardano_network = var.cardano_network - } -} - -resource "aws_subnet" "sub01" { - vpc_id = aws_vpc.vpc.id - cidr_block = "172.30.0.0/20" - availability_zone = "eu-west-1a" - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_route_table_association" "rta" { - subnet_id = aws_subnet.sub01.id - route_table_id = aws_route_table.rt.id -} - -resource "aws_security_group" "out_sg" { - name = "out" - description = "Allow outbound traffic" - - vpc_id = aws_vpc.vpc.id - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - ipv6_cidr_blocks = ["::/0"] - } - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_security_group" "ssh_sg" { - name = "ssh" - description = "Allow inbound traffic on port 22" - - vpc_id = aws_vpc.vpc.id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - description = "BinarApps Office" - cidr_blocks = ["185.48.176.106/32"] - } - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - description = "BinarApps a.guderski" - cidr_blocks = ["85.89.169.166/32"] - } - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - description = "public internet, replace with github actions ip in the future" - cidr_blocks = ["0.0.0.0/0"] - } - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_security_group" "web_sg" { - name = "web" - description = "Allow inbound traffic on ports 80 and 443" - - vpc_id = aws_vpc.vpc.id - - ingress { - from_port = 80 - to_port = 80 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - ingress { - from_port = 443 - to_port = 443 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_instance" "govtool" { - ami = var.ami - instance_type = var.instance_type - - root_block_device { - volume_size = var.volume_size - volume_type = var.volume_type - } - - user_data = file("${path.module}/user_data.sh") - user_data_replace_on_change = false - - credit_specification { - cpu_credits = "unlimited" - } - - subnet_id = aws_subnet.sub01.id - vpc_security_group_ids = [ - aws_security_group.out_sg.id, - aws_security_group.ssh_sg.id, - aws_security_group.web_sg.id - ] - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_eip" "govtool" { - instance = aws_instance.govtool.id - - tags = { - Name = "${var.app_name}_${var.app_env}_${var.cardano_network}" - app_name = var.app_name - app_env = var.app_env - cardano_network = var.cardano_network - } -} - -resource "aws_route53_record" "frontend" { - count = var.app_env == "beta" ? 0 : 1 - zone_id = var.dns_zone_id - name = "${var.custom_subdomain != "" ? "${var.custom_subdomain}" : "${var.app_env}-${var.cardano_network}"}" - type = "A" - ttl = 180 - records = [aws_eip.govtool.public_ip] -} - -resource "aws_route53_record" "participation" { - count = var.app_env == "beta" ? 0 : 1 - zone_id = var.dns_zone_id - name = "${var.custom_subdomain != "" ? "participation.${var.custom_subdomain}" : "participation.${var.app_env}-${var.cardano_network}"}" - type = "A" - ttl = 180 - records = [aws_eip.govtool.public_ip] -} \ No newline at end of file diff --git a/infra/terraform/modules/govtool-ec2/outputs.tf b/infra/terraform/modules/govtool-ec2/outputs.tf deleted file mode 100644 index 99a2f8920..000000000 --- a/infra/terraform/modules/govtool-ec2/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "govtool_eip" { - value = aws_eip.govtool.public_ip -} - -output "frontend_domain" { - value = var.app_env == "beta" ? aws_eip.govtool.public_ip : aws_route53_record.frontend[0].fqdn -} diff --git a/infra/terraform/modules/govtool-ec2/user_data.sh b/infra/terraform/modules/govtool-ec2/user_data.sh deleted file mode 100644 index 0e47ac34a..000000000 --- a/infra/terraform/modules/govtool-ec2/user_data.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -# setup ssh access -mkdir -p /home/ubuntu/.ssh -users="a.guderski,michal.jankun,p.placzynski,michal.szalowski" -curl --retry 5 --retry-delay 5 -L keys.binarapps.com/ssh/{$users} | tee -a /home/ubuntu/.ssh/authorized_keys -echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKeM0HOF9szWhOfbQM8XkIfznORTtTaCJJStALYjQuy6 (voltaire-era-github-actions)" | tee -a /home/ubuntu/.ssh/authorized_keys -chmod 700 /home/ubuntu/.ssh -chmod 600 /home/ubuntu/.ssh/authorized_keys -chown ubuntu:ubuntu -R /home/ubuntu/.ssh - -# install docker -sudo apt-get update -sudo apt-get install -y ca-certificates curl gnupg -sudo install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -sudo chmod a+r /etc/apt/keyrings/docker.gpg -echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ - "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null -sudo apt-get update -sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin prometheus-node-exporter -sudo usermod -aG docker ubuntu - -# initialize deployment log -sudo touch /var/log/deployment.log -sudo chown ubuntu:ubuntu /var/log/deployment.log diff --git a/infra/terraform/modules/govtool-ec2/variables.tf b/infra/terraform/modules/govtool-ec2/variables.tf deleted file mode 100644 index 0f58d46b4..000000000 --- a/infra/terraform/modules/govtool-ec2/variables.tf +++ /dev/null @@ -1,26 +0,0 @@ -variable "ami" { - default = "ami-01dd271720c1ba44f" -} -variable "app_env" {} -variable "app_name" { - default = "govtool" -} -variable "cardano_network" {} -variable "custom_subdomain" { - default = "" -} -variable "dns_zone_id" { - default = "" -} -variable "instance_type" { - default = "t3.micro" -} -variable "region" { - default = "eu-west-1" -} -variable "volume_size" { - default = 400 -} -variable "volume_type" { - default = "gp3" -} diff --git a/infra/terraform/modules/govtool-ec2/vva_to_govtool_migration_20240212.tf b/infra/terraform/modules/govtool-ec2/vva_to_govtool_migration_20240212.tf deleted file mode 100644 index 27673a071..000000000 --- a/infra/terraform/modules/govtool-ec2/vva_to_govtool_migration_20240212.tf +++ /dev/null @@ -1,15 +0,0 @@ -# FIXME This Terraform file in question is designed to migrate resource and -# module names, serving as a transitional tool to ensure consistent naming -# conventions across the infrastructure. Once it is confirmed that all users -# have applied this migration, the file can be safely removed in the future to -# maintain clarity and efficiency in the codebase. - -moved { - from = aws_instance.vva - to = aws_instance.govtool -} - -moved { - from = aws_eip.vva - to = aws_eip.govtool -} diff --git a/infra/terraform/versions.tf b/infra/terraform/versions.tf deleted file mode 100644 index d3ce0ef4b..000000000 --- a/infra/terraform/versions.tf +++ /dev/null @@ -1,16 +0,0 @@ -terraform { - backend "s3" { - bucket = "vva-terraform-state" - key = "terraform.tfstate" - region = "eu-west-1" - dynamodb_table = "vva-terraform-locks" - encrypt = true - } - required_version = ">= 1.5.3" - required_providers { - aws = { - source = "hashicorp/aws" - version = "~> 5.0" - } - } -} diff --git a/infra/terraform/vva_to_govtool_migration_20240212.tf b/infra/terraform/vva_to_govtool_migration_20240212.tf deleted file mode 100644 index 067ea4d77..000000000 --- a/infra/terraform/vva_to_govtool_migration_20240212.tf +++ /dev/null @@ -1,35 +0,0 @@ -# FIXME This Terraform file in question is designed to migrate resource and -# module names, serving as a transitional tool to ensure consistent naming -# conventions across the infrastructure. Once it is confirmed that all users -# have applied this migration, the file can be safely removed in the future to -# maintain clarity and efficiency in the codebase. - -moved { - from = module.vva-ecr-be - to = module.govtool-ecr-backend -} - -moved { - from = module.vva-ecr-fe - to = module.govtool-ecr-frontend -} - -moved { - from = module.vva-dev-sanchonet - to = module.govtool-dev-sanchonet -} - -moved { - from = module.vva-test-sanchonet - to = module.govtool-test-sanchonet -} - -moved { - from = module.vva-staging-sanchonet - to = module.govtool-staging-sanchonet -} - -moved { - from = module.vva-beta-sanchonet - to = module.govtool-beta-sanchonet -} diff --git a/scripts/govtool/.envrc b/scripts/govtool/.envrc deleted file mode 100644 index fa6f7d8f1..000000000 --- a/scripts/govtool/.envrc +++ /dev/null @@ -1,16 +0,0 @@ -source_up -env_vars_required ENVIRONMENT AWS_PROFILE CARDANO_NETWORK DBSYNC_POSTGRES_USER DBSYNC_POSTGRES_PASSWORD DBSYNC_POSTGRES_DB TRAEFIK_LE_EMAIL GTM_ID SENTRY_DSN_BACKEND SENTRY_DSN SENTRY_ENVIRONMENT GRAFANA_ADMIN_PASSWORD GRAFANA_SLACK_RECIPIENT GRAFANA_SLACK_OAUTH_TOKEN IP_ADDRESS_BYPASSING_BASIC_AUTH1 IP_ADDRESS_BYPASSING_BASIC_AUTH2 - -case "$ENVIRONMENT" in - "dev") - env_vars_required DEV_NGINX_BASIC_AUTH - ;; - "test") - env_vars_required TEST_NGINX_BASIC_AUTH - ;; - "staging") - env_vars_required STAGING_NGINX_BASIC_AUTH - ;; -esac - -use flake --extra-experimental-features nix-command --extra-experimental-features flakes ../..#scripts diff --git a/scripts/govtool/Makefile b/scripts/govtool/Makefile deleted file mode 100644 index b17b3b2a0..000000000 --- a/scripts/govtool/Makefile +++ /dev/null @@ -1,76 +0,0 @@ -include backend.mk -include frontend.mk -include status-service.mk -include metadata-validation.mk -include analytics-dashboard.mk -include custom-cardano-db-sync.mk -include utils.mk -include info.mk -include config.mk - -.DEFAULT_GOAL := info - -# image tags -cardano_node_image_tag := 10.1.4 -cardano_db_sync_image_tag := 13.6.0.4 - -.PHONY: all -all: deploy-stack notify - -.PHONY: deploy-stack -deploy-stack: upload-config push-backend push-frontend push-status-service push-metadata-validation push-analytics-dashboard push-custom-cardano-db-sync - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - export CARDANO_NETWORK=$(cardano_network); \ - export DOCKER_HOST=ssh://$(ssh_url); \ - export ENVIRONMENT=$(env); \ - export GRAFANA_ADMIN_PASSWORD=$${GRAFANA_ADMIN_PASSWORD}; \ - export BACKEND_TAG=$(backend_image_tag); \ - export FRONTEND_TAG=$(frontend_image_tag); \ - export STATUS_SERVICE_TAG=$(status_service_image_tag); \ - export METADATA_VALIDATION_TAG=$(metadata_validation_image_tag); \ - export ANALYTICS_DASHBOARD_TAG=$(analytics_dashboard_image_tag); \ - export CARDANO_NODE_TAG=$(cardano_node_image_tag); \ - export CARDANO_DB_SYNC_TAG=$(cardano_db_sync_image_tag); \ - $(ssh-keyscan) $(docker_host) 2>/dev/null >> ~/.ssh/known_hosts; \ - $(docker) compose -f $(docker_compose_file) -p $(compose_stack_name) up --detach --build --pull always --remove-orphans - -.PHONY: destroy-cardano-node-and-dbsync -destroy-cardano-node-and-dbsync: prepare-config - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - export CARDANO_NETWORK=$(cardano_network); \ - export DOCKER_HOST=ssh://$(ssh_url); \ - export ENVIRONMENT=$(env); \ - export BACKEND_TAG=$(backend_image_tag); \ - export FRONTEND_TAG=$(frontend_image_tag); \ - export STATUS_SERVICE_TAG=$(status_service_image_tag); \ - export METADATA_VALIDATION_TAG=$(metadata_validation_image_tag); \ - export ANALYTICS_DASHBOARD_TAG=$(analytics_dashboard_image_tag); \ - export CARDANO_NODE_TAG=$(cardano_node_image_tag); \ - export CARDANO_DB_SYNC_TAG=$(cardano_db_sync_image_tag); \ - $(ssh-keyscan) $(docker_host) 2>/dev/null >> ~/.ssh/known_hosts; \ - containers="$$(docker container ls --no-trunc --format '{{.Names}}' -f name=cardano-node -f name=cardano-db-sync -f name=postgres)"; \ - volumes="$$(docker volume ls --format '{{.Name}}' -f name=db-sync-data -f name=node-db -f name=node-ipc -f name=postgres)"; \ - $(docker) container stop $${containers}; \ - $(docker) container rm $${containers}; \ - $(docker) volume rm $${volumes} - -.PHONY: toggle-maintenance -toggle-maintenance: docker-login - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - @:$(call check_defined, maintenance) - export ENVIRONMENT=$(env); \ - export CARDANO_NETWORK=$(cardano_network); \ - export DOCKER_HOST=ssh://$(ssh_url); \ - export BACKEND_TAG=$(backend_image_tag); \ - export FRONTEND_TAG=$(frontend_image_tag); \ - export STATUS_SERVICE_TAG=$(status_service_image_tag); \ - export METADATA_VALIDATION_TAG=$(metadata_validation_image_tag); \ - export ANALYTICS_DASHBOARD_TAG=$(analytics_dashboard_image_tag); \ - export CARDANO_NODE_TAG=$(cardano_node_image_tag); \ - export CARDANO_DB_SYNC_TAG=$(cardano_db_sync_image_tag); \ - $(ssh-keyscan) $(docker_host) 2>/dev/null >> ~/.ssh/known_hosts; \ - if [[ "$(maintenance)" = "enabled" ]]; then $(docker) compose -f $(docker_compose_file) -p $(compose_stack_name) exec frontend touch /var/run/maintenance_enabled; \ - else $(docker) compose -f $(docker_compose_file) -p $(compose_stack_name) exec frontend rm /var/run/maintenance_enabled; fi diff --git a/scripts/govtool/README.md b/scripts/govtool/README.md deleted file mode 100644 index 914633805..000000000 --- a/scripts/govtool/README.md +++ /dev/null @@ -1,183 +0,0 @@ -# GovTool deployment stack - -Welcome to the GovTool deployment stack documentation. This guide provides an -overview of the deployment process, including the setup and management of -various environments, the use of Docker Compose for service orchestration, Nix -for tooling, and detailed steps for deploying the GovTool stack. - -This README aims to provide clear and concise instructions for the deployment of -the GovTool stack. Should you have any questions or require further -clarification, please consult the development team. - -## Environments - -The GovTool project supports four distinct environments, each tailored to -specific stages of the development and deployment lifecycle: - -- **dev**: Designed for developers, this environment facilitates development and - testing of new features. -- **test**: Dedicated to QA testers, enabling thorough testing and validation of - software before it moves to the next stage. -- **staging**: Acts as a pre-production environment, allowing for final checks - and adjustments. -- **beta**: Serves as the current production environment, where the software is - available to end-users. - -## Docker Compose files - -Docker Compose is utilized to manage per-environment services setups -effectively. This includes configurations for: - -- Services such as Cardano Node, Cardano DB Sync, GovTool backend, GovTool - frontend, the metadata validation service and an analytics dashboard. -- Network configuration with Traefik to handle requests efficiently. -- Monitoring solutions with Prometheus and Grafana to ensure optimal performance - and availability. - -Each environment has its own Docker Compose file, generated from the template, -enabling tailored setups that meet specific requirements. - -## Context - -Deployment context, including environment variables, is managed through `.env` -files. For an example configuration, refer to the `.env.example` file. - -## Deployment process - -The deployment process is automated using a system of Makefiles, which include -several targets to streamline the deployment steps: - -### [Deployment `Makefile`](./Makefile) - -This is the main Makefile that provides full support for the deployment and -maintenance of the stack. It includes backend and frontend modules and uses -particular Makefiles for additional purposes. - -#### `prepare-config` - -Prepares the configuration files required for the application. This involves -generating or fetching files, then placing them in the appropriate directory -structure. Key components include: - -- `cardano-node` Cardano Node configuration files, with modifications for - Prometheus. -- `dbsync-secrets` for database credentials used by Cardano DB Sync. -- `backend-config.json` for backend configurations. -- `prometheus.yml` and `grafana-provisioning` for monitoring setup. -- `nginx` configuration for basic authentication where necessary. - -#### `upload-config` - -Uploads the generated configuration to the target server. - -#### `deploy-stack` - -Updates the target server with the latest images and (re)launches the Docker -Compose stack. - -#### `destroy-cardano-node-and-dbsync` - -Removes the Cardano Node and Cardano DB Sync services, purges their volumes, and -removes PostgreSQL database files. This step ensures that the Cardano network's -state has been fully removed so the blockchain will be synchronized from the -beginning. - -#### `toggle-maintenance` - -This step turns on the "maintenance page" that blocks users from usage of the -application and provides an explanation of that in form of the simple page with -error message. - -### Helpers, variables and targets in [`common.mk`](./common.mk) file - -This Makefile defines some common variables and helpers, as well as highly -reused targets, such as: - -#### `check-env-defined` - -Verifies the given environment value. When there is no definition for such -environment in Terraform configuration it stops the execution. - -#### `docker-login` - -Ensures that the user is logged into the correct Docker account, providing -access to necessary Docker resources. - -### Notifications in [`info.mk`](./info.mk) file - -This file is responsible for giving the proper notifications and feedback about -the current execution: - -#### `info` - -Displays deployment parameters for review and verification. - -#### `notify` - -Sends a notification to stakeholders about the deployment status via Slack. - -### Building backend images steps in [`backend.mk`](./backend.mk) - -This file is responsible for generating images for backend and for pushing them -to the ECR repository. - -#### `build-backend` - -Builds the backend image locally and a base image for the backend when changes -are detected in the Cabal file. - -#### `push-backend` - -Pushes the backend and backend-base images to the Docker repository. - -### Building frontend images steps in [`frontend.mk`](./frontend.mk) - -This file is responsible for generating images for frontend and for pushing them -to the ECR repository. - -#### `build-frontend` - -Handles the construction of the frontend image. - -#### `push-frontend` - -Pushes the frontend image to the Docker repository. - -### Building status service images steps in [`status-service.mk`](./status-service.mk) - -This file is responsible for generating images for status service and for -pushing them to the ECR repository. - -#### `build-status-service` - -Handles the construction of the status service image. - -#### `push-status-service` - -Pushes the status service image to the Docker repository. - -### Building metadata validation service images steps in [`metadata-validation.mk`](./metadata-validation.mk) - -This file is responsible for generating images for metadata validation service -and for pushing them to the ECR repository. - -#### `build-metadata-validation` - -Handles the construction of the metadata validation service image. - -#### `push-metadata-validation` - -Pushes the metadata validation service image to the Docker repository. - -### Building analytics dashboard service images steps in [`analytics-dashboard.mk`](./analytics-dashboard.mk) - -This file is responsible for generating images for analytics dashboard service -and for pushing them to the ECR repository. - -#### `build-analytics-dashboard` - -Handles the construction of the analytics dashboard service image. - -#### `push-analytics-dashboard` - -Pushes the analytics dashboard service image to the Docker repository. diff --git a/scripts/govtool/analytics-dashboard.mk b/scripts/govtool/analytics-dashboard.mk deleted file mode 100644 index 59dae3974..000000000 --- a/scripts/govtool/analytics-dashboard.mk +++ /dev/null @@ -1,22 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-analytics-dashboard - -# image tags -analytics_dashboard_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/analytics-dashboard)-$(env) - -.PHONY: build-analytics-dashboard -build-analytics-dashboard: - $(call check_image_on_ecr,analytics-dashboard,$(analytics_dashboard_image_tag)) || \ - $(docker) build --tag "$(repo_url)/analytics-dashboard:$(analytics_dashboard_image_tag)" \ - --build-arg NEXT_PUBLIC_API_URL="$${NEXT_PUBLIC_API_URL}" \ - $(root_dir)/govtool/analytics-dashboard - -.PHONY: push-analytics-dashboard -push-analytics-dashboard: build-analytics-dashboard - $(call check_image_on_ecr,analytics-dashboard,$(analytics_dashboard_image_tag)) || \ - $(docker) push $(repo_url)/analytics-dashboard:$(analytics_dashboard_image_tag) diff --git a/scripts/govtool/backend.mk b/scripts/govtool/backend.mk deleted file mode 100644 index a523973dd..000000000 --- a/scripts/govtool/backend.mk +++ /dev/null @@ -1,31 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-backend - -# image tags -base_backend_image_tag := $(shell git hash-object $(root_dir)/govtool/backend/vva-be.cabal) -backend_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/backend) - -.PHONY: build-backend-base -build-backend-base: docker-login - $(call check_image_on_ecr,backend-base,$(base_backend_image_tag)) || \ - $(docker) build --file $(root_dir)/govtool/backend/Dockerfile.base --tag "$(repo_url)/backend-base:$(base_backend_image_tag)" $(root_dir)/govtool/backend - -.PHONY: push-backend-base -push-backend-base: build-backend-base - $(call check_image_on_ecr,backend-base,$(base_backend_image_tag)) || \ - $(docker) push $(repo_url)/backend-base:$(base_backend_image_tag) - -.PHONY: build-backend -build-backend: build-backend-base - $(call check_image_on_ecr,backend,$(backend_image_tag)) || \ - $(docker) build --build-arg BASE_IMAGE_TAG=$(base_backend_image_tag) --tag "$(repo_url)/backend:$(backend_image_tag)" $(root_dir)/govtool/backend - -.PHONY: push-backend -push-backend: push-backend-base build-backend - $(call check_image_on_ecr,backend,$(backend_image_tag)) || \ - $(docker) push $(repo_url)/backend:$(backend_image_tag) diff --git a/scripts/govtool/common.mk b/scripts/govtool/common.mk deleted file mode 100644 index 77cd61747..000000000 --- a/scripts/govtool/common.mk +++ /dev/null @@ -1,65 +0,0 @@ -SHELL := bash -ONESHELL: -.SHELLFLAGS := -eu -o pipefail -c - -# tools -docker ?= docker -curl ?= curl -rsync ?= rsync -ssh-keyscan ?= ssh-keyscan - -# environment variables -env := $(shell echo $${ENVIRONMENT}) -cardano_network := $(shell echo $${CARDANO_NETWORK}) - -# git state -commit := $(shell git rev-parse HEAD) -branch := $(shell git rev-parse --abbrev-ref HEAD) -root_dir := $(shell git rev-parse --show-toplevel) - -# target addresses -domain := $(shell echo $${DOMAIN}) -repo_url ?= 733019650473.dkr.ecr.eu-west-1.amazonaws.com -docker_host := $(domain) -docker_user := ubuntu -ssh_url := $(docker_user)@$(docker_host) - -# stack configuration -compose_stack_name := govtool-$(env)-$(cardano_network) - -# helper function for checking undefined variables -check_defined = \ - $(strip $(foreach 1,$1, \ - $(call __check_defined,$1,$(strip $(value 2))))) -__check_defined = \ - $(if $(value $1),, \ - $(error Undefined $1$(if $2, ($2)))) - -force_rebuild := $(shell echo $${FORCE_REBUILD:-false}) -# helper function for checking if image exists on ECR -check_image_on_ecr = \ - if [ "$(force_rebuild)" = "true" ]; then \ - false; \ - else \ - $(docker) manifest inspect "$(repo_url)/$1:$2" > /dev/null 2>&1; \ - fi - -.PHONY: check-env-defined -check-env-defined: - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - @grep -q "module \"$(compose_stack_name)\"" ../../infra/terraform/main.tf && \ - echo "Environment $(env) for network $(cardano_network) is defined in Terraform" || \ - { echo "Environment $(env) for network $(cardano_network) is NOT defined in Terraform, cannot deploy there"; exit 1; } - -.PHONY: ssh-client-setup -ssh-client-setup: - if [[ "$${CI}" == "true" ]]; then mkdir -p ~/.ssh; sed -e "s/DOCKER_HOST/$(docker_host)/g" ./config/templates/ssh-client-config > ~/.ssh/config; fi - -.PHONY: docker-login -docker-login: ssh-client-setup - aws ecr get-login-password --region eu-west-1 | $(docker) login --username AWS --password-stdin $(repo_url)/analytics-dashboard - aws ecr get-login-password --region eu-west-1 | $(docker) login --username AWS --password-stdin $(repo_url)/backend - aws ecr get-login-password --region eu-west-1 | $(docker) login --username AWS --password-stdin $(repo_url)/frontend - aws ecr get-login-password --region eu-west-1 | $(docker) login --username AWS --password-stdin $(repo_url)/metadata-validation - aws ecr get-login-password --region eu-west-1 | $(docker) login --username AWS --password-stdin $(repo_url)/status-service diff --git a/scripts/govtool/config.mk b/scripts/govtool/config.mk deleted file mode 100644 index a10faad12..000000000 --- a/scripts/govtool/config.mk +++ /dev/null @@ -1,129 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -# metadata -cardano_config_provider := https://book.world.dev.cardano.org - -# directories -config_dir := $(root_dir)/scripts/govtool/config -template_config_dir := $(config_dir)/templates -target_config_dir := $(config_dir)/target - -# files -docker_compose_file := $(target_config_dir)/docker-compose.yml -cardano_configs := alonzo-genesis byron-genesis conway-genesis shelley-genesis submit-api-config topology -cardano_config_files := $(addprefix $(target_config_dir)/cardano-node/,$(addsuffix .json,$(cardano_configs))) -outputs := cardano-node/config.json \ - cardano-node/db-sync-config.json \ - dbsync-secrets/postgres_user \ - dbsync-secrets/postgres_db \ - dbsync-secrets/postgres_password \ - backend-config.json prometheus.yml \ - promtail.yml \ - loki.yml \ - nginx/auth.conf \ - nginx/govtool.htpasswd \ - grafana-provisioning/alerting/alerting.yml \ - grafana-provisioning/datasources/datasource.yml \ - grafana-provisioning/dashboards/dashboard.yml \ - grafana-provisioning/dashboards/govtool.json \ - grafana-provisioning/dashboards/traefik_rev4.json - -output_files := $(addprefix $(target_config_dir)/,$(outputs)) $(cardano_config_files) $(docker_compose_file) -output_dirs := $(sort $(foreach file,$(output_files),$(dir $(file)))) - -.PHONY: prepare-config -prepare-config: clear $(output_files) - -.PHONY: upload-config -upload-config: check-env-defined prepare-config - @:$(call check_defined, ssh_url) - $(rsync) -av -e 'ssh -o StrictHostKeyChecking=no' config/target/. $(ssh_url):config - -.PHONY: clear -clear: - rm -rf $(target_config_dir) - -.SECONDEXPANSION: -$(output_dirs): - mkdir -p $@ - -$(docker_compose_file): $(template_config_dir)/docker-compose.yml.tpl $(target_config_dir)/ - if [[ "$(env)" == "dev" ]]; then CSP_ALLOWED_HOSTS=",http://localhost,http://localhost:5173"; else CSP_ALLOWED_HOSTS=; fi; \ - sed -e "s||$(domain)|g" \ - -e "s||$(docker_user)|g" \ - -e "s||$(repo_url)|g" \ - -e "s||$${CSP_ALLOWED_HOSTS}|g" \ - -e "s||$${PDF_API_URL}|g" \ - $< > $@ - -$(cardano_config_files): $(target_config_dir)/cardano-node/ - @:$(call check_defined, cardano_network) - $(curl) -s "$(cardano_config_provider)/environments/$(cardano_network)/$(notdir $@)" -o $@ - -$(target_config_dir)/cardano-node/config.json: $(target_config_dir)/cardano-node/ - $(curl) -s "$(cardano_config_provider)/environments/$(cardano_network)/$(notdir $@)" -o $@ - sed -i '/"hasPrometheus"/ { N; s/"127\.0\.0\.1"/"0.0.0.0"/ }' "$(target_config_dir)/cardano-node/config.json" - -$(target_config_dir)/cardano-node/db-sync-config.json: $(target_config_dir)/cardano-node/ - $(curl) -s "$(cardano_config_provider)/environments/$(cardano_network)/$(notdir $@)" -o $@ - sed -i '1a\ "EnableFutureGenesis" : true,' "$(target_config_dir)/cardano-node/db-sync-config.json" - -$(target_config_dir)/dbsync-secrets/postgres_user: $(target_config_dir)/dbsync-secrets/ - echo "$${DBSYNC_POSTGRES_USER}" > $@ - -$(target_config_dir)/dbsync-secrets/postgres_password: $(target_config_dir)/dbsync-secrets/ - echo "$${DBSYNC_POSTGRES_PASSWORD}" > $@ - -$(target_config_dir)/dbsync-secrets/postgres_db: $(target_config_dir)/dbsync-secrets/ - echo "$${DBSYNC_POSTGRES_DB}" > $@ - -$(target_config_dir)/backend-config.json: $(config_dir)/templates/backend-config.json.tpl $(target_config_dir)/ - sed -e "s||$${DBSYNC_POSTGRES_DB}|" \ - -e "s||$${DBSYNC_POSTGRES_USER}|" \ - -e "s||$${DBSYNC_POSTGRES_PASSWORD}|" \ - -e "s||$${DBSYNC_POSTGRES_HOST}|" \ - -e "s||$${DBSYNC_POSTGRES_PORT}|" \ - -e "s||$${SENTRY_DSN_BACKEND}|" \ - -e "s||$(env)|" \ - $< > $@ - -$(target_config_dir)/%.yml: $(template_config_dir)/%.yml $(target_config_dir)/ - cp -a $< $@ - -$(target_config_dir)/grafana-provisioning/datasources/%: $(template_config_dir)/grafana-provisioning/datasources/% $(target_config_dir)/grafana-provisioning/datasources/ - cp -a $< $@ - -$(target_config_dir)/grafana-provisioning/dashboards/%: $(template_config_dir)/grafana-provisioning/dashboards/% $(target_config_dir)/grafana-provisioning/dashboards/ - cp -a $< $@ - -$(target_config_dir)/grafana-provisioning/alerting/alerting.yml: $(template_config_dir)/grafana-provisioning/alerting/alerting.yml $(target_config_dir)/grafana-provisioning/alerting/ - cp -a $< $@ - sed -e "s||$${GRAFANA_SLACK_RECIPIENT}|" \ - -e "s||$${GRAFANA_SLACK_OAUTH_TOKEN}|" \ - -i $@ - -$(target_config_dir)/nginx/auth.conf: $(target_config_dir)/nginx/ - @:$(call check_defined, env) - if [[ "$(env)" != "beta" ]]; then \ - echo 'map $$http_x_forwarded_for $$auth {' > $@; \ - echo " default \"Restricted\";" >> $@; \ - echo " $${IP_ADDRESS_BYPASSING_BASIC_AUTH1} \"off\";" >> $@; \ - echo " $${IP_ADDRESS_BYPASSING_BASIC_AUTH2} \"off\";" >> $@; \ - echo "}" >> $@; \ - echo 'auth_basic $$auth;' >> $@; \ - echo "auth_basic_user_file /etc/nginx/conf.d/govtool.htpasswd;" >> $@; \ - else \ - echo > $@; \ - fi - -$(target_config_dir)/nginx/govtool.htpasswd: $(target_config_dir)/nginx/ - @:$(call check_defined, env) - if [[ "$(env)" != "beta" ]]; then \ - echo "$${$(shell echo $(env) | tr a-z A-Z)_NGINX_BASIC_AUTH}" > $@; \ - else \ - echo > $@; \ - fi diff --git a/scripts/govtool/config/templates/backend-config.json.tpl b/scripts/govtool/config/templates/backend-config.json.tpl deleted file mode 100644 index a50b894f6..000000000 --- a/scripts/govtool/config/templates/backend-config.json.tpl +++ /dev/null @@ -1,14 +0,0 @@ -{ - "dbsyncconfig" : { - "host" : "", - "dbname" : "", - "user" : "", - "password" : "", - "port" : "" - }, - "port" : 9876, - "host" : "0.0.0.0", - "cachedurationseconds": 20, - "sentrydsn": "", - "sentryenv": "" -} diff --git a/scripts/govtool/config/templates/docker-compose.yml.tpl b/scripts/govtool/config/templates/docker-compose.yml.tpl deleted file mode 100644 index 80b64cafc..000000000 --- a/scripts/govtool/config/templates/docker-compose.yml.tpl +++ /dev/null @@ -1,308 +0,0 @@ -version: "3.9" - -services: - traefik: - image: traefik:v3.0 - command: - - "--providers.docker=true" - - "--providers.docker.exposedbydefault=false" - - "--entrypoints.web.address=:80" - - "--entrypoints.websecure.address=:443" - - "--certificatesresolvers.myresolver.acme.httpchallenge=true" - - "--certificatesresolvers.myresolver.acme.httpchallenge.entrypoint=web" - - "--certificatesresolvers.myresolver.acme.email=${TRAEFIK_LE_EMAIL}" - - "--certificatesresolvers.myresolver.acme.storage=/letsencrypt/acme.json" - - "--log.level=DEBUG" - - "--entryPoints.metrics.address=:8082" - - "--metrics.prometheus=true" - - "--metrics.prometheus.entryPoint=metrics" - - "--metrics.prometheus.buckets=0.1,0.3,1.2,5.0" - ports: - - 80:80 - - 443:443 - volumes: - - letsencrypt:/letsencrypt - - "/var/run/docker.sock:/var/run/docker.sock:ro" - restart: always - logging: &logging - driver: "json-file" - options: - max-size: "200k" - max-file: "10" - labels: - - "traefik.enable=true" - - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" - - "traefik.http.routers.to-http-catchall.rule=hostregexp(`{host:.+}`)" - - "traefik.http.routers.to-http-catchall.entrypoints=web" - - "traefik.http.routers.to-http-catchall.middlewares=redirect-to-https" - - loki: - image: grafana/loki:2.9.4 - ports: - - "3100:3100" - command: -config.file=/etc/loki/loki.yml - volumes: - - loki-data:/loki - - /home//config/loki.yml:/etc/loki/loki.yml - - promtail: - image: grafana/promtail:2.9.4 - volumes: - - /var/log/deployment.log:/var/log/deployment.log - - /home//config/promtail.yml:/etc/promtail/promtail.yml - command: -config.file=/etc/promtail/promtail.yml - depends_on: - - loki - - prometheus: - image: prom/prometheus:v2.47.1 - volumes: - - prometheus-data:/prometheus - - /home//config/prometheus.yml:/etc/prometheus/prometheus.yml - extra_hosts: - - "host.docker.internal:host-gateway" - restart: always - logging: *logging - - grafana: - image: grafana/grafana:10.0.8 - volumes: - - grafana-data:/var/lib/grafana - - /home//config/grafana-provisioning:/etc/grafana/provisioning - environment: - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} - - GF_USERS_ALLOW_SIGN_UP=false - - GF_INSTALL_PLUGINS=grafana-piechart-panel - - GF_SERVER_ROOT_URL=https:///grafana - - GF_SERVER_SERVE_FROM_SUB_PATH=true - restart: always - logging: *logging - labels: - - "traefik.enable=true" - - "traefik.http.routers.to-grafana.rule=Host(``) && PathPrefix(`/grafana`)" - - "traefik.http.routers.to-grafana.entrypoints=websecure" - - "traefik.http.routers.to-grafana.tls.certresolver=myresolver" - - "traefik.http.services.grafana.loadbalancer.server.port=3000" - - postgres: - image: postgres:15-alpine - environment: - - POSTGRES_LOGGING=true - - POSTGRES_DB_FILE=/run/secrets/postgres_db - - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password - - POSTGRES_USER_FILE=/run/secrets/postgres_user - secrets: - - postgres_password - - postgres_user - - postgres_db - volumes: - - postgres:/var/lib/postgresql/data - restart: always - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 10s - timeout: 5s - retries: 5 - command: ${POSTGRES_ARGS:--c maintenance_work_mem=1GB -c max_parallel_maintenance_workers=4} - logging: *logging - - cardano-node: - image: ghcr.io/intersectmbo/cardano-node:${CARDANO_NODE_TAG} - entrypoint: - - "cardano-node" - - "run" - - "--topology" - - "/configuration/topology.json" - - "--database-path" - - "/data/db" - - "--socket-path" - - "/ipc/node.socket" - - "--host-addr" - - "0.0.0.0" - - "--port" - - "3001" - - "--config" - - "/configuration/config.json" - - "+RTS" - - "-N2" - - "-I0" - - "-A16m" - - "-qg" - - "-qb" - - "--disable-delayed-os-memory-return" - - "-RTS" - environment: - - NETWORK=${CARDANO_NETWORK} - volumes: - - node-db:/data/db - - node-ipc:/ipc - - /home//config/cardano-node:/configuration - restart: always - healthcheck: - test: ["CMD-SHELL", "curl -f 127.0.0.1:12788 || exit 1"] - interval: 60s - timeout: 10s - retries: 5 - logging: *logging - - cardano-db-sync: - image: /custom-cardano-db-sync:latest - environment: - - NETWORK=${CARDANO_NETWORK} - - POSTGRES_HOST=postgres - - POSTGRES_PORT=5432 - - RESTORE_RECREATE_DB=N - - DB_SYNC_ENABLE_FUTURE_GENESIS=Y - depends_on: - cardano-node: - condition: service_healthy - postgres: - condition: service_healthy - secrets: - - postgres_password - - postgres_user - - postgres_db - volumes: - - db-sync-data:/var/lib/cexplorer - - node-ipc:/node-ipc - - /home//config/cardano-node:/configuration - restart: always - logging: *logging - - status-service: - image: /status-service:${STATUS_SERVICE_TAG} - environment: - - GRAFANA_USERNAME=admin - - GRAFANA_PASSWORD=${GRAFANA_ADMIN_PASSWORD} - restart: always - logging: *logging - labels: - - "traefik.enable=true" - - "traefik.http.routers.to-status-service.rule=Host(``) && PathPrefix(`/status`)" - - "traefik.http.routers.to-status-service.entrypoints=websecure" - - "traefik.http.routers.to-status-service.tls.certresolver=myresolver" - - "traefik.http.services.status-service.loadbalancer.server.port=8000" - - metadata-validation: - image: /metadata-validation:${METADATA_VALIDATION_TAG} - environment: - - PORT=3000 - - IPFS_GATEWAY=${IPFS_GATEWAY} - - IPFS_PROJECT_ID=${IPFS_PROJECT_ID} - logging: *logging - restart: always - healthcheck: - test: ["CMD-SHELL", "curl -f 127.0.0.1:3000/health || exit 1"] - interval: 5s - timeout: 5s - retries: 5 - labels: - - "traefik.enable=true" - - "traefik.http.middlewares.metadata-validation-stripprefix.stripprefix.prefixes=/metadata-validation" - - "traefik.http.middlewares.metadata-validation-cors.headers.accesscontrolallowmethods=*" - - "traefik.http.middlewares.metadata-validation-cors.headers.accesscontrolallowheaders=*" - - "traefik.http.middlewares.metadata-validation-cors.headers.accesscontrolalloworiginlist=https://" - - "traefik.http.middlewares.metadata-validation-cors.headers.accesscontrolmaxage=100" - - "traefik.http.routers.to-metadata-validation.rule=Host(``) && PathPrefix(`/metadata-validation`)" - - "traefik.http.routers.to-metadata-validation.middlewares=metadata-validation-stripprefix@docker,metadata-validation-cors@docker" - - "traefik.http.routers.to-metadata-validation.entrypoints=websecure" - - "traefik.http.routers.to-metadata-validation.tls.certresolver=myresolver" - - "traefik.http.services.metadata-validation.loadbalancer.server.port=3000" - - "traefik.http.services.metadata-validation.loadbalancer.healthcheck.path=/health" - - "traefik.http.services.metadata-validation.loadbalancer.healthcheck.port=3000" - - "traefik.http.services.metadata-validation.loadbalancer.healthcheck.interval=10s" - - "traefik.http.services.metadata-validation.loadbalancer.healthcheck.timeout=5s" - - analytics-dashboard: - image: /analytics-dashboard:${ANALYTICS_DASHBOARD_TAG} - environment: - - GA_CLIENT_EMAIL=${GA_CLIENT_EMAIL} - - GA_PRIVATE_KEY=${GA_PRIVATE_KEY} - - GOOGLE_APPLICATION_CREDENTIALS=/run/secrets/google-credentials.json - - NEXT_PUBLIC_API_URL=${NEXT_PUBLIC_API_URL} - - NEXT_PUBLIC_GA4_PROPERTY_ID=${NEXT_PUBLIC_GA4_PROPERTY_ID} - - SENTRY_IGNORE_API_RESOLUTION_ERROR=${SENTRY_IGNORE_API_RESOLUTION_ERROR} - secrets: - - google-credentials.json - logging: *logging - restart: always - labels: - - "traefik.enable=true" - - "traefik.http.routers.to-analytics-dashboard.rule=Host(`participation.sanchogov.tools`)" - - "traefik.http.routers.to-analytics-dashboard.entrypoints=websecure" - - "traefik.http.routers.to-analytics-dashboard.tls.certresolver=myresolver" - - "traefik.http.services.analytics-dashboard.loadbalancer.server.port=3000" - - backend: - image: /backend:${BACKEND_TAG} - command: /usr/local/bin/vva-be -c /run/secrets/backend-config.json start-app - depends_on: - cardano-node: - condition: service_healthy - postgres: - condition: service_healthy - secrets: - - backend-config.json - restart: always - logging: *logging - labels: - - "traefik.enable=true" - - "traefik.http.middlewares.backend-stripprefix.stripprefix.prefixes=/api" - - "traefik.http.middlewares.backend-cors.headers.accesscontrolallowmethods=GET,POST,HEAD,OPTIONS" - - "traefik.http.middlewares.backend-cors.headers.accesscontrolallowheaders=*" - - "traefik.http.middlewares.backend-cors.headers.accesscontrolalloworiginlist=https://" - - "traefik.http.middlewares.backend-cors.headers.accesscontrolmaxage=100" - - "traefik.http.middlewares.backend-cors.headers.addvaryheader=true" - - "traefik.http.routers.to-backend.rule=Host(``) && PathPrefix(`/api`)" - - "traefik.http.routers.to-backend.middlewares=backend-stripprefix@docker,backend-cors@docker" - - "traefik.http.routers.to-backend.entrypoints=websecure" - - "traefik.http.routers.to-backend.tls.certresolver=myresolver" - - "traefik.http.services.backend.loadbalancer.server.port=9876" - - "traefik.http.services.backend.loadbalancer.healthcheck.path=/epoch/params" - - "traefik.http.services.backend.loadbalancer.healthcheck.port=9876" - - "traefik.http.services.backend.loadbalancer.healthcheck.interval=10s" - - "traefik.http.services.backend.loadbalancer.healthcheck.timeout=5s" - - frontend: - image: /frontend:${FRONTEND_TAG} - volumes: - - /home//config/nginx/auth.conf:/etc/nginx/conf.d/auth.conf - - /home//config/nginx/govtool.htpasswd:/etc/nginx/conf.d/govtool.htpasswd - depends_on: - cardano-node: - condition: service_healthy - postgres: - condition: service_healthy - restart: always - logging: *logging - labels: - - "traefik.enable=true" - - "traefik.http.middlewares.frontend-csp.headers.contentSecurityPolicy=default-src 'self'; img-src *.usersnap.com https://www.googletagmanager.com 'self' data:; script-src 'unsafe-inline' *.usersnap.com 'self' https://www.googletagmanager.com https://browser.sentry-cdn.com; style-src *.usersnap.com *.googleapis.com 'self' 'unsafe-inline' https://fonts.googleapis.com; connect-src *.usersnap.com https://s3.eu-central-1.amazonaws.com/upload.usersnap.com 'self' *.sentry.io *.google-analytics.com ; font-src *.usersnap.com *.gstatic.com 'self' https://fonts.gstatic.com data:; worker-src blob:" - - "traefik.http.routers.to-frontend.rule=Host(``)" - - "traefik.http.routers.to-frontend.entrypoints=websecure" - - "traefik.http.routers.to-frontend.tls.certresolver=myresolver" - - "traefik.http.routers.to-frontend.middlewares=frontend-csp@docker" - - "traefik.http.routers.to-frontend.priority=1" - - "traefik.http.services.frontend.loadbalancer.server.port=80" - -secrets: - postgres_db: - file: /home//config/dbsync-secrets/postgres_db - postgres_password: - file: /home//config/dbsync-secrets/postgres_password - postgres_user: - file: /home//config/dbsync-secrets/postgres_user - backend-config.json: - file: /home//config/backend-config.json - google-credentials.json: - file: /home//config/google-credentials.json - -volumes: - letsencrypt: - db-sync-data: - grafana-data: - postgres: - prometheus-data: - node-db: - node-ipc: - loki-data: diff --git a/scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml b/scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml deleted file mode 100644 index 67be6d45a..000000000 --- a/scripts/govtool/config/templates/grafana-provisioning/alerting/alerting.yml +++ /dev/null @@ -1,379 +0,0 @@ -apiVersion: 1 - -contactPoints: - - orgId: 1 - name: Slack GovTool - receivers: - - uid: d3b7ee9d-ca06-452b-8e73-687b6cefc611 - name: Slack GovTool - type: slack - settings: - recipient: - token: - disableResolveMessage: false - -policies: - - orgId: 1 - receiver: "Slack GovTool" - group_by: - - grafana_folder - - alertname - -groups: - - orgId: 1 - name: status - folder: GovTool - interval: 1m - rules: - - uid: ee72520a-2e0b-4789-a6c3-f29e68240511 - title: Cardano Node slotInEpoch change rate over 5 min - condition: C - data: - - refId: A - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: PBFA97CFB590B2093 - model: - datasource: - type: prometheus - uid: PBFA97CFB590B2093 - editorMode: builder - expr: rate(cardano_node_metrics_slotInEpoch_int{instance="cardano-node:12798"}[5m]) - instant: false - interval: "" - intervalMs: 15000 - maxDataPoints: 43200 - range: true - refId: A - - refId: B - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: [] - type: gt - operator: - type: and - query: - params: - - B - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - hide: false - intervalMs: 1000 - maxDataPoints: 43200 - reducer: last - refId: B - settings: - mode: replaceNN - replaceWithValue: 0 - type: reduce - - refId: C - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0.4 - type: lt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: B - hide: false - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - dashboardUid: e3697f62-6a01-4596-b626-d46141441b32 - panelId: 5 - noDataState: NoData - execErrState: Error - for: 5m - annotations: - __dashboardUid__: e3697f62-6a01-4596-b626-d46141441b32 - __panelId__: "5" - isPaused: false - labels: - app_component: cardano-node - #- orgId: 1 - # name: node_dbsync_block_diff - # folder: GovTool - # interval: 1m - # rules: - - uid: c341df64-edda-4d43-b9bd-ef8989cfb638 - title: DB Sync node-db block height difference - condition: F - data: - - refId: A - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: PBFA97CFB590B2093 - model: - datasource: - type: prometheus - uid: PBFA97CFB590B2093 - editorMode: builder - expr: cardano_db_sync_node_block_height{instance="cardano-db-sync:8080"} - instant: false - interval: "" - intervalMs: 15000 - maxDataPoints: 43200 - range: true - refId: A - - refId: B - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: PBFA97CFB590B2093 - model: - datasource: - type: prometheus - uid: PBFA97CFB590B2093 - editorMode: builder - expr: cardano_db_sync_db_block_height{instance="cardano-db-sync:8080"} - hide: false - instant: false - interval: "" - intervalMs: 15000 - maxDataPoints: 43200 - range: true - refId: B - - refId: C - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - - 0 - type: gt - operator: - type: and - query: - params: [] - reducer: - params: [] - type: avg - type: query - datasource: - name: Expression - type: __expr__ - uid: __expr__ - expression: A - intervalMs: 1000 - maxDataPoints: 43200 - reducer: last - refId: C - settings: - mode: "" - type: reduce - - refId: D - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - - 0 - type: gt - operator: - type: and - query: - params: [] - reducer: - params: [] - type: avg - type: query - datasource: - name: Expression - type: __expr__ - uid: __expr__ - expression: B - intervalMs: 1000 - maxDataPoints: 43200 - reducer: last - refId: D - type: reduce - - refId: E - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - - 0 - type: gt - operator: - type: and - query: - params: [] - reducer: - params: [] - type: avg - type: query - datasource: - name: Expression - type: __expr__ - uid: __expr__ - expression: abs($C-$D) - hide: false - intervalMs: 1000 - maxDataPoints: 43200 - refId: E - type: math - - refId: F - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0 - - 0 - type: gt - operator: - type: and - query: - params: [] - reducer: - params: [] - type: avg - type: query - datasource: - name: Expression - type: __expr__ - uid: __expr__ - expression: E - intervalMs: 1000 - maxDataPoints: 43200 - refId: F - type: threshold - dashboardUid: e3697f62-6a01-4596-b626-d46141441b32 - panelId: 6 - noDataState: NoData - execErrState: Error - for: 5m - annotations: - __dashboardUid__: e3697f62-6a01-4596-b626-d46141441b32 - __panelId__: "6" - isPaused: false - labels: - app_component: cardano-db-sync - - uid: f68d71fb-561d-4a39-b6d9-043bb3053a58 - title: Backend HTTP 502s - condition: C - data: - - refId: A - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: PBFA97CFB590B2093 - model: - datasource: - type: prometheus - uid: PBFA97CFB590B2093 - editorMode: builder - expr: sum by(code) (rate(traefik_service_requests_total{service="backend@docker", code="502"}[$__rate_interval])) - hide: false - instant: false - interval: "" - intervalMs: 15000 - legendFormat: '{{code}}' - maxDataPoints: 43200 - range: true - refId: A - - refId: B - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: [] - type: gt - operator: - type: and - query: - params: - - C - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: A - hide: false - intervalMs: 1000 - maxDataPoints: 43200 - reducer: last - refId: B - type: reduce - - refId: C - relativeTimeRange: - from: 1800 - to: 0 - datasourceUid: __expr__ - model: - conditions: - - evaluator: - params: - - 0.1 - type: gt - operator: - type: and - query: - params: - - D - reducer: - params: [] - type: last - type: query - datasource: - type: __expr__ - uid: __expr__ - expression: B - hide: false - intervalMs: 1000 - maxDataPoints: 43200 - refId: C - type: threshold - dashboardUid: e3697f62-6a01-4596-b626-d46141441b32 - panelId: 4 - noDataState: OK - execErrState: Error - for: 5m - annotations: - __dashboardUid__: e3697f62-6a01-4596-b626-d46141441b32 - __panelId__: "4" - isPaused: false - labels: - app_component: backend diff --git a/scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml b/scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml deleted file mode 100644 index 3f9c826ce..000000000 --- a/scripts/govtool/config/templates/grafana-provisioning/dashboards/dashboard.yml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: 1 - -providers: - # provider name -- name: 'default' - # org id. will default to orgId 1 if not specified - orgId: 1 - # name of the dashboard folder. Required - folder: '' - # folder UID. will be automatically generated if not specified - folderUid: '' - # provider type. Required - type: file - # disable dashboard deletion - disableDeletion: false - # enable dashboard editing - editable: true - # how often Grafana will scan for changed dashboards - updateIntervalSeconds: 10 - options: - path: /etc/grafana/provisioning/dashboards diff --git a/scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json b/scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json deleted file mode 100644 index fdb8da4c7..000000000 --- a/scripts/govtool/config/templates/grafana-provisioning/dashboards/govtool.json +++ /dev/null @@ -1,921 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "links": [], - "liveNow": false, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 0, - "y": 0 - }, - "id": 1, - "options": { - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "exemplar": true, - "expr": "traefik_service_requests_total{service=\"frontend@docker\"}", - "instant": false, - "legendFormat": "{{method}} : {{code}}", - "range": true, - "refId": "A" - } - ], - "title": "Frontend HTTP return codes", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": -1, - "drawStyle": "bars", - "fillOpacity": 100, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 5, - "y": 0 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sum by(code) (rate(traefik_service_requests_total{service=\"frontend@docker\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{code}}", - "range": true, - "refId": "A" - } - ], - "title": "Frontend HTTP requests by return code", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 17, - "y": 0 - }, - "id": 5, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "rate(cardano_node_metrics_slotInEpoch_int{instance=\"cardano-node:12798\"}[5m])", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Cardano Node slotInEpoch change rate ofer 5 min", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 5, - "x": 0, - "y": 8 - }, - "id": 2, - "options": { - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "value", - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "exemplar": true, - "expr": "traefik_service_requests_total{service=\"backend@docker\"}", - "instant": false, - "legendFormat": "{{method}} : {{code}}", - "range": true, - "refId": "A" - } - ], - "title": "Backend HTTP return codes", - "type": "piechart" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "bars", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 5, - "y": 8 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "sum by(code) (rate(traefik_service_requests_total{service=\"backend@docker\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{code}}", - "range": true, - "refId": "A" - } - ], - "title": "Backend HTTP requests by return code", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null - }, - { - "color": "green", - "value": 0 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 4, - "x": 17, - "y": 8 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.0.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "cardano_db_sync_node_block_height{instance=\"cardano-db-sync:8080\"}", - "instant": false, - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "cardano_db_sync_db_block_height{instance=\"cardano-db-sync:8080\"}", - "hide": false, - "instant": false, - "range": true, - "refId": "B" - } - ], - "title": "DB Sync node-db block height difference", - "transformations": [ - { - "id": "calculateField", - "options": { - "mode": "reduceRow", - "reduce": { - "include": [ - "cardano_db_sync_node_block_height{instance=\"cardano-db-sync:8080\", job=\"cardano_db_sync\"}", - "cardano_db_sync_db_block_height{instance=\"cardano-db-sync:8080\", job=\"cardano_db_sync\"}" - ], - "reducer": "diff" - }, - "replaceFields": true - } - } - ], - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 3, - "x": 0, - "y": 16 - }, - "id": 7, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "10.0.8", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": " 100 - (node_filesystem_avail_bytes{mountpoint=\"/\", instance=\"host.docker.internal:9100\"} * 100 / node_filesystem_size_bytes{mountpoint=\"/\", instance=\"host.docker.internal:9100\"})", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Used disk space", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 3, - "y": 16 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "rate(node_disk_io_time_weighted_seconds_total{instance=\"host.docker.internal:9100\", device=\"nvme0n1\"}[5m])", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "Average disk IO queue depth", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 9, - "y": 16 - }, - "id": 10, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "rate(node_disk_read_bytes_total{instance=\"host.docker.internal:9100\", device=\"nvme0n1\"}[5m])", - "instant": false, - "legendFormat": "read", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "rate(node_disk_written_bytes_total{instance=\"host.docker.internal:9100\", device=\"nvme0n1\"}[5m])", - "hide": false, - "instant": false, - "legendFormat": "write", - "range": true, - "refId": "B" - } - ], - "title": "Disk R/W throughput over 5 min", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 15, - "y": 16 - }, - "id": 11, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "builder", - "expr": "node_load5{instance=\"host.docker.internal:9100\"}", - "instant": false, - "range": true, - "refId": "A" - } - ], - "title": "System load (avg 5 min)", - "type": "timeseries" - }, - { - "datasource": { - "type": "loki", - "uid": "a698987a-0cba-42a9-86aa-3ba65104ef72" - }, - "description": "Trace of every deploy made on this machine. This is a good place to start when something goes wrong.", - "gridPos": { - "h": 4, - "w": 21, - "x": 0, - "y": 22 - }, - "id": 12, - "options": { - "dedupStrategy": "none", - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showLabels": false, - "showTime": false, - "sortOrder": "Descending", - "wrapLogMessage": false - }, - "targets": [ - { - "datasource": { - "type": "loki", - "uid": "a698987a-0cba-42a9-86aa-3ba65104ef72" - }, - "editorMode": "builder", - "expr": "{filename=\"/var/log/deployment.log\"}", - "queryType": "range", - "refId": "A" - } - ], - "timeFrom": "1w", - "title": "Deployment events", - "type": "logs" - } - ], - "refresh": "10s", - "schemaVersion": 38, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "GovTool basic stats", - "uid": "e3697f62-6a01-4596-b626-d46141441b32", - "version": 50, - "weekStart": "" -} diff --git a/scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json b/scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json deleted file mode 100644 index 4ab3fd72d..000000000 --- a/scripts/govtool/config/templates/grafana-provisioning/dashboards/traefik_rev4.json +++ /dev/null @@ -1,693 +0,0 @@ -{ - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "description": "Traefik dashboard prometheus", - "editable": true, - "gnetId": 4475, - "graphTooltip": 0, - "id": 1, - "iteration": 1574689270305, - "links": [], - "panels": [ - { - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 10, - "title": "$service stats", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], - "datasource": "Prometheus", - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 7, - "w": 8, - "x": 0, - "y": 1 - }, - "id": 1, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "options": {}, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "", - "targets": [ - { - "expr": "count(traefik_service_open_connections)", - "format": "time_series", - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "0,1", - "title": "Number of Traefik Services ", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "OK", - "value": "1" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "gridPos": { - "h": 7, - "w": 8, - "x": 8, - "y": 1 - }, - "id": 2, - "links": [], - "options": { - "fieldOptions": { - "calcs": [ - "mean" - ], - "defaults": { - "mappings": [ - { - "id": 0, - "op": "=", - "text": "N/A", - "type": 1, - "value": "null" - } - ], - "max": 5, - "min": 0, - "nullValueMode": "connected", - "thresholds": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 3 - } - ], - "unit": "none" - }, - "override": {}, - "values": false - }, - "orientation": "horizontal", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "6.5.0", - "targets": [ - { - "expr": "traefik_service_request_duration_seconds_sum{service=\"$service\"}", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{$service}} | {{method}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average response time", - "type": "gauge" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": "Prometheus", - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 7, - "w": 8, - "x": 16, - "y": 1 - }, - "id": 4, - "interval": null, - "legend": { - "show": true, - "values": true - }, - "legendType": "Under graph", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "options": {}, - "pieType": "pie", - "pluginVersion": "6.5.0", - "strokeWidth": 1, - "targets": [ - { - "expr": "avg_over_time(traefik_service_request_duration_seconds_sum{service=\"$service\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{service}}", - "refId": "A" - } - ], - "title": "Avergae Service response time", - "type": "grafana-piechart-panel", - "valueName": "current" - }, - { - "cacheTimeout": null, - "datasource": "Prometheus", - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 3, - "links": [], - "options": { - "displayMode": "basic", - "fieldOptions": { - "calcs": [ - "range" - ], - "defaults": { - "mappings": [], - "max": 100, - "min": 0, - "thresholds": [ - { - "color": "green", - "value": null - } - ], - "unit": "short" - }, - "override": {}, - "values": false - }, - "orientation": "horizontal" - }, - "pluginVersion": "6.5.0", - "targets": [ - { - "expr": "avg(traefik_service_requests_total{service=\"$service\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{$service}} | {{method}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Total requests over 5min $service", - "type": "bargauge" - }, - { - "collapsed": false, - "datasource": null, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 15 - }, - "id": 12, - "panels": [], - "title": "Global stats", - "type": "row" - }, - { - "aliasColors": {}, - "bars": true, - "dashLength": 10, - "dashes": false, - "datasource": "Prometheus", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 16 - }, - "hiddenSeries": false, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "max": true, - "min": true, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": false, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "options": { - "dataLinks": [] - }, - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code=\"200\"}[5m])", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{method}} : {{code}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Status code 200 over 5min", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": null, - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "decimals": null, - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "datasource": "Prometheus", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 16 - }, - "id": 6, - "links": [], - "options": { - "displayMode": "basic", - "fieldOptions": { - "calcs": [ - "mean" - ], - "defaults": { - "decimals": 0, - "mappings": [], - "max": 100, - "min": 0, - "thresholds": [ - { - "color": "green", - "value": null - } - ], - "unit": "short" - }, - "limit": 10, - "override": {}, - "values": false - }, - "orientation": "horizontal" - }, - "pluginVersion": "6.5.0", - "targets": [ - { - "expr": "rate(traefik_entrypoint_requests_total{entrypoint=~\"$entrypoint\",code!=\"200\"}[5m]) * 100", - "format": "time_series", - "instant": false, - "intervalFactor": 2, - "legendFormat": "{{ method }} : {{code}}", - "refId": "A" - }, - { - "expr": "avg_over_time", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Entrypoint Error Codes", - "type": "bargauge" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": "Prometheus", - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 7, - "interval": null, - "legend": { - "show": true, - "values": true - }, - "legendType": "Right side", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "options": {}, - "pieType": "pie", - "strokeWidth": 1, - "targets": [ - { - "expr": "sum(rate(traefik_service_requests_total[5m])) by (service) ", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ service }}", - "refId": "A" - } - ], - "title": "Requests by service", - "type": "grafana-piechart-panel", - "valueName": "total" - }, - { - "aliasColors": {}, - "breakPoint": "50%", - "cacheTimeout": null, - "combine": { - "label": "Others", - "threshold": 0 - }, - "datasource": "Prometheus", - "fontSize": "80%", - "format": "short", - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 8, - "interval": null, - "legend": { - "show": true, - "values": true - }, - "legendType": "Right side", - "links": [], - "maxDataPoints": 3, - "nullPointMode": "connected", - "options": {}, - "pieType": "pie", - "strokeWidth": 1, - "targets": [ - { - "expr": "sum(rate(traefik_entrypoint_requests_total{entrypoint =~ \"$entrypoint\"}[5m])) by (entrypoint) ", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ entrypoint }}", - "refId": "A" - } - ], - "title": "Requests by protocol", - "type": "grafana-piechart-panel", - "valueName": "total" - } - ], - "schemaVersion": 21, - "style": "dark", - "tags": [ - "traefik", - "prometheus" - ], - "templating": { - "list": [ - { - "allValue": null, - "current": { - "tags": [], - "text": "prometheus@docker", - "value": "prometheus@docker" - }, - "datasource": "Prometheus", - "definition": "label_values(service)", - "hide": 0, - "includeAll": true, - "label": null, - "multi": false, - "name": "service", - "options": [], - "query": "label_values(service)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "selected": false, - "tags": [], - "text": "All", - "value": [ - "$__all" - ] - }, - "datasource": "Prometheus", - "definition": "label_values(entrypoint)", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "entrypoint", - "options": [], - "query": "label_values(entrypoint)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - { - "allValue": null, - "current": { - "text": "200", - "value": "200" - }, - "datasource": "Prometheus", - "definition": "label_values(code)", - "hide": 2, - "includeAll": false, - "label": null, - "multi": false, - "name": "code", - "options": [ - { - "selected": true, - "text": "200", - "value": "200" - }, - { - "selected": false, - "text": "302", - "value": "302" - }, - { - "selected": false, - "text": "304", - "value": "304" - }, - { - "selected": false, - "text": "400", - "value": "400" - }, - { - "selected": false, - "text": "404", - "value": "404" - }, - { - "selected": false, - "text": "429", - "value": "429" - }, - { - "selected": false, - "text": "499", - "value": "499" - }, - { - "selected": false, - "text": "500", - "value": "500" - }, - { - "selected": false, - "text": "503", - "value": "503" - } - ], - "query": "label_values(code)", - "refresh": 0, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-5m", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "Traefik2", - "uid": "qPdAviJmz1", - "version": 1 -} diff --git a/scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml b/scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml deleted file mode 100644 index a4ea24d58..000000000 --- a/scripts/govtool/config/templates/grafana-provisioning/datasources/datasource.yml +++ /dev/null @@ -1,84 +0,0 @@ -# config file version -apiVersion: 1 - -# list of datasources that should be deleted from the database -deleteDatasources: - - name: Prometheus - orgId: 1 - -# list of datasources to insert/update depending -# whats available in the database -datasources: - # name of the datasource. Required -- name: Prometheus - # datasource UID. - uid: PBFA97CFB590B2093 - # datasource type. Required - type: prometheus - # access mode. direct or proxy. Required - access: browser - # org id. will default to orgId 1 if not specified - orgId: 1 - # url - url: http://prometheus:9090 - # database password, if used - password: - # database user, if used - user: - # database name, if used - database: - # enable/disable basic auth - basicAuth: false - # basic auth username - basicAuthUser: admin - # basic auth password - basicAuthPassword: foobar - # enable/disable with credentials headers - withCredentials: - # mark as default datasource. Max one per org - isDefault: true - # fields that will be converted to json and stored in json_data - jsonData: - graphiteVersion: "1.1" - tlsAuth: false - tlsAuthWithCACert: false - # json object of data that will be encrypted. - secureJsonData: - tlsCACert: "..." - tlsClientCert: "..." - tlsClientKey: "..." - version: 1 - # allow users to edit datasources from the UI. - editable: true - - # name of the datasource. Required -- name: Loki - # datasource UID. - uid: a698987a-0cba-42a9-86aa-3ba65104ef72 - # datasource type. Required - type: loki - # access mode. direct or proxy. Required - access: browser - # org id. will default to orgId 1 if not specified - orgId: 1 - # url - url: http://loki:3100 - # database password, if used - password: - # database user, if used - user: - # database name, if used - database: - # enable/disable basic auth - basicAuth: false - # basic auth username - basicAuthUser: admin - # basic auth password - basicAuthPassword: foobar - # enable/disable with credentials headers - withCredentials: - # mark as default datasource. Max one per org - isDefault: false - version: 1 - # allow users to edit datasources from the UI. - editable: true diff --git a/scripts/govtool/config/templates/loki.yml b/scripts/govtool/config/templates/loki.yml deleted file mode 100644 index 54b6e3344..000000000 --- a/scripts/govtool/config/templates/loki.yml +++ /dev/null @@ -1,56 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - grpc_listen_port: 9095 - -ingester: - wal: - enabled: false - lifecycler: - address: 127.0.0.1 - ring: - kvstore: - store: inmemory - replication_factor: 1 - chunk_idle_period: 15m - chunk_block_size: 262144 - chunk_retain_period: 1m - max_transfer_retries: 0 - -compactor: - working_directory: /loki/compactor - shared_store: filesystem - -schema_config: - configs: - - from: 2024-01-01 - store: boltdb-shipper - object_store: filesystem - schema: v11 - index: - prefix: index_ - period: 24h - -storage_config: - boltdb_shipper: - active_index_directory: /loki/index - cache_location: /loki/index_cache - cache_ttl: 24h - shared_store: filesystem - filesystem: - directory: /loki/chunks - -limits_config: - enforce_metric_name: false - reject_old_samples: true - reject_old_samples_max_age: 168h - max_cache_freshness_per_query: 10m - retention_period: 672h - -chunk_store_config: - max_look_back_period: 672h - -table_manager: - retention_deletes_enabled: true - retention_period: 672h diff --git a/scripts/govtool/config/templates/prometheus.yml b/scripts/govtool/config/templates/prometheus.yml deleted file mode 100644 index 0d637f3f1..000000000 --- a/scripts/govtool/config/templates/prometheus.yml +++ /dev/null @@ -1,23 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - external_labels: - monitor: 'govtool' -scrape_configs: - - job_name: 'traefik' - scrape_interval: 5s - static_configs: - - targets: ['traefik:8082'] - - job_name: 'cardano' - scrape_interval: 5s - static_configs: - - targets: ['cardano-node:12798'] - - job_name: 'cardano_db_sync' - scrape_interval: 5s - metrics_path: / - static_configs: - - targets: ['cardano-db-sync:8080'] - - job_name: 'host' - scrape_interval: 5s - static_configs: - - targets: ['host.docker.internal:9100'] diff --git a/scripts/govtool/config/templates/promtail.yml b/scripts/govtool/config/templates/promtail.yml deleted file mode 100644 index e24b9d32a..000000000 --- a/scripts/govtool/config/templates/promtail.yml +++ /dev/null @@ -1,18 +0,0 @@ -server: - http_listen_port: 9080 - grpc_listen_port: 0 - -positions: - filename: /tmp/positions.yaml - -clients: - - url: http://loki:3100/loki/api/v1/push - -scrape_configs: -- job_name: system - static_configs: - - targets: - - localhost - labels: - job: varlogs - __path__: /var/log/deployment.log diff --git a/scripts/govtool/config/templates/ssh-client-config b/scripts/govtool/config/templates/ssh-client-config deleted file mode 100644 index 60c61fe94..000000000 --- a/scripts/govtool/config/templates/ssh-client-config +++ /dev/null @@ -1,4 +0,0 @@ -Host DOCKER_HOST - ControlMaster auto - ControlPath ~/.ssh/control-%C - ControlPersist yes diff --git a/scripts/govtool/custom-cardano-db-sync.Dockerfile b/scripts/govtool/custom-cardano-db-sync.Dockerfile deleted file mode 100644 index 93b6f6b7c..000000000 --- a/scripts/govtool/custom-cardano-db-sync.Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM ghcr.io/intersectmbo/cardano-db-sync:13.5.0.2 - -COPY custom-cardano-db-sync.entrypoint.sh /usr/local/bin/custom-cardano-db-sync.entrypoint.sh -RUN chmod +x /usr/local/bin/custom-cardano-db-sync.entrypoint.sh - -ENTRYPOINT [ "/usr/local/bin/custom-cardano-db-sync.entrypoint.sh" ] diff --git a/scripts/govtool/custom-cardano-db-sync.entrypoint.sh b/scripts/govtool/custom-cardano-db-sync.entrypoint.sh deleted file mode 100644 index f9e9c621c..000000000 --- a/scripts/govtool/custom-cardano-db-sync.entrypoint.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -echo "Custom Cardano DB Sync entrypoint" -set -euo pipefail -mkdir -p -m 1777 /tmp -mkdir -p /configuration -CARDANO_NODE_SOCKET_PATH=/node-ipc/node.socket -CARDANO_DB_SYNC_CONFIG_PATH=/configuration/db-sync-config.json - -# set pgpass file -echo "-> Generating PGPASS file" -SECRET_DIR=/run/secrets -POSTGRES_DB=${POSTGRES_DB:-$(< ${SECRET_DIR}/postgres_db)} -POSTGRES_USER=${POSTGRES_USER:-$(< ${SECRET_DIR}/postgres_user)} -POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-$(< ${SECRET_DIR}/postgres_password)} -echo "${POSTGRES_HOST}:${POSTGRES_PORT}:${POSTGRES_DB}:${POSTGRES_USER}:${POSTGRES_PASSWORD}" > /configuration/pgpass -chmod 0600 /configuration/pgpass -export PGPASSFILE=/configuration/pgpass - -# wait for cardano node to start -echo -n "-> Waiting for $CARDANO_NODE_SOCKET_PATH" -until [ -S "$CARDANO_NODE_SOCKET_PATH" ]; do - echo -n "." - sleep 10 -done -echo - -# find schema directory -echo "-> Finding schema directory" -SCHEMA_DIR=$(find /nix/store -type d -name '*-schema') - -echo "-> Running Cardano DB Sync" -exec cardano-db-sync \ - --config "$CARDANO_DB_SYNC_CONFIG_PATH" \ - --socket-path "$CARDANO_NODE_SOCKET_PATH" \ - --schema-dir "$SCHEMA_DIR" \ - --state-dir /state \ - $@ - -echo "-> Cleaning up" diff --git a/scripts/govtool/custom-cardano-db-sync.mk b/scripts/govtool/custom-cardano-db-sync.mk deleted file mode 100644 index cb5088e51..000000000 --- a/scripts/govtool/custom-cardano-db-sync.mk +++ /dev/null @@ -1,22 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-custom-cardano-db-sync - -# image tags -custom-cardano-db-sync_image_tag := latest - -.PHONY: build-custom-cardano-db-sync -build-custom-cardano-db-sync: docker-login - @:$(call check_defined, cardano_network) - $(call check_image_on_ecr,custom-cardano-db-sync,$(custom-cardano-db-sync_image_tag)) || \ - $(docker) build --tag "$(repo_url)/custom-cardano-db-sync:$(custom-cardano-db-sync_image_tag)" --file "$(root_dir)/scripts/govtool/custom-cardano-db-sync.Dockerfile"\ - $(root_dir)/scripts/govtool - -.PHONY: push-custom-cardano-db-sync -push-custom-cardano-db-sync: build-custom-cardano-db-sync - $(call check_image_on_ecr,custom-cardano-db-sync,$(custom-cardano-db-sync_image_tag)) || \ - $(docker) push $(repo_url)/custom-cardano-db-sync:$(custom-cardano-db-sync_image_tag) diff --git a/scripts/govtool/default.nix b/scripts/govtool/default.nix deleted file mode 100644 index 420c999e6..000000000 --- a/scripts/govtool/default.nix +++ /dev/null @@ -1,29 +0,0 @@ -{ pkgs ? import { } }: -pkgs.mkShell { - buildInputs = with pkgs; [ awscli curl docker git gnumake openssh rsync ]; - shellHook = '' - function warn() { tput setaf $2; echo "$1"; tput sgr0; } - - tput bold - warn "Welcome to GovTool!" 4 - warn "This is a deployment shell." 4 - warn "Read the ${./README.md} to get more info about the deployment processes." 8 - case "''${ENVIRONMENT}" in - "dev") - warn "Your configuration is set to deploy to DEV environment, you are safe." 2 - ;; - "test") - warn "Your configuration is set to deploy to TEST environment, be careful…" 5 - ;; - "staging") - warn "Your configuration is set to deploy to STAGING environment, be careful…" 5 - ;; - "beta") - warn "Your configuration is set to deploy to BETA environment, BE CAREFUL!" 1 - ;; - *) - warn "Your configuration is not set, deployment will not work…" 1 - ;; - esac - ''; -} diff --git a/scripts/govtool/docker-compose.node+dbsync.yml b/scripts/govtool/docker-compose.node+dbsync.yml deleted file mode 100644 index 60621b496..000000000 --- a/scripts/govtool/docker-compose.node+dbsync.yml +++ /dev/null @@ -1,99 +0,0 @@ -# NOTE: This onfiguration file is designed to facilitate the development process -# for developers working with the Cardano blockchain by provisioning a Cardano -# Node on SanchoNet, Cardano DB Sync, and a PostgreSQL database as supporting -# infrastructure. -# -# By automatically setting up these essential services, it allows developers to -# easily interact with the Cardano blockchain and perform data queries through -# the exposed PostgreSQL database on port 5432 on the DBSync database. -# -# Before start there are three text files that has to be set up in a GovTool -# configuration directory in order to provide sufficient database configuration -# for both PostgreSQL and Cardano DB Sync services: -# * ${TARGET_HOST_CONFIG_DIR}/dev-postgres_db -# * ${TARGET_HOST_CONFIG_DIR}/dev-postgres_password -# * ${TARGET_HOST_CONFIG_DIR}/dev-postgres_user -# -# The GovTool configuration directory under ${TARGET_HOST_CONFIG_DIR} has to be -# created, the sufficient privileges has to be applied to it to read it's -# context and the ENV has to be provided with proper value. -# -# USAGE: -# docker compose --file docker-compose.node+dbsync.yml up --detach -# docker compose --file docker-compose.node+dbsync.yml down --remove-orphans -# docker compose --file docker-compose.node+dbsync.yml logs --follow --timestamps cardano-node -# docker compose --file docker-compose.node+dbsync.yml logs --follow --timestamps cardano-db-sync -# docker-compose run postgres psql - -version: "3.9" - -services: - postgres: - image: postgres:15-alpine - environment: - - POSTGRES_DB_FILE=/run/secrets/postgres_db - - POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password - - POSTGRES_USER_FILE=/run/secrets/postgres_user - secrets: - - postgres_password - - postgres_user - - postgres_db - volumes: - - postgres:/var/lib/postgresql/data - ports: - - "5432:5432" - restart: always - command: ${POSTGRES_ARGS:--c maintenance_work_mem=1GB -c max_parallel_maintenance_workers=4} - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 10s - timeout: 5s - retries: 5 - - cardano-node: - image: ghcr.io/intersectmbo/cardano-node:10.1.4 - environment: - - NETWORK=sanchonet - volumes: - - node-db:/data/db - - node-ipc:/ipc - restart: on-failure - healthcheck: - test: ["CMD-SHELL", "curl -f 127.0.0.1:12788 || exit 1"] - interval: 10s - timeout: 10s - retries: 10 - - cardano-db-sync: - image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4 - environment: - - NETWORK=sanchonet - - POSTGRES_HOST=postgres - - POSTGRES_PORT=5432 - depends_on: - cardano-node: - condition: service_healthy - postgres: - condition: service_healthy - secrets: - - postgres_password - - postgres_user - - postgres_db - volumes: - - db-sync-data:/var/lib/cexplorer - - node-ipc:/node-ipc - restart: on-failure - -secrets: - postgres_db: - file: ${TARGET_HOST_CONFIG_DIR}/dev-postgres_db - postgres_password: - file: ${TARGET_HOST_CONFIG_DIR}/dev-postgres_password - postgres_user: - file: ${TARGET_HOST_CONFIG_DIR}/dev-postgres_user - -volumes: - db-sync-data: - postgres: - node-db: - node-ipc: diff --git a/scripts/govtool/frontend.mk b/scripts/govtool/frontend.mk deleted file mode 100644 index 910cbd030..000000000 --- a/scripts/govtool/frontend.mk +++ /dev/null @@ -1,35 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-frontend - -# image tags -frontend_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/frontend)-$(env) - -.PHONY: build-frontend -build-frontend: docker-login - @:$(call check_defined, cardano_network) - if [[ "$(cardano_network)" = "mainnet" ]]; then NETWORK_FLAG=1; else NETWORK_FLAG=0; fi; \ - $(call check_image_on_ecr,frontend,$(frontend_image_tag)) || \ - $(docker) build --tag "$(repo_url)/frontend:$(frontend_image_tag)" \ - --build-arg VITE_APP_ENV="$(env)" \ - --build-arg VITE_BASE_URL="https://$(domain)/api" \ - --build-arg VITE_METADATA_API_URL="https://$(domain)/metadata-validation" \ - --build-arg VITE_GTM_ID="$${GTM_ID}" \ - --build-arg VITE_NETWORK_FLAG="$$NETWORK_FLAG" \ - --build-arg VITE_SENTRY_DSN="$${SENTRY_DSN}" \ - --build-arg VITE_USERSNAP_SPACE_API_KEY="$${USERSNAP_SPACE_API_KEY}" \ - --build-arg VITE_IS_PROPOSAL_DISCUSSION_FORUM_ENABLED="$${IS_PROPOSAL_DISCUSSION_FORUM_ENABLED}" \ - --build-arg NPMRC_TOKEN="$${NPMRC_TOKEN}" \ - --build-arg VITE_PDF_API_URL="$${PDF_API_URL}" \ - --build-arg VITE_IPFS_GATEWAY="$${IPFS_GATEWAY}" \ - --build-arg VITE_IPFS_PROJECT_ID="$${IPFS_PROJECT_ID}" \ - $(root_dir)/govtool/frontend - -.PHONY: push-frontend -push-frontend: build-frontend - $(call check_image_on_ecr,frontend,$(frontend_image_tag)) || \ - $(docker) push $(repo_url)/frontend:$(frontend_image_tag) diff --git a/scripts/govtool/info.mk b/scripts/govtool/info.mk deleted file mode 100644 index 0a0433f1e..000000000 --- a/scripts/govtool/info.mk +++ /dev/null @@ -1,44 +0,0 @@ -# pipeline information -pipeline_url := $(shell echo $${PIPELINE_URL:-}) -ifeq ($(pipeline_url),) - pipeline_info := _Deployed from local machine._ -else - pipeline_info := _Deployed from <$(pipeline_url)|GitHub>._ -endif - -deployment_log_file := /var/log/deployment.log - -.PHONY: log-deployment -log-deployment: - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - ssh $(ssh_url) 'echo "$(shell date +'%Y-%m-%d %H:%M:%S%z') INFO: $(branch) ($(commit)); cardano-node $(cardano_node_image_tag); cardano-dbsync $(cardano_db_sync_image_tag); backend $(backend_image_tag); frontend $(frontend_image_tag); $(pipeline_url)" >> $(deployment_log_file)' - -.PHONY: info -info: - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - @echo "+-----------" - @echo "| TIME $(shell date +'%Y-%m-%d %H:%M:%S%z')" - @echo "| BRANCH $(branch) [$(commit)]" - @echo "| ENV $(env)" - @echo "| NETWORK $(cardano_network)" - @echo "| BACKEND $(repo_url)/backend:$(backend_image_tag)" - @echo "| FRONTEND $(repo_url)/frontend:$(frontend_image_tag)" - @echo "| STATUS $(repo_url)/status-service:$(status_service_image_tag)" - @echo "| METADATA $(repo_url)/metadata-validation:$(metadata_validation_image_tag)" - @echo "| ANALYTICS $(repo_url)/analytics-dashboard:$(analytics_dashboard_image_tag)" - @echo "| NODE ghcr.io/intersectmbo/cardano-node:$(cardano_node_image_tag)" - @echo "| DBSYNC ghcr.io/intersectmbo/cardano-db-sync:$(cardano_db_sync_image_tag)" - @echo "| SSH $(ssh_url)" - @echo "| URL https://$(docker_host)" - @echo "+-----------" - -.PHONY: notify -notify: info log-deployment - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - $(curl) -X POST https://slack.com/api/chat.postMessage\ - -H "Authorization: Bearer $${GRAFANA_SLACK_OAUTH_TOKEN}" \ - -H "Content-Type: application/json; charset=utf-8" \ - --data "{ \"channel\":\"$${GRAFANA_SLACK_RECIPIENT}\", \"text\":\":rocket: *Deploy performed on \`$(env)\`*\n- from *branch* \`$(branch)\` (\`$(commit)\`),\n- using *Cardano Node* version \`$(cardano_node_image_tag)\`,\n- using *Cardano DB Sync* version \`$(cardano_db_sync_image_tag)\`,\n- using *GovTool backend* version \`$(backend_image_tag)\`,\n- using *Govtool frontend* version \`$(frontend_image_tag)\`,\n- using *Govtool status-service* version \`$(status_service_image_tag)\`,\n- using *Govtool metadata-validation* version \`$(metadata_validation_image_tag)\`,\n- using *Govtool analytics-dashboard* version \`$(analytics_dashboard_image_tag)\`.\n$(pipeline_info)\" }" diff --git a/scripts/govtool/metadata-validation.mk b/scripts/govtool/metadata-validation.mk deleted file mode 100644 index fb3352c59..000000000 --- a/scripts/govtool/metadata-validation.mk +++ /dev/null @@ -1,23 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-metadata-validation - -# image tags -metadata_validation_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/metadata-validation) - -.PHONY: build-metadata-validation -build-metadata-validation: docker-login - $(call check_image_on_ecr,metadata-validation,$(metadata_validation_image_tag)) || \ - $(docker) build --tag "$(repo_url)/metadata-validation:$(metadata_validation_image_tag)" \ - --build-arg IPFS_GATEWAY="$${IPFS_GATEWAY}" \ - --build-arg IPFS_PROJECT_ID="$${IPFS_PROJECT_ID}" \ - $(root_dir)/govtool/metadata-validation - -.PHONY: push-metadata-validation -push-metadata-validation: build-metadata-validation - $(call check_image_on_ecr,metadata-validation,$(metadata_validation_image_tag)) || \ - $(docker) push $(repo_url)/metadata-validation:$(metadata_validation_image_tag) diff --git a/scripts/govtool/status-service.mk b/scripts/govtool/status-service.mk deleted file mode 100644 index 1337f776a..000000000 --- a/scripts/govtool/status-service.mk +++ /dev/null @@ -1,21 +0,0 @@ -common_mk := common.mk -ifeq ($(origin $(common_mk)), undefined) - $(eval $(common_mk) := included) - include $(common_mk) -endif - -.DEFAULT_GOAL := push-status-service - -# image tags -status_service_image_tag := $(shell git log -n 1 --format="%H" -- $(root_dir)/govtool/status-service) - -.PHONY: build-status-service -build-status-service: docker-login - $(call check_image_on_ecr,status-service,$(status_service_image_tag)) || \ - $(docker) build --tag "$(repo_url)/status-service:$(status_service_image_tag)" \ - $(root_dir)/govtool/status-service - -.PHONY: push-status-service -push-status-service: build-status-service - $(call check_image_on_ecr,status-service,$(status_service_image_tag)) || \ - $(docker) push $(repo_url)/status-service:$(status_service_image_tag) diff --git a/scripts/govtool/utils.mk b/scripts/govtool/utils.mk deleted file mode 100644 index decd80891..000000000 --- a/scripts/govtool/utils.mk +++ /dev/null @@ -1,30 +0,0 @@ -.PHONY: ssh -ssh: - export TERM=xterm-256color; \ - ssh $(ssh_url) - -.PHONY: docker-compose -docker-compose: - @:$(call check_defined, cardano_network) - @:$(call check_defined, env) - @:$(call check_defined, cmd) - export CARDANO_NETWORK=$(cardano_network); \ - export DOCKER_HOST=ssh://$(ssh_url); \ - export ENVIRONMENT=$(env); \ - export GRAFANA_ADMIN_PASSWORD=$${GRAFANA_ADMIN_PASSWORD}; \ - export BACKEND_TAG=$(backend_image_tag); \ - export FRONTEND_TAG=$(frontend_image_tag); \ - export STATUS_SERVICE_TAG=$(status_service_image_tag); \ - export METADATA_VALIDATION_TAG=$(metadata_validation_image_tag); \ - export ANALYTICS_DASHBOARD_TAG=$(analytics_dashboard_image_tag); \ - export CARDANO_NODE_TAG=$(cardano_node_image_tag); \ - export CARDANO_DB_SYNC_TAG=$(cardano_db_sync_image_tag); \ - $(ssh-keyscan) $(docker_host) 2>/dev/null >> ~/.ssh/known_hosts; \ - $(docker) compose -f $(docker_compose_file) -p $(compose_stack_name) $(cmd) - -.PHONY: reload-grafana -reload-grafana: - @:$(call check_defined, domain) - curl -X POST -u "admin:$${GRAFANA_ADMIN_PASSWORD}" https://$(domain)/grafana/api/admin/provisioning/alerting/reload - curl -X POST -u "admin:$${GRAFANA_ADMIN_PASSWORD}" https://$(domain)/grafana/api/admin/provisioning/dashboards/reload - curl -X POST -u "admin:$${GRAFANA_ADMIN_PASSWORD}" https://$(domain)/grafana/api/admin/provisioning/notifications/reload From 20df7425e31da34b5424c91753235cd218522978 Mon Sep 17 00:00:00 2001 From: Niraj Date: Fri, 21 Feb 2025 19:31:42 +0545 Subject: [PATCH 02/23] fix: ensure proposal cards are visible in proposal search --- .../tests/8-proposal-discussion/proposalDiscussion.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/govtool-frontend/playwright/tests/8-proposal-discussion/proposalDiscussion.spec.ts b/tests/govtool-frontend/playwright/tests/8-proposal-discussion/proposalDiscussion.spec.ts index 852e6d74c..9b0303019 100644 --- a/tests/govtool-frontend/playwright/tests/8-proposal-discussion/proposalDiscussion.spec.ts +++ b/tests/govtool-frontend/playwright/tests/8-proposal-discussion/proposalDiscussion.spec.ts @@ -115,6 +115,7 @@ test("8C. Should search the list of proposed governance actions.", async ({ async () => { const proposalCards = await proposalDiscussionPage.getAllProposals(); for (const proposalCard of proposalCards) { + await expect(proposalCard).toBeVisible(); const proposalTitle = await proposalCard .locator('[data-testid^="proposal-"][data-testid$="-title"]') .innerText(); From 692779989f587f0f5bd639448739c0ae64b8281c Mon Sep 17 00:00:00 2001 From: Niraj Date: Fri, 21 Feb 2025 19:32:14 +0545 Subject: [PATCH 03/23] fix: drep filter issue due to default active filter --- .../govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts | 3 --- .../playwright/tests/2-delegation/delegation.spec.ts | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts b/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts index 767ddef7c..ab6e74897 100644 --- a/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts +++ b/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts @@ -68,9 +68,6 @@ export default class DRepDirectoryPage { async filterDReps(filterOptions: string[]) { for (const option of filterOptions) { await this.page.getByTestId(`${option}-checkbox`).click(); - if (option !== "Active" && filterOptions.length === 1) { - await this.page.getByTestId(`Active-checkbox`).click(); - } } } diff --git a/tests/govtool-frontend/playwright/tests/2-delegation/delegation.spec.ts b/tests/govtool-frontend/playwright/tests/2-delegation/delegation.spec.ts index cdd5e5be5..e894e1450 100644 --- a/tests/govtool-frontend/playwright/tests/2-delegation/delegation.spec.ts +++ b/tests/govtool-frontend/playwright/tests/2-delegation/delegation.spec.ts @@ -129,6 +129,7 @@ test("2K_1. Should filter DReps", async ({ page }) => { await dRepDirectory.goto(); await dRepDirectory.filterBtn.click(); + await page.getByTestId(`Active-checkbox`).click(); // Single filter for (const option of dRepFilterOptions) { From 40b950173947c747a0429eb14bb811700009ae3b Mon Sep 17 00:00:00 2001 From: Niraj Date: Fri, 21 Feb 2025 19:32:23 +0545 Subject: [PATCH 04/23] fix: increase timeout for system bootstrapping warning visibility check --- .../playwright/tests/6-miscellaneous/miscellaneous.spec.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/govtool-frontend/playwright/tests/6-miscellaneous/miscellaneous.spec.ts b/tests/govtool-frontend/playwright/tests/6-miscellaneous/miscellaneous.spec.ts index aaa88a863..3ca166579 100644 --- a/tests/govtool-frontend/playwright/tests/6-miscellaneous/miscellaneous.spec.ts +++ b/tests/govtool-frontend/playwright/tests/6-miscellaneous/miscellaneous.spec.ts @@ -285,7 +285,9 @@ test("6S. Should Warn users that they are in bootstrapping phase via banner", as await responsePromise; - await expect(page.getByTestId("system-bootstrapping-warning")).toBeVisible(); + await expect(page.getByTestId("system-bootstrapping-warning")).toBeVisible({ + timeout: 60_000, + }); const [bootstrap] = await Promise.all([ context.waitForEvent("page"), From 42cfe4b493773833600878b3768d3116624cee50 Mon Sep 17 00:00:00 2001 From: Niraj Date: Mon, 24 Feb 2025 10:30:30 +0545 Subject: [PATCH 05/23] refactor: replace fs operations with helper functions for file handling --- .../playwright/lib/helpers/file.ts | 19 +++++++++++- .../playwright/lib/walletManager.ts | 31 ++----------------- 2 files changed, 21 insertions(+), 29 deletions(-) diff --git a/tests/govtool-frontend/playwright/lib/helpers/file.ts b/tests/govtool-frontend/playwright/lib/helpers/file.ts index a803ab288..25fa3239b 100644 --- a/tests/govtool-frontend/playwright/lib/helpers/file.ts +++ b/tests/govtool-frontend/playwright/lib/helpers/file.ts @@ -1,4 +1,4 @@ -import { writeFile } from "fs"; +import { readFile, writeFile } from "fs"; const path = require("path"); const baseFilePath = path.resolve(__dirname, "../_mock"); @@ -18,3 +18,20 @@ export async function createFile(fileName: string, data?: any) { ) ); } + +export async function getFile(fileName: string): Promise { + const data: string = await new Promise((resolve, reject) => + readFile(`${baseFilePath}/${fileName}`, "utf8", (err, data) => { + if (err) { + if (err.code === "ENOENT") { + resolve(undefined); + } else { + reject(err); + } + } else { + resolve(data); + } + }) + ); + return data ? JSON.parse(data) : undefined; +} diff --git a/tests/govtool-frontend/playwright/lib/walletManager.ts b/tests/govtool-frontend/playwright/lib/walletManager.ts index 7021d9a5f..af0d08854 100644 --- a/tests/govtool-frontend/playwright/lib/walletManager.ts +++ b/tests/govtool-frontend/playwright/lib/walletManager.ts @@ -1,6 +1,6 @@ import { StaticWallet } from "@types"; -import * as fs from "fs"; import { LockInterceptor } from "./lockInterceptor"; +import { createFile, getFile } from "@helpers/file"; const path = require("path"); const baseFilePath = path.resolve(__dirname, "./_mock"); @@ -28,36 +28,11 @@ class WalletManager { } async writeWallets(wallets: StaticWallet[], purpose: Purpose) { - await new Promise((resolve, reject) => - fs.writeFile( - `${baseFilePath}/${purpose}Wallets.json`, - JSON.stringify(wallets, null, 2), - (err) => { - if (err) { - reject(err); - } else { - resolve(); - } - } - ) - ); + await createFile(`${purpose}Wallets.json`, wallets); } async readWallets(purpose: Purpose): Promise { - const data: string = await new Promise((resolve, reject) => - fs.readFile( - `${baseFilePath}/${purpose}Wallets.json`, - "utf8", - (err, data) => { - if (err) { - reject(err); - } else { - resolve(data); - } - } - ) - ); - return JSON.parse(data); + return await getFile(`${purpose}Wallets.json`); } async removeCopyWallet(walletToRemove: StaticWallet, purpose: Purpose) { From 8faffa0da29bf07e4b0fe7ed6eff8a2176db643d Mon Sep 17 00:00:00 2001 From: Niraj Date: Mon, 24 Feb 2025 10:32:05 +0545 Subject: [PATCH 06/23] fix: improve protocol parameters retrieval with file caching --- tests/govtool-frontend/playwright/.gitignore | 1 + .../playwright/lib/helpers/cardano.ts | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/govtool-frontend/playwright/.gitignore b/tests/govtool-frontend/playwright/.gitignore index 57e9aea00..f8c7b5f89 100644 --- a/tests/govtool-frontend/playwright/.gitignore +++ b/tests/govtool-frontend/playwright/.gitignore @@ -21,4 +21,5 @@ lib/_mock/registerDRepCopyWallets.json lib/_mock/registeredDRepCopyWallets.json lib/_mock/wallets.json lib/_mock/proposals.json +lib/_mock/protocolParameter.json ./lock_logs.txt diff --git a/tests/govtool-frontend/playwright/lib/helpers/cardano.ts b/tests/govtool-frontend/playwright/lib/helpers/cardano.ts index b9c19ffff..ad7126a94 100644 --- a/tests/govtool-frontend/playwright/lib/helpers/cardano.ts +++ b/tests/govtool-frontend/playwright/lib/helpers/cardano.ts @@ -4,6 +4,8 @@ import kuberService from "@services/kuberService"; import { ProposalType, ProtocolParams } from "@types"; import { allure } from "allure-playwright"; import { bech32 } from "bech32"; +import { functionWaitedAssert } from "./waitedLoop"; +import { createFile, getFile } from "./file"; export function lovelaceToAda(lovelace: number) { if (lovelace === 0) return 0; @@ -17,8 +19,16 @@ export function generateWalletAddress() { } export async function getProtocolParamsMajorVersion() { - const protocolParameter: ProtocolParams = - await kuberService.queryProtocolParams(); + let protocolParameter = await getFile("protocolParameter.json"); + if (protocolParameter === undefined) { + await functionWaitedAssert( + async () => { + protocolParameter = await kuberService.queryProtocolParams(); + await createFile("protocolParameter.json", protocolParameter); + }, + { name: "queryProtocolParams" } + ); + } return protocolParameter.protocolVersion.major; } From cfd329af2b61c666544d6a34f7c9cdd86222fa0c Mon Sep 17 00:00:00 2001 From: Niraj Date: Mon, 24 Feb 2025 10:57:33 +0545 Subject: [PATCH 07/23] fix: enhance frontend validation with detailed error messages for dRep sorting --- .../playwright/lib/pages/dRepDirectoryPage.ts | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts b/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts index ab6e74897..5ce76d7f9 100644 --- a/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts +++ b/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts @@ -140,20 +140,28 @@ export default class DRepDirectoryPage { const isValid = validationFn(dRepList[i], dRepList[i + 1]); expect(isValid).toBe(true); } - // Frontend validation - const cip105DRepListFE = await this.getAllListedCIP105DRepIds(); - const cip129DRepListFE = await this.getAllListedCIP129DRepIds(); - const cip129DRepListApi = dRepList.map((dRep) => - convertDRepToCIP129(dRep.drepId, dRep.isScriptBased) - ); + await functionWaitedAssert( + async () => { + // Frontend validation + const cip105DRepListFE = await this.getAllListedCIP105DRepIds(); + const cip129DRepListFE = await this.getAllListedCIP129DRepIds(); - for (let i = 0; i <= cip105DRepListFE.length - 1; i++) { - await expect(cip129DRepListFE[i]).toHaveText(cip129DRepListApi[i]); - await expect(cip105DRepListFE[i]).toHaveText( - `(CIP-105) ${dRepList[i].view}` - ); - } + const cip129DRepListApi = dRepList.map((dRep) => + convertDRepToCIP129(dRep.drepId, dRep.isScriptBased) + ); + + for (let i = 0; i <= cip105DRepListFE.length - 1; i++) { + await expect(cip129DRepListFE[i], { + message: `Cip129 dRep Id from Api:${cip129DRepListApi[i]} is not equal to ${await cip129DRepListFE[i].textContent()} on sort ${option}`, + }).toHaveText(cip129DRepListApi[i]); + await expect(cip105DRepListFE[i], { + message: `Cip105 dRep Id from Api:${dRepList[i].view} is not equal to ${await cip105DRepListFE[i].textContent()} on sort ${option}`, + }).toHaveText(`(CIP-105) ${dRepList[i].view}`); + } + }, + { name: `frontend sort validation of ${option}` } + ); } getDRepCard(dRepId: string) { return this.page.getByTestId(`${dRepId}-drep-card`); From c581c85708e6c5f8a1bb4df1b6c824206a7ffda2 Mon Sep 17 00:00:00 2001 From: Niraj Date: Mon, 24 Feb 2025 11:55:45 +0545 Subject: [PATCH 08/23] fix: wrap filter and sort validation with functionWaitedAssert --- .../lib/pages/governanceActionsPage.ts | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/tests/govtool-frontend/playwright/lib/pages/governanceActionsPage.ts b/tests/govtool-frontend/playwright/lib/pages/governanceActionsPage.ts index 8491c0143..2e093f769 100644 --- a/tests/govtool-frontend/playwright/lib/pages/governanceActionsPage.ts +++ b/tests/govtool-frontend/playwright/lib/pages/governanceActionsPage.ts @@ -4,7 +4,7 @@ import { GovernanceActionType, IProposal } from "@types"; import environments from "lib/constants/environments"; import GovernanceActionDetailsPage from "./governanceActionDetailsPage"; import { getEnumKeyByValue } from "@helpers/enum"; -import { waitedLoop } from "@helpers/waitedLoop"; +import { functionWaitedAssert, waitedLoop } from "@helpers/waitedLoop"; const MAX_SLIDES_DISPLAY_PER_TYPE = 6; @@ -106,18 +106,20 @@ export default class GovernanceActionsPage { } async validateFilters(filters: string[]) { - const proposalCards = await this.getAllProposals(); - - for (const proposalCard of proposalCards) { - const hasFilter = await this._validateFiltersInProposalCard( - proposalCard, - filters - ); - expect( - hasFilter, - "A proposal card does not contain any of the filters" - ).toBe(true); - } + await functionWaitedAssert(async () => { + const proposalCards = await this.getAllProposals(); + + for (const proposalCard of proposalCards) { + const hasFilter = await this._validateFiltersInProposalCard( + proposalCard, + filters + ); + expect( + hasFilter, + `A proposal card does not contain any of the ${filters}` + ).toBe(true); + } + }); } async sortProposal(option: string) { @@ -167,29 +169,36 @@ export default class GovernanceActionsPage { this.page.getByRole("progressbar").getByRole("img") ).toBeHidden({ timeout: 20_000 }); - // Frontend validation - for (let dIdx = 0; dIdx <= proposalsByType.length - 1; dIdx++) { - const proposals = proposalsByType[0] as IProposal[]; - const filterOptionKey = getEnumKeyByValue( - GovernanceActionType, - proposals[0].type - ); - - const slides = await this.page - .locator(`[data-testid="govaction-${filterOptionKey}-card"]`) - .all(); - - const actualSlidesInDisplay = - proposals.length > MAX_SLIDES_DISPLAY_PER_TYPE - ? MAX_SLIDES_DISPLAY_PER_TYPE - : proposals.length; - - expect(slides).toHaveLength(actualSlidesInDisplay); - - for (let i = 0; i <= slides.length - 1; i++) { - await expect(slides[i]).toContainText(`${proposals[i].txHash}`); + await functionWaitedAssert( + async () => { + // Frontend validation + for (let dIdx = 0; dIdx <= proposalsByType.length - 1; dIdx++) { + const proposals = proposalsByType[0] as IProposal[]; + const filterOptionKey = getEnumKeyByValue( + GovernanceActionType, + proposals[0].type + ); + + const slides = await this.page + .locator(`[data-testid="govaction-${filterOptionKey}-card"]`) + .all(); + + const actualSlidesInDisplay = + proposals.length > MAX_SLIDES_DISPLAY_PER_TYPE + ? MAX_SLIDES_DISPLAY_PER_TYPE + : proposals.length; + + expect(slides).toHaveLength(actualSlidesInDisplay); + + for (let i = 0; i <= slides.length - 1; i++) { + await expect(slides[i]).toContainText(`${proposals[i].txHash}`); + } + } + }, + { + name: `frontend sort validation of ${sortOption} and filter ${filterKeys}`, } - } + ); } async _validateFiltersInProposalCard( From ccd08e93a961f37480553f8df8561bc94d065cff Mon Sep 17 00:00:00 2001 From: Niraj Date: Mon, 24 Feb 2025 12:57:27 +0545 Subject: [PATCH 09/23] fix: enhance visibility checks with detailed error messages in dRep and proposal --- .../playwright/lib/forms/dRepForm.ts | 137 +++++++++++---- .../playwright/lib/pages/dRepDirectoryPage.ts | 11 +- .../lib/pages/proposalSubmissionPage.ts | 165 +++++++++++++----- .../proposalFunctionality.dRep.spec.ts | 10 +- 4 files changed, 247 insertions(+), 76 deletions(-) diff --git a/tests/govtool-frontend/playwright/lib/forms/dRepForm.ts b/tests/govtool-frontend/playwright/lib/forms/dRepForm.ts index 3985e96d3..e11d23551 100644 --- a/tests/govtool-frontend/playwright/lib/forms/dRepForm.ts +++ b/tests/govtool-frontend/playwright/lib/forms/dRepForm.ts @@ -13,7 +13,7 @@ const formErrors = { linkDescription: "max-80-characters-error", email: "invalid-email-address-error", links: { - url:"link-reference-description-1-error", + url: "link-reference-description-1-error", description: "link-reference-description-1-error", }, identity: { @@ -160,21 +160,52 @@ export default class DRepForm { for (const err of formErrors.dRepName) { await expect(this.form.getByTestId(err)).toBeHidden(); } + const objectivesInputText = await this.objectivesInput.textContent(); + const motivationsInputText = await this.motivationsInput.textContent(); + const qualificationsInputText = + await this.qualificationsInput.textContent(); + const isReferenceLinkErrorVisible = await this.form + .getByTestId(formErrors.links.url) + .isVisible(); + const isIdentityLinkErrorVisible = await this.form + .getByTestId(formErrors.identity.url) + .isVisible(); + const isPaymentAddressErrorVisible = await this.form + .getByTestId(formErrors.paymentAddress) + .isVisible(); - expect(await this.objectivesInput.textContent()).toEqual( - dRepInfo.objectives - ); + expect(await this.objectivesInput.textContent(), { + message: + objectivesInputText !== dRepInfo.objectives && + `${dRepInfo.objectives} is not equal to ${await this.objectivesInput.textContent()}`, + }).toEqual(dRepInfo.objectives); - expect(await this.motivationsInput.textContent()).toEqual( - dRepInfo.motivations - ); - expect(await this.qualificationsInput.textContent()).toEqual( - dRepInfo.qualifications - ); + expect(await this.motivationsInput.textContent(), { + message: + motivationsInputText !== dRepInfo.motivations && + `${dRepInfo.motivations} is not equal to ${await this.motivationsInput.textContent()}`, + }).toEqual(dRepInfo.motivations); + expect(await this.qualificationsInput.textContent(), { + message: + qualificationsInputText !== dRepInfo.qualifications && + `${dRepInfo.qualifications} is not equal to ${await this.qualificationsInput.textContent()}`, + }).toEqual(dRepInfo.qualifications); - await expect(this.form.getByTestId(formErrors.links.url)).toBeHidden(); - await expect(this.form.getByTestId(formErrors.identity.url)).toBeHidden(); - await expect(this.form.getByTestId(formErrors.paymentAddress)).toBeHidden(); + await expect(this.form.getByTestId(formErrors.links.url), { + message: + isReferenceLinkErrorVisible && + `${dRepInfo.linksReferenceLinks[0].url} is an invalid url`, + }).toBeHidden(); + await expect(this.form.getByTestId(formErrors.identity.url), { + message: + isIdentityLinkErrorVisible && + `${dRepInfo.identityReferenceLinks[0].url} is an invalid url`, + }).toBeHidden(); + await expect(this.form.getByTestId(formErrors.paymentAddress), { + message: + isPaymentAddressErrorVisible && + `${dRepInfo.paymentAddress} is an invalid paymentAddress`, + }).toBeHidden(); await expect(this.continueBtn).toBeEnabled(); } @@ -200,28 +231,68 @@ export default class DRepForm { expect(nameErrors.length).toBeGreaterThanOrEqual(1); - await expect( - this.form.getByTestId(formErrors.paymentAddress) - ).toBeVisible(); + const objectivesInputText = await this.objectivesInput.textContent(); + const motivationsInputText = await this.motivationsInput.textContent(); + const qualificationsInputText = + await this.qualificationsInput.textContent(); + const isReferenceLinkErrorVisible = await this.form + .getByTestId(formErrors.links.url) + .isVisible(); + const isReferenceLinkDescriptionErrorVisible = await this.form + .getByTestId(formErrors.links.description) + .isVisible(); + const isIdentityLinkErrorVisible = await this.form + .getByTestId(formErrors.identity.url) + .isVisible(); + const isIdentityLinkDescriptionErrorVisible = await this.form + .getByTestId(formErrors.identity.description) + .isVisible(); + const isPaymentAddressErrorVisible = await this.form + .getByTestId(formErrors.paymentAddress) + .isVisible(); - expect(await this.objectivesInput.textContent()).not.toEqual( - dRepInfo.objectives - ); - expect(await this.motivationsInput.textContent()).not.toEqual( - dRepInfo.qualifications - ); - expect(await this.qualificationsInput.textContent()).not.toEqual( - dRepInfo.qualifications - ); + await expect(this.form.getByTestId(formErrors.paymentAddress), { + message: + !isPaymentAddressErrorVisible && + `${dRepInfo.paymentAddress} is a valid paymentAddress`, + }).toBeVisible(); + + expect(await this.objectivesInput.textContent(), { + message: + objectivesInputText === dRepInfo.objectives && + `${dRepInfo.objectives} is equal to ${await this.objectivesInput.textContent()}`, + }).not.toEqual(dRepInfo.objectives); + expect(await this.motivationsInput.textContent(), { + message: + motivationsInputText === dRepInfo.motivations && + `${dRepInfo.motivations} is equal to ${await this.motivationsInput.textContent()}`, + }).not.toEqual(dRepInfo.qualifications); + expect(await this.qualificationsInput.textContent(), { + message: + qualificationsInputText === dRepInfo.qualifications && + `${dRepInfo.qualifications} is equal to ${await this.qualificationsInput.textContent()}`, + }).not.toEqual(dRepInfo.qualifications); - await expect(this.form.getByTestId(formErrors.links.url)).toBeVisible(); - await expect( - this.form.getByTestId(formErrors.links.description) - ).toBeVisible(); - await expect(this.form.getByTestId(formErrors.identity.url)).toBeVisible(); - await expect( - this.form.getByTestId(formErrors.identity.description) - ).toBeVisible(); + await expect(this.form.getByTestId(formErrors.links.url), { + message: + !isReferenceLinkErrorVisible && + `${dRepInfo.linksReferenceLinks[0].url} is a valid url`, + }).toBeVisible(); + await expect(this.form.getByTestId(formErrors.links.description), { + message: + !isReferenceLinkDescriptionErrorVisible && + `${dRepInfo.linksReferenceLinks[0].description} is a valid description`, + }).toBeVisible(); + await expect(this.form.getByTestId(formErrors.identity.url), { + message: + !isIdentityLinkErrorVisible && + `${dRepInfo.identityReferenceLinks[0].url} is a valid url`, + }).toBeVisible(); + await expect(this.form.getByTestId(formErrors.identity.description), { + message: + !isIdentityLinkDescriptionErrorVisible && + `${dRepInfo.identityReferenceLinks[0].description} is a valid description`, + }).toBeVisible(); await expect(this.continueBtn).toBeDisabled(); } diff --git a/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts b/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts index 5ce76d7f9..572865992 100644 --- a/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts +++ b/tests/govtool-frontend/playwright/lib/pages/dRepDirectoryPage.ts @@ -199,8 +199,15 @@ export default class DRepDirectoryPage { await this.goto(); await this.searchInput.fill(dRepId); - - await expect(this.page.getByText("No DReps found")).toBeVisible({ + const isEmptyContainerVisible = await this.page + .getByText("No DReps found") + .isVisible(); + + await expect(this.page.getByText("No DReps found"), { + message: + !isEmptyContainerVisible && + `DRep with id ${dRepId} is found in the list`, + }).toBeVisible({ timeout: 20_000, }); } diff --git a/tests/govtool-frontend/playwright/lib/pages/proposalSubmissionPage.ts b/tests/govtool-frontend/playwright/lib/pages/proposalSubmissionPage.ts index 32f48e4f4..25739eb5c 100644 --- a/tests/govtool-frontend/playwright/lib/pages/proposalSubmissionPage.ts +++ b/tests/govtool-frontend/playwright/lib/pages/proposalSubmissionPage.ts @@ -235,7 +235,7 @@ export default class ProposalSubmissionPage { async getAllDrafts() { await expect( this.page.locator('[data-testid^="draft-"][data-testid$="-card"]') - ).toBeVisible({ timeout: 10_000 }); // slow rendering + ).toBeVisible({ timeout: 60_000 }); // slow rendering return await this.page .locator('[data-testid^="draft-"][data-testid$="-card"]') @@ -245,7 +245,7 @@ export default class ProposalSubmissionPage { async getFirstDraft() { await expect( this.page.locator('[data-testid^="draft-"][data-testid$="-card"]').first() - ).toBeVisible({ timeout: 10_000 }); // slow rendering + ).toBeVisible({ timeout: 60_000 }); // slow rendering return this.page .locator('[data-testid^="draft-"][data-testid$="-card"]') @@ -270,39 +270,80 @@ export default class ProposalSubmissionPage { await expect(this.page.getByTestId(err)).toBeHidden(); } - expect(await this.abstractInput.textContent()).toEqual( - governanceProposal.prop_abstract - ); - - expect(await this.rationaleInput.textContent()).toEqual( - governanceProposal.prop_rationale - ); - - expect(await this.motivationInput.textContent()).toEqual( - governanceProposal.prop_motivation - ); + const abstractInputContent = await this.abstractInput.textContent(); + const rationaleInputContent = await this.rationaleInput.textContent(); + const motivationInputContent = await this.motivationInput.textContent(); + const isLinkErrorVisible = await this.page + .getByTestId(formErrors.link) + .isVisible(); + + expect(await this.abstractInput.textContent(), { + message: + abstractInputContent !== governanceProposal.prop_abstract && + `${governanceProposal.prop_abstract} is not equal to ${await this.abstractInput.textContent()}`, + }).toEqual(governanceProposal.prop_abstract); + + expect(await this.rationaleInput.textContent(), { + message: + rationaleInputContent !== governanceProposal.prop_rationale && + `${governanceProposal.prop_rationale} is not equal to ${await this.rationaleInput.textContent()}`, + }).toEqual(governanceProposal.prop_rationale); + + expect(await this.motivationInput.textContent(), { + message: + motivationInputContent !== governanceProposal.prop_motivation && + `${governanceProposal.prop_motivation} is not equal to ${await this.motivationInput.textContent()}`, + }).toEqual(governanceProposal.prop_motivation); if (governanceProposal.gov_action_type_id === 1) { - await expect( - this.page.getByTestId(formErrors.receivingAddress) - ).toBeHidden(); + const isReceivingAddressErrorVisible = await this.page + .getByTestId(formErrors.receivingAddress) + .isVisible(); + const isAmountErrorVisible = await this.page + .getByTestId(formErrors.amount) + .isVisible(); + + await expect(this.page.getByTestId(formErrors.receivingAddress), { + message: + isReceivingAddressErrorVisible && + `${governanceProposal.prop_receiving_address} is an invalid receiving address`, + }).toBeHidden(); for (const err of formErrors.amount) { - await expect(this.page.getByTestId(err)).toBeHidden(); + await expect(this.page.getByTestId(err), { + message: + isAmountErrorVisible && + `${governanceProposal.prop_amount} is an invalid amount`, + }).toBeHidden(); } } if (governanceProposal.gov_action_type_id === 2) { - await expect( - this.page.getByTestId(formErrors.constitutionalUrl) - ).toBeHidden(); - - await expect( - this.page.getByTestId(formErrors.guardrailsScriptUrl) - ).toBeHidden(); + const isConstitutionalUrlErrorVisible = await this.page + .getByTestId(formErrors.constitutionalUrl) + .isVisible(); + const isGuardrailsScriptUrlErrorVisible = await this.page + .getByTestId(formErrors.guardrailsScriptUrl) + .isVisible(); + + await expect(this.page.getByTestId(formErrors.constitutionalUrl), { + message: + isConstitutionalUrlErrorVisible && + `${governanceProposal.prop_constitution_url} is an invalid constitution url`, + }).toBeHidden(); + + await expect(this.page.getByTestId(formErrors.guardrailsScriptUrl), { + message: + isGuardrailsScriptUrlErrorVisible && + `${governanceProposal.prop_guardrails_script_url} is an invalid guardrails script url`, + }).toBeHidden(); } - await expect(this.page.getByTestId(formErrors.link)).toBeHidden(); + await expect(this.page.getByTestId(formErrors.link), { + message: + isLinkErrorVisible && + `${governanceProposal.proposal_links[0].prop_link} is an invalid link`, + }).toBeHidden(); await expect(this.continueBtn).toBeEnabled(); } @@ -313,8 +354,14 @@ export default class ProposalSubmissionPage { value: string, logMessage: string ) { + const isErrorFieldVisible = await this.page + .getByTestId(errorField) + .isVisible(); if (value === " ") { - await expect(this.page.getByTestId(errorField)).toBeVisible(); + await expect(this.page.getByTestId(errorField), { + message: + !isErrorFieldVisible && `Empty value is assumed as ${logMessage}`, + }).toBeVisible(); } else { expectWithInfo( async () => expect(await input.textContent()).not.toEqual(value), @@ -325,7 +372,18 @@ export default class ProposalSubmissionPage { async inValidateForm(governanceProposal: ProposalCreateRequest) { await this.fillupFormWithTypeSelected(governanceProposal); - await expect(this.page.getByTestId(formErrors.proposalTitle)).toBeVisible(); + const isProposalTitleErrorVisible = await this.page + .getByTestId(formErrors.proposalTitle) + .isVisible(); + const isLinkErrorVisible = await this.page + .getByTestId(formErrors.link) + .isVisible(); + + await expect(this.page.getByTestId(formErrors.proposalTitle), { + message: + !isProposalTitleErrorVisible && + `${governanceProposal.prop_name} is a valid proposal title`, + }).toBeVisible(); await this.assertFieldValidation( this.abstractInput, @@ -346,24 +404,51 @@ export default class ProposalSubmissionPage { "valid rationale" ); - await expect(this.page.getByTestId(formErrors.link)).toBeVisible(); + await expect(this.page.getByTestId(formErrors.link), { + message: + !isLinkErrorVisible && + `${governanceProposal.proposal_links[0].prop_link} is a valid link`, + }).toBeVisible(); if (governanceProposal.gov_action_type_id === 1) { - await expect( - this.page.getByTestId(formErrors.receivingAddress) - ).toBeVisible(); - - await expect(this.page.getByTestId(formErrors.amount)).toBeVisible(); + const isReceivingAddressErrorVisible = await this.page + .getByTestId(formErrors.receivingAddress) + .isVisible(); + const isAmountErrorVisible = await this.page + .getByTestId(formErrors.amount) + .isVisible(); + + await expect(this.page.getByTestId(formErrors.receivingAddress), { + message: + !isReceivingAddressErrorVisible && + `${governanceProposal.prop_receiving_address} is a valid receiving address`, + }).toBeVisible(); + + await expect(this.page.getByTestId(formErrors.amount), { + message: + !isAmountErrorVisible && + `${governanceProposal.prop_amount} is a valid amount`, + }).toBeVisible(); } if (governanceProposal.gov_action_type_id === 2) { - await expect( - this.page.getByTestId(formErrors.constitutionalUrl) - ).toBeVisible(); - - await expect( - this.page.getByTestId(formErrors.guardrailsScriptUrl) - ).toBeVisible(); + const isConstitutionalUrlErrorVisible = await this.page + .getByTestId(formErrors.constitutionalUrl) + .isVisible(); + const isGuardrailsScriptUrlErrorVisible = await this.page + .getByTestId(formErrors.guardrailsScriptUrl) + .isVisible(); + await expect(this.page.getByTestId(formErrors.constitutionalUrl), { + message: + !isConstitutionalUrlErrorVisible && + `${governanceProposal.prop_constitution_url} is a valid constitution url`, + }).toBeVisible(); + + await expect(this.page.getByTestId(formErrors.guardrailsScriptUrl), { + message: + !isGuardrailsScriptUrlErrorVisible && + `${governanceProposal.prop_guardrails_script_url} is a valid guardrails script url`, + }).toBeVisible(); } await expect(this.continueBtn).toBeDisabled(); diff --git a/tests/govtool-frontend/playwright/tests/5-proposal-functionality/proposalFunctionality.dRep.spec.ts b/tests/govtool-frontend/playwright/tests/5-proposal-functionality/proposalFunctionality.dRep.spec.ts index f69a3fa54..ea81ec51c 100644 --- a/tests/govtool-frontend/playwright/tests/5-proposal-functionality/proposalFunctionality.dRep.spec.ts +++ b/tests/govtool-frontend/playwright/tests/5-proposal-functionality/proposalFunctionality.dRep.spec.ts @@ -186,8 +186,16 @@ test.describe("Perform voting", () => { await govActionDetailsPage.reVote(); await governanceActionsPage.votedTab.click(); + const isNoVoteVisible = await govActionDetailsPage.currentPage + .getByTestId("my-vote") + .getByText("No") + .isVisible(); + await expect( - govActionDetailsPage.currentPage.getByTestId("my-vote").getByText("No") + govActionDetailsPage.currentPage.getByTestId("my-vote").getByText("No"), + { + message: !isNoVoteVisible && "No vote not visible", + } ).toBeVisible({ timeout: 60_000 }); }); From 4e0f73e62158458c3a0c4b58a69ee48fd3cfb9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Sza=C5=82owski?= Date: Mon, 24 Feb 2025 09:39:32 +0100 Subject: [PATCH 10/23] fix(#3055, #3057): Top nav adjustments --- CHANGELOG.md | 4 +++- govtool/frontend/src/components/organisms/TopNav.tsx | 2 +- govtool/frontend/src/consts/navItems.tsx | 12 ++++++------ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 259b34714..252d08f85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,13 +14,15 @@ changes. ### Fixed +- Fix responsive error on menu [Issue 3055](https://github.com/IntersectMBO/govtool/issues/3055) +- Fix wrong placement of nav items in disconnected menu [Issue 3057](https://github.com/IntersectMBO/govtool/issues/3057) + ### Changed ### Removed ## [v2.0.12](https://github.com/IntersectMBO/govtool/releases/tag/v2.0.12) 2025-02-21 - ### Added - Add metadata url and hash to drep details [Issue 2911](https://github.com/IntersectMBO/govtool/issues/2911) diff --git a/govtool/frontend/src/components/organisms/TopNav.tsx b/govtool/frontend/src/components/organisms/TopNav.tsx index a9ef5f064..5a85776cf 100644 --- a/govtool/frontend/src/components/organisms/TopNav.tsx +++ b/govtool/frontend/src/components/organisms/TopNav.tsx @@ -179,7 +179,7 @@ export const TopNav = ({ isConnectButton = true }) => { src={IMAGES.appLogo} /> - {screenWidth >= 1024 ? renderDesktopNav() : renderMobileNav()} + {screenWidth >= 1145 ? renderDesktopNav() : renderMobileNav()} ); diff --git a/govtool/frontend/src/consts/navItems.tsx b/govtool/frontend/src/consts/navItems.tsx index 1eee2e44d..ca6319782 100644 --- a/govtool/frontend/src/consts/navItems.tsx +++ b/govtool/frontend/src/consts/navItems.tsx @@ -24,6 +24,12 @@ export const NAV_ITEMS = [ navTo: PATHS.dRepDirectory, label: i18n.t("dRepDirectory.title"), }, + { + dataTestId: "proposed-governance-actions-link", + navTo: PDF_PATHS.proposalDiscussion, + label: i18n.t("proposalDiscussion.title"), + newTabLink: null, + }, { dataTestId: "governance-actions-link", navTo: PATHS.governanceActions, @@ -36,12 +42,6 @@ export const NAV_ITEMS = [ navTo: OUTCOMES_PATHS.governanceActionsOutcomes, newTabLink: null, }, - { - dataTestId: "proposed-governance-actions-link", - navTo: PDF_PATHS.proposalDiscussion, - label: i18n.t("proposalDiscussion.title"), - newTabLink: null, - }, { dataTestId: "guides-link", navTo: "", From fc0eb18d470ae62fafc79cc22ad0ed873aa6d912 Mon Sep 17 00:00:00 2001 From: Emmanuel Mutisya Date: Fri, 21 Feb 2025 23:27:26 +0300 Subject: [PATCH 11/23] fix(outcomes): add outcomes title and upgrade ui package version --- govtool/frontend/.env.example | 1 + govtool/frontend/package-lock.json | 12 ++++++------ govtool/frontend/package.json | 2 +- govtool/frontend/src/pages/Dashboard.tsx | 17 ++++++++++++++++- .../src/pages/GovernanceActionOutComes.tsx | 4 +++- govtool/frontend/yarn.lock | 9 ++++----- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/govtool/frontend/.env.example b/govtool/frontend/.env.example index 5050248ee..db8d2ae78 100644 --- a/govtool/frontend/.env.example +++ b/govtool/frontend/.env.example @@ -9,5 +9,6 @@ VITE_USERSNAP_SPACE_API_KEY="" VITE_IS_PROPOSAL_DISCUSSION_FORUM_ENABLED='true' VITE_IS_GOVERNANCE_OUTCOMES_PILLAR_ENABLED='true' VITE_PDF_API_URL="" +VITE_OUTCOMES_API_URL="" VITE_IPFS_GATEWAY="" VITE_IPFS_PROJECT_ID="" \ No newline at end of file diff --git a/govtool/frontend/package-lock.json b/govtool/frontend/package-lock.json index c7d901ef2..e50585ed3 100644 --- a/govtool/frontend/package-lock.json +++ b/govtool/frontend/package-lock.json @@ -13,7 +13,7 @@ "@emotion/styled": "^11.11.0", "@emurgo/cardano-serialization-lib-asmjs": "^12.1.1", "@hookform/resolvers": "^3.3.1", - "@intersect.mbo/govtool-outcomes-pillar-ui": "1.2.0", + "@intersect.mbo/govtool-outcomes-pillar-ui": "1.2.1", "@intersect.mbo/intersectmbo.org-icons-set": "^1.0.8", "@intersect.mbo/pdf-ui": "0.6.2", "@mui/icons-material": "^5.14.3", @@ -3372,17 +3372,16 @@ } }, "node_modules/@intersect.mbo/govtool-outcomes-pillar-ui": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@intersect.mbo/govtool-outcomes-pillar-ui/-/govtool-outcomes-pillar-ui-1.2.0.tgz", - "integrity": "sha512-AzY2pOAExXCz7M6S2YeIglZ06Gdf0N9gWxJBRDQ2BuDsRC2arn4DwT2F+XHyOhM2AWSZeql0Dx86a0pZP7a1+w==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@intersect.mbo/govtool-outcomes-pillar-ui/-/govtool-outcomes-pillar-ui-1.2.1.tgz", + "integrity": "sha512-LQShglbHzZY3NiqkLMSdFKrrHNGV/vqDkHy8fYtexX4OgH1+qDCMFArxvjCfHS6NNxUk8PdpOWkeVsW38ZCUpg==", "license": "ISC", "dependencies": { "@fontsource/poppins": "^5.0.14", "@intersect.mbo/intersectmbo.org-icons-set": "^1.0.8", "axios": "^1.7.9", "bech32": "^2.0.0", - "buffer": "^6.0.3", - "react-query": "^3.39.3" + "buffer": "^6.0.3" }, "peerDependencies": { "@emotion/react": "^11.11.4", @@ -3391,6 +3390,7 @@ "react": "^18.3.1", "react-dom": "^18.3.1", "react-markdown": "^9.0.1", + "react-query": "^3.39.3", "react-router-dom": "^6.23.1", "sass": "^1.77.2" } diff --git a/govtool/frontend/package.json b/govtool/frontend/package.json index 09218386d..c767f5b38 100644 --- a/govtool/frontend/package.json +++ b/govtool/frontend/package.json @@ -27,7 +27,7 @@ "@emotion/styled": "^11.11.0", "@emurgo/cardano-serialization-lib-asmjs": "^12.1.1", "@hookform/resolvers": "^3.3.1", - "@intersect.mbo/govtool-outcomes-pillar-ui": "1.2.0", + "@intersect.mbo/govtool-outcomes-pillar-ui": "1.2.1", "@intersect.mbo/intersectmbo.org-icons-set": "^1.0.8", "@intersect.mbo/pdf-ui": "0.6.2", "@mui/icons-material": "^5.14.3", diff --git a/govtool/frontend/src/pages/Dashboard.tsx b/govtool/frontend/src/pages/Dashboard.tsx index 497b34413..684558337 100644 --- a/govtool/frontend/src/pages/Dashboard.tsx +++ b/govtool/frontend/src/pages/Dashboard.tsx @@ -3,7 +3,12 @@ import { useLocation, Outlet, useNavigate } from "react-router-dom"; import { Box } from "@mui/material"; import { Background, ScrollToManage } from "@atoms"; -import { CONNECTED_NAV_ITEMS, DRAWER_WIDTH, PATHS } from "@consts"; +import { + CONNECTED_NAV_ITEMS, + DRAWER_WIDTH, + OUTCOMES_PATHS, + PATHS, +} from "@consts"; import { useCardano } from "@context"; import { useScreenDimension, useTranslation } from "@hooks"; import { DashboardTopNav, Drawer, Footer } from "@organisms"; @@ -19,6 +24,16 @@ export const Dashboard = () => { const getPageTitle = (path: string) => { if (path === PATHS.dashboard) return t("dashboard.title"); + + if (path.startsWith(OUTCOMES_PATHS.governanceActionsOutcomes)) { + const outcomesNavItem = findNavItem( + CONNECTED_NAV_ITEMS, + OUTCOMES_PATHS.governanceActionsOutcomes, + ); + + return outcomesNavItem ?? ""; + } + return findNavItem(CONNECTED_NAV_ITEMS, path) ?? ""; }; diff --git a/govtool/frontend/src/pages/GovernanceActionOutComes.tsx b/govtool/frontend/src/pages/GovernanceActionOutComes.tsx index 27548f6e2..cf703d9b0 100644 --- a/govtool/frontend/src/pages/GovernanceActionOutComes.tsx +++ b/govtool/frontend/src/pages/GovernanceActionOutComes.tsx @@ -43,7 +43,9 @@ export const GovernanceActionOutComesPillar = () => { } > - + {!context.isEnabled &&