From fd78c1947c2f496dd2914914b6377e85b81a31bb Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 6 Mar 2025 15:37:20 +0100 Subject: [PATCH 01/12] Add docker image build on workflow dispatch --- .github/workflows/build.yaml | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/build.yaml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 00000000..d4a9ccc3 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,42 @@ +name: Build images +on: + workflow_dispatch: + release: + types: [published] + +env: + AWS_REGION: eu-west-1 + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.MM_SERVER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.MM_SERVER_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + - name: Build, tag, and push CE images to Amazon ECR + env: + BUILD_HASH: ${{ github.sha }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: mergin + run: | + IMAGE_TAG="build-ce-$GITHUB_SHA" + echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV" + cd server + docker build -f Dockerfile . --build-arg BUILD_HASH=${BUILD_HASH} -t $ECR_REGISTRY/mergin/$REPOSITORY-back:$IMAGE_TAG; + cd ../web-app + docker build -f Dockerfile . -t $ECR_REGISTRY/mergin/$REPOSITORY-front:$IMAGE_TAG + docker push $ECR_REGISTRY/mergin/$REPOSITORY-back:$IMAGE_TAG + docker push $ECR_REGISTRY/mergin/$REPOSITORY-front:$IMAGE_TAG + - name: Build info + run: | + echo "IMAGE_TAG=$IMAGE_TAG" + echo "GITHUB_SHA=$GITHUB_SHA" + echo "TARGET_BRANCH=${{ github.ref_name }}" From 29ccd4f8f6834e158157e989df62c85f38e25203 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Fri, 7 Mar 2025 09:38:32 +0100 Subject: [PATCH 02/12] Push dev images to dedicated ECR repository --- .github/workflows/build.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d4a9ccc3..28a7fb65 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -25,16 +25,16 @@ jobs: env: BUILD_HASH: ${{ github.sha }} ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} - REPOSITORY: mergin + REPOSITORY: mergin/mergin-ce run: | - IMAGE_TAG="build-ce-$GITHUB_SHA" + IMAGE_TAG="build-$GITHUB_SHA" echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV" cd server - docker build -f Dockerfile . --build-arg BUILD_HASH=${BUILD_HASH} -t $ECR_REGISTRY/mergin/$REPOSITORY-back:$IMAGE_TAG; + docker build -f Dockerfile . --build-arg BUILD_HASH=${BUILD_HASH} -t $ECR_REGISTRY/$REPOSITORY-back:$IMAGE_TAG; cd ../web-app - docker build -f Dockerfile . -t $ECR_REGISTRY/mergin/$REPOSITORY-front:$IMAGE_TAG - docker push $ECR_REGISTRY/mergin/$REPOSITORY-back:$IMAGE_TAG - docker push $ECR_REGISTRY/mergin/$REPOSITORY-front:$IMAGE_TAG + docker build -f Dockerfile . -t $ECR_REGISTRY/$REPOSITORY-front:$IMAGE_TAG + docker push $ECR_REGISTRY/$REPOSITORY-back:$IMAGE_TAG + docker push $ECR_REGISTRY/$REPOSITORY-front:$IMAGE_TAG - name: Build info run: | echo "IMAGE_TAG=$IMAGE_TAG" From f181ce9644002cc6ae87da52833f079bacce38c7 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Mar 2025 09:39:50 +0100 Subject: [PATCH 03/12] Update version to 2025.2.2 --- server/mergin/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/version.py b/server/mergin/version.py index acd1cb0b..efbe6d52 100644 --- a/server/mergin/version.py +++ b/server/mergin/version.py @@ -4,4 +4,4 @@ def get_version(): - return "2025.2.1" + return "2025.2.2" From 7bfb1956c502f0c96687a8e59f08f8a6000fea72 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Mar 2025 10:00:52 +0100 Subject: [PATCH 04/12] Bump version in setup.py --- server/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/setup.py b/server/setup.py index 16252a16..fb67ae9a 100644 --- a/server/setup.py +++ b/server/setup.py @@ -6,7 +6,7 @@ setup( name="mergin", - version="2025.2.1", + version="2025.2.2", url="https://github.com/MerginMaps/mergin", license="AGPL-3.0-only", author="Lutra Consulting Limited", From 93c4837b0819897bbcb23019ea7a88655f796b5d Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Fri, 7 Mar 2025 10:20:12 +0100 Subject: [PATCH 05/12] chore: update secrets --- .github/workflows/build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 28a7fb65..e5b85bb7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -15,8 +15,8 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-access-key-id: ${{ secrets.MM_SERVER_AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.MM_SERVER_AWS_SECRET_ACCESS_KEY }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_ACCESS_PASSWORD }} aws-region: ${{ env.AWS_REGION }} - name: Login to Amazon ECR id: login-ecr From d8043196f7406ef4d209d861bd539505eebc5401 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Mon, 10 Mar 2025 08:39:39 +0100 Subject: [PATCH 06/12] Chore: Publish docker images to dockerhub --- .github/workflows/publish.yaml | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/publish.yaml diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 00000000..f8821d00 --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,40 @@ +name: Build and publish public images +on: + workflow_dispatch: + +env: + REGISTRY: docker.io + REPOSITORY: lutraconsulting/merginmaps + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build, tag, and push CE images to Dockerhub + env: + BUILD_HASH: ${{ github.sha }} + run: | + IMAGE_TAG=${{ github.ref_name }} + if [[ $GITHUB_REF_TYPE != "tag" ]]; then + echo "Not a tag, exiting" + exit 1 + fi + echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV" + cd server + docker build -f Dockerfile . --build-arg BUILD_HASH=${BUILD_HASH} -t $REGISTRY/$REPOSITORY-backend:$IMAGE_TAG; + cd ../web-app + docker build -f Dockerfile . -t $REGISTRY/$REPOSITORY-frontend:$IMAGE_TAG + docker push $REGISTRY/$REPOSITORY-backend:$IMAGE_TAG + docker push $REGISTRY/$REPOSITORY-frontend:$IMAGE_TAG + - name: Build info + run: | + echo "IMAGE_TAG=$IMAGE_TAG" + echo "GITHUB_SHA=$GITHUB_SHA" From 72815676cee1b0465e8c0eab28ec5f8482b164fb Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Mon, 10 Mar 2025 10:45:37 +0100 Subject: [PATCH 07/12] move variable to env in gh action --- .github/workflows/publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index f8821d00..bb7d5a39 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -21,8 +21,8 @@ jobs: - name: Build, tag, and push CE images to Dockerhub env: BUILD_HASH: ${{ github.sha }} + IMAGE_TAG: ${{ github.ref_name }} run: | - IMAGE_TAG=${{ github.ref_name }} if [[ $GITHUB_REF_TYPE != "tag" ]]; then echo "Not a tag, exiting" exit 1 From 62c2a956ac0c91b2f85b8f1788d244c7b973f977 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Tue, 11 Mar 2025 09:26:03 +0100 Subject: [PATCH 08/12] Bump new public images in docker compose --- docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3c4162bf..fd19df78 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,13 +15,13 @@ services: volumes: - ./mergin_db:/var/lib/postgresql/data redis: - image: redis + image: redis:6.2.17 container_name: merginmaps-redis restart: always networks: - merginmaps server-gunicorn: - image: lutraconsulting/merginmaps-backend:2024.2.2 + image: lutraconsulting/merginmaps-backend:2025.2.2 container_name: merginmaps-server restart: always user: 901:999 @@ -37,7 +37,7 @@ services: networks: - merginmaps celery-beat: - image: lutraconsulting/merginmaps-backend:2024.2.2 + image: lutraconsulting/merginmaps-backend:2025.2.2 container_name: celery-beat restart: always env_file: @@ -54,7 +54,7 @@ services: networks: - merginmaps celery-worker: - image: lutraconsulting/merginmaps-backend:2024.2.2 + image: lutraconsulting/merginmaps-backend:2025.2.2 container_name: celery-worker restart: always user: 901:999 @@ -74,7 +74,7 @@ services: networks: - merginmaps web: - image: lutraconsulting/merginmaps-frontend:2024.2.2 + image: lutraconsulting/merginmaps-frontend:2025.2.2 container_name: merginmaps-web restart: always depends_on: From 20869df9968065b51636334ff2df06922f7d69d4 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 20 Mar 2025 15:47:25 +0100 Subject: [PATCH 09/12] Fix access request emails: - introduce get_email_receivers method which will get users able to get mails about project --- server/mergin/sync/interfaces.py | 6 +++ server/mergin/sync/private_api_controller.py | 13 +------ server/mergin/sync/project_handler.py | 22 +++++++++++ server/mergin/tests/test_project_handler.py | 41 +++++++++++++++++++- server/mergin/tests/utils.py | 2 +- 5 files changed, 70 insertions(+), 14 deletions(-) diff --git a/server/mergin/sync/interfaces.py b/server/mergin/sync/interfaces.py index aa916b38..f57f103c 100644 --- a/server/mergin/sync/interfaces.py +++ b/server/mergin/sync/interfaces.py @@ -188,6 +188,12 @@ def get_push_permission(self, changes: dict): """ pass + def get_email_receivers(self, project_id: str): + """ + Return list of members who should receive email notifications about project changes + """ + pass + class WorkspaceRole(Enum): GUEST = "guest" diff --git a/server/mergin/sync/private_api_controller.py b/server/mergin/sync/private_api_controller.py index 00d0a13d..24b30596 100644 --- a/server/mergin/sync/private_api_controller.py +++ b/server/mergin/sync/private_api_controller.py @@ -17,8 +17,6 @@ AccessRequest, ProjectRole, RequestStatus, - ProjectVersion, - ProjectUser, ) from .schemas import ( ProjectListSchema, @@ -64,16 +62,7 @@ def create_project_access_request(namespace, project_name): # noqa: E501 db.session.add(access_request) db.session.commit() # notify project owners - owners = ( - User.query.join(UserProfile, ProjectUser) - .filter( - ProjectUser.project_id == project.id, - ProjectUser.role == ProjectRole.OWNER.value, - User.verified_email, - UserProfile.receive_notifications, - ) - .all() - ) + owners = current_app.project_handler.get_email_receivers(project.id) for owner in owners: email_data = { "subject": "Project access requested", diff --git a/server/mergin/sync/project_handler.py b/server/mergin/sync/project_handler.py index 8bf17af8..0e6eada5 100644 --- a/server/mergin/sync/project_handler.py +++ b/server/mergin/sync/project_handler.py @@ -1,7 +1,29 @@ +from .models import ProjectRole, ProjectUser from .interfaces import AbstractProjectHandler from .permissions import ProjectPermissions +from sqlalchemy import or_, and_ +from typing import List +from ..auth.models import User, UserProfile class ProjectHandler(AbstractProjectHandler): def get_push_permission(self, changes: dict): return ProjectPermissions.Upload + + def get_email_receivers(self, project_id: str) -> List[User]: + return ( + User.query.join(UserProfile) + .outerjoin(ProjectUser, ProjectUser.user_id == User.id) + .filter( + or_( + and_( + ProjectUser.project_id == project_id, + ProjectUser.role == ProjectRole.OWNER.value, + ), + User.is_admin, + ), + User.verified_email, + UserProfile.receive_notifications, + ) + .all() + ) diff --git a/server/mergin/tests/test_project_handler.py b/server/mergin/tests/test_project_handler.py index d44bc5f5..0b2ebe81 100644 --- a/server/mergin/tests/test_project_handler.py +++ b/server/mergin/tests/test_project_handler.py @@ -1,8 +1,47 @@ +from ..sync.models import Project, ProjectRole +from .utils import add_user, create_project, create_workspace from ..sync.project_handler import ProjectHandler from ..sync.permissions import ProjectPermissions +from ..auth.models import User +from ..app import db -def test_project_handler(): + +def test_project_permissions(client): project_handler = ProjectHandler() project_permission = project_handler.get_push_permission(None) assert project_permission == ProjectPermissions.Upload + + +def test_email_receivers(client): + project_handler = ProjectHandler() + # test project email receivers (owners and super admins) + workspace = create_workspace() + user = add_user() + project = create_project("test_project", workspace, user) + project.set_role(user.id, ProjectRole.READER) + db.session.commit() + receivers = project_handler.get_email_receivers(project.id) + assert len(receivers) == 1 + + project.set_role(user.id, ProjectRole.OWNER) + db.session.commit() + receivers = project_handler.get_email_receivers(project.id) + assert len(receivers) == 2 + + user.verified_email = False + db.session.commit() + receivers = project_handler.get_email_receivers(project.id) + assert len(receivers) == 1 + + user.verified_email = True + user.profile.receive_notifications = False + db.session.commit() + receivers = project_handler.get_email_receivers(project.id) + assert len(receivers) == 1 + + admin = User.query.filter(User.username == "mergin").first() + admin.is_admin = False + db.session.commit() + receivers = project_handler.get_email_receivers(project.id) + assert len(receivers) == 0 diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index 0d4d4f9c..0f768c6e 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -28,7 +28,7 @@ CHUNK_SIZE = 1024 -def add_user(username="random", password="random", is_admin=False): +def add_user(username="random", password="random", is_admin=False) -> User: """Helper function to create not-privileged user. Associated user workspace is created with db hook. From 8d3b7d3faed79badf62228ad6501dfaccfd1e770 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 20 Mar 2025 17:40:40 +0100 Subject: [PATCH 10/12] Handle project instead of project_id --- server/mergin/sync/interfaces.py | 2 +- server/mergin/sync/private_api_controller.py | 2 +- server/mergin/sync/project_handler.py | 6 +++--- server/mergin/tests/test_project_handler.py | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/server/mergin/sync/interfaces.py b/server/mergin/sync/interfaces.py index f57f103c..bb2e9843 100644 --- a/server/mergin/sync/interfaces.py +++ b/server/mergin/sync/interfaces.py @@ -188,7 +188,7 @@ def get_push_permission(self, changes: dict): """ pass - def get_email_receivers(self, project_id: str): + def get_email_receivers(self, project): """ Return list of members who should receive email notifications about project changes """ diff --git a/server/mergin/sync/private_api_controller.py b/server/mergin/sync/private_api_controller.py index 24b30596..643c274c 100644 --- a/server/mergin/sync/private_api_controller.py +++ b/server/mergin/sync/private_api_controller.py @@ -62,7 +62,7 @@ def create_project_access_request(namespace, project_name): # noqa: E501 db.session.add(access_request) db.session.commit() # notify project owners - owners = current_app.project_handler.get_email_receivers(project.id) + owners = current_app.project_handler.get_email_receivers(project) for owner in owners: email_data = { "subject": "Project access requested", diff --git a/server/mergin/sync/project_handler.py b/server/mergin/sync/project_handler.py index 0e6eada5..b4f1b09f 100644 --- a/server/mergin/sync/project_handler.py +++ b/server/mergin/sync/project_handler.py @@ -1,4 +1,4 @@ -from .models import ProjectRole, ProjectUser +from .models import ProjectRole, ProjectUser, Project from .interfaces import AbstractProjectHandler from .permissions import ProjectPermissions from sqlalchemy import or_, and_ @@ -10,14 +10,14 @@ class ProjectHandler(AbstractProjectHandler): def get_push_permission(self, changes: dict): return ProjectPermissions.Upload - def get_email_receivers(self, project_id: str) -> List[User]: + def get_email_receivers(self, project: Project) -> List[User]: return ( User.query.join(UserProfile) .outerjoin(ProjectUser, ProjectUser.user_id == User.id) .filter( or_( and_( - ProjectUser.project_id == project_id, + ProjectUser.project_id == project.id, ProjectUser.role == ProjectRole.OWNER.value, ), User.is_admin, diff --git a/server/mergin/tests/test_project_handler.py b/server/mergin/tests/test_project_handler.py index 0b2ebe81..e49cb119 100644 --- a/server/mergin/tests/test_project_handler.py +++ b/server/mergin/tests/test_project_handler.py @@ -21,27 +21,27 @@ def test_email_receivers(client): project = create_project("test_project", workspace, user) project.set_role(user.id, ProjectRole.READER) db.session.commit() - receivers = project_handler.get_email_receivers(project.id) + receivers = project_handler.get_email_receivers(project) assert len(receivers) == 1 project.set_role(user.id, ProjectRole.OWNER) db.session.commit() - receivers = project_handler.get_email_receivers(project.id) + receivers = project_handler.get_email_receivers(project) assert len(receivers) == 2 user.verified_email = False db.session.commit() - receivers = project_handler.get_email_receivers(project.id) + receivers = project_handler.get_email_receivers(project) assert len(receivers) == 1 user.verified_email = True user.profile.receive_notifications = False db.session.commit() - receivers = project_handler.get_email_receivers(project.id) + receivers = project_handler.get_email_receivers(project) assert len(receivers) == 1 admin = User.query.filter(User.username == "mergin").first() admin.is_admin = False db.session.commit() - receivers = project_handler.get_email_receivers(project.id) + receivers = project_handler.get_email_receivers(project) assert len(receivers) == 0 From 11fb9dc040aec4b4ee204756621bbe35c33a73ce Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 21 Mar 2025 10:00:20 +0100 Subject: [PATCH 11/12] Add User.active condition --- server/mergin/sync/project_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/server/mergin/sync/project_handler.py b/server/mergin/sync/project_handler.py index b4f1b09f..8299935a 100644 --- a/server/mergin/sync/project_handler.py +++ b/server/mergin/sync/project_handler.py @@ -22,6 +22,7 @@ def get_email_receivers(self, project: Project) -> List[User]: ), User.is_admin, ), + User.active, User.verified_email, UserProfile.receive_notifications, ) From 208c3f2794f944cc8f4ffe3297889373ed7ba5ff Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 21 Mar 2025 10:43:13 +0100 Subject: [PATCH 12/12] Add test for inactive user --- server/mergin/tests/test_project_handler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/server/mergin/tests/test_project_handler.py b/server/mergin/tests/test_project_handler.py index e49cb119..76040ca8 100644 --- a/server/mergin/tests/test_project_handler.py +++ b/server/mergin/tests/test_project_handler.py @@ -40,6 +40,12 @@ def test_email_receivers(client): receivers = project_handler.get_email_receivers(project) assert len(receivers) == 1 + user.profile.receive_notifications = True + user.active = False + db.session.commit() + receivers = project_handler.get_email_receivers(project) + assert len(receivers) == 1 + admin = User.query.filter(User.username == "mergin").first() admin.is_admin = False db.session.commit()