diff --git a/.prod.env b/.prod.env index a9ef451f..da2011a0 100644 --- a/.prod.env +++ b/.prod.env @@ -177,3 +177,5 @@ GLOBAL_STORAGE=10737418240 # Gunicorn server socket PORT=5000 + +GEVENT_WORKER=True diff --git a/server/application.py b/server/application.py index 6f5fea24..2a4220b6 100644 --- a/server/application.py +++ b/server/application.py @@ -9,14 +9,17 @@ # Modules that had direct imports (NOT patched): ['urllib3.util, 'urllib3.util.ssl']" # which comes from using requests (its deps) lib in webhooks -import os -from random import randint +from mergin.config import Configuration as MainConfig -if not os.getenv("NO_MONKEY_PATCH", False): +if MainConfig.GEVENT_WORKER: import gevent.monkey + import psycogreen.gevent + + gevent.monkey.patch_all() + psycogreen.gevent.patch_psycopg() - gevent.monkey.patch_all(subprocess=True) +from random import randint from celery.schedules import crontab from mergin.app import create_app from mergin.auth.tasks import anonymize_removed_users diff --git a/server/config.py b/server/config.py index 7c291f20..7c122dc3 100644 --- a/server/config.py +++ b/server/config.py @@ -25,18 +25,6 @@ """ import logging -try: - from psycogreen.gevent import patch_psycopg -except ImportError: - import sys - import traceback - - exception_info = traceback.format_exc() - sys.stderr.write( - f"Failed to load required functions from the psycogreen library: { exception_info }\n" - ) - sys.exit(1) - worker_class = "gevent" workers = 2 @@ -59,12 +47,7 @@ max_requests_jitter = 5000 - -def do_post_fork(server, worker): - patch_psycopg() - - -post_fork = do_post_fork +timeout = 30 """ diff --git a/server/mergin/__init__.py b/server/mergin/__init__.py index b1d7cb3f..f8ea3438 100644 --- a/server/mergin/__init__.py +++ b/server/mergin/__init__.py @@ -1,5 +1,3 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial - -from .app import db, mail, ma, create_app diff --git a/server/mergin/app.py b/server/mergin/app.py index 4460e010..751b6561 100644 --- a/server/mergin/app.py +++ b/server/mergin/app.py @@ -7,11 +7,12 @@ import os import connexion import wtforms_json +import gevent from marshmallow import fields from sqlalchemy.schema import MetaData from flask_sqlalchemy import SQLAlchemy from flask_marshmallow import Marshmallow -from flask import json, jsonify, request, abort, current_app, Flask +from flask import json, jsonify, request, abort, current_app, Flask, Request, Response from flask_login import current_user, LoginManager from flask_wtf.csrf import generate_csrf, CSRFProtect from flask_migrate import Migrate @@ -27,6 +28,7 @@ from typing import List, Dict, Optional from .sync.utils import get_blacklisted_dirs, get_blacklisted_files +from .config import Configuration convention = { "ix": "ix_%(column_0_label)s", @@ -105,9 +107,24 @@ def update_obj(self, obj): field.populate_obj(obj, name) -def create_simple_app() -> Flask: - from .config import Configuration +class GeventTimeoutMiddleware: + """Middleware to implement gevent.Timeout() for all requests""" + + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + request = Request(environ) + try: + with gevent.Timeout(Configuration.GEVENT_REQUEST_TIMEOUT): + return self.app(environ, start_response) + except gevent.Timeout: + logging.error(f"Gevent worker: Request {request.path} timed out") + resp = Response("Gateway Timeout", mimetype="text/plain", status=504) + return resp(environ, start_response) + +def create_simple_app() -> Flask: app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir)) flask_app = app.app @@ -117,6 +134,9 @@ def create_simple_app() -> Flask: ma.init_app(flask_app) Migrate(flask_app, db) flask_app.connexion_app = app + # in case of gevent worker type use middleware to implement custom request timeout + if Configuration.GEVENT_WORKER: + flask_app.wsgi_app = GeventTimeoutMiddleware(flask_app.wsgi_app) @flask_app.cli.command() def init_db(): @@ -133,7 +153,6 @@ def init_db(): def create_app(public_keys: List[str] = None) -> Flask: """Factory function to create Flask app instance""" from itsdangerous import BadTimeSignature, BadSignature - from .config import Configuration from .auth import auth_required, decode_token from .auth.models import User diff --git a/server/mergin/auth/app.py b/server/mergin/auth/app.py index 78363361..1a3caba4 100644 --- a/server/mergin/auth/app.py +++ b/server/mergin/auth/app.py @@ -12,7 +12,7 @@ from .commands import add_commands from .config import Configuration from .models import User, UserProfile -from .. import db +from ..app import db # signal for other versions to listen to user_account_closed = signal("user_account_closed") diff --git a/server/mergin/auth/commands.py b/server/mergin/auth/commands.py index 1f690cf7..80cc7fb8 100644 --- a/server/mergin/auth/commands.py +++ b/server/mergin/auth/commands.py @@ -6,7 +6,7 @@ from flask import Flask from sqlalchemy import or_, func -from .. import db +from ..app import db from .models import User, UserProfile diff --git a/server/mergin/auth/controller.py b/server/mergin/auth/controller.py index bccb98ba..cdea3181 100644 --- a/server/mergin/auth/controller.py +++ b/server/mergin/auth/controller.py @@ -35,8 +35,7 @@ UserChangePasswordForm, ApiLoginForm, ) -from .. import db -from ..app import DEPRECATION_API_MSG +from ..app import DEPRECATION_API_MSG, db from ..utils import format_time_delta diff --git a/server/mergin/auth/models.py b/server/mergin/auth/models.py index 0c5ab341..58c706ad 100644 --- a/server/mergin/auth/models.py +++ b/server/mergin/auth/models.py @@ -9,7 +9,7 @@ from flask import current_app, request from sqlalchemy import or_, func -from .. import db +from ..app import db from ..sync.utils import get_user_agent, get_ip, get_device_id diff --git a/server/mergin/auth/schemas.py b/server/mergin/auth/schemas.py index 10475b2c..ea0bcd71 100644 --- a/server/mergin/auth/schemas.py +++ b/server/mergin/auth/schemas.py @@ -5,9 +5,8 @@ from flask import current_app from marshmallow import fields -from .. import ma from .models import User, UserProfile -from ..app import DateTimeWithZ +from ..app import DateTimeWithZ, ma class UserProfileSchema(ma.SQLAlchemyAutoSchema): diff --git a/server/mergin/auth/tasks.py b/server/mergin/auth/tasks.py index cd91e153..a09848a8 100644 --- a/server/mergin/auth/tasks.py +++ b/server/mergin/auth/tasks.py @@ -6,7 +6,7 @@ from sqlalchemy.sql.operators import isnot from ..celery import celery -from .. import db +from .app import db from .models import User from .config import Configuration diff --git a/server/mergin/celery.py b/server/mergin/celery.py index 156e14bf..8442924d 100644 --- a/server/mergin/celery.py +++ b/server/mergin/celery.py @@ -9,7 +9,7 @@ from smtplib import SMTPException, SMTPServerDisconnected from .config import Configuration -from . import mail +from .app import mail # create on flask app independent object diff --git a/server/mergin/config.py b/server/mergin/config.py index fdb8108b..94130176 100644 --- a/server/mergin/config.py +++ b/server/mergin/config.py @@ -3,10 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial import os -from .version import get_version from decouple import config, Csv -config_dir = os.path.abspath(os.path.dirname(__file__)) +from .version import get_version class Configuration(object): @@ -95,6 +94,7 @@ class Configuration(object): # build hash number BUILD_HASH = config("BUILD_HASH", default="") + # Allow changing access to admin panel ENABLE_SUPERADMIN_ASSIGNMENT = config( "ENABLE_SUPERADMIN_ASSIGNMENT", default=True, cast=bool @@ -102,3 +102,8 @@ class Configuration(object): # backend version VERSION = config("VERSION", default=get_version()) SERVER_TYPE = config("SERVER_TYPE", default="ce") + + # whether to run flask app with gevent worker type in gunicorn + # using gevent type of worker impose some requirements on code, e.g. to be greenlet safe, custom timeouts + GEVENT_WORKER = config("GEVENT_WORKER", default=False, cast=bool) + GEVENT_REQUEST_TIMEOUT = config("GEVENT_REQUEST_TIMEOUT", default=30, cast=int) diff --git a/server/mergin/stats/models.py b/server/mergin/stats/models.py index 6d6ffbbd..320e0137 100644 --- a/server/mergin/stats/models.py +++ b/server/mergin/stats/models.py @@ -5,7 +5,7 @@ import uuid from sqlalchemy.dialects.postgresql import UUID -from .. import db +from ..app import db class MerginInfo(db.Model): diff --git a/server/mergin/sync/commands.py b/server/mergin/sync/commands.py index a07d5966..a71be8fe 100644 --- a/server/mergin/sync/commands.py +++ b/server/mergin/sync/commands.py @@ -10,9 +10,10 @@ from flask import Flask, current_app from .files import UploadChanges -from .. import db +from ..app import db from .models import Project, ProjectAccess, ProjectVersion from .utils import split_project_path +from ..auth.models import User def add_commands(app: Flask): @@ -24,10 +25,21 @@ def project(): @project.command() @click.argument("name") @click.argument("namespace") - @click.argument("user") - def create(name, namespace, user): # pylint: disable=W0612 + @click.argument("username") + def create(name, namespace, username): # pylint: disable=W0612 """Create blank project""" workspace = current_app.ws_handler.get_by_name(namespace) + if not workspace: + print("ERROR: Workspace not found") + return + user = User.query.filter_by(username=username).first() + if not user: + print("ERROR: User not found") + return + p = Project.query.filter_by(name=name, workspace_id=workspace.id).first() + if p: + print("ERROR: Project name already exists") + return project_params = dict( creator=user, name=name, @@ -50,7 +62,7 @@ def create(name, namespace, user): # pylint: disable=W0612 @project.command() @click.argument("project-name") - @click.option("--version", required=True) + @click.option("--version", type=int, required=True) @click.option("--directory", type=click.Path(), required=True) def download(project_name, version, directory): # pylint: disable=W0612 """Download files for project at particular version""" diff --git a/server/mergin/sync/db_events.py b/server/mergin/sync/db_events.py index dbba5ed3..18d1ce60 100644 --- a/server/mergin/sync/db_events.py +++ b/server/mergin/sync/db_events.py @@ -6,7 +6,7 @@ from flask import current_app, abort from sqlalchemy import event -from .. import db +from ..app import db def check(session): diff --git a/server/mergin/sync/files.py b/server/mergin/sync/files.py index 204edf86..12b30afe 100644 --- a/server/mergin/sync/files.py +++ b/server/mergin/sync/files.py @@ -8,8 +8,7 @@ from marshmallow import fields, EXCLUDE, pre_load, post_load, post_dump from pathvalidate import sanitize_filename -from .. import ma -from ..app import DateTimeWithZ +from ..app import DateTimeWithZ, ma def mergin_secure_filename(filename: str) -> str: diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index 08fecb71..cff88229 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -29,7 +29,7 @@ ProjectFile, ) from .storages.disk import move_to_tmp -from .. import db +from ..app import db from .storages import DiskStorage from .utils import is_versioned_file, is_qgis @@ -100,7 +100,7 @@ def workspace(self): return project_workspace def cache_latest_files(self) -> None: - """Get project files from changes (FileHistory) and saved them for later use""" + """Get project files from changes (FileHistory) and save them for later use.""" if self.latest_version is None: return @@ -108,7 +108,7 @@ def cache_latest_files(self) -> None: WITH latest_changes AS ( SELECT fp.id, - fp.project_id, + pv.project_id, max(pv.name) AS version FROM project_version pv @@ -118,14 +118,13 @@ def cache_latest_files(self) -> None: pv.project_id = :project_id AND pv.name <= :latest_version GROUP BY - fp.id, fp.project_id + fp.id, pv.project_id ), aggregates AS ( SELECT project_id, - array_agg(fh.id) AS files_ids + COALESCE(array_agg(fh.id) FILTER (WHERE fh.change != 'delete'), ARRAY[]::INTEGER[]) AS files_ids FROM latest_changes ch LEFT OUTER JOIN file_history fh ON (fh.file_path_id = ch.id AND fh.project_version_name = ch.version) - WHERE fh.change != 'delete' GROUP BY project_id ) UPDATE latest_project_files pf diff --git a/server/mergin/sync/private_api_controller.py b/server/mergin/sync/private_api_controller.py index 57da6492..01c5af6e 100644 --- a/server/mergin/sync/private_api_controller.py +++ b/server/mergin/sync/private_api_controller.py @@ -8,7 +8,7 @@ from sqlalchemy.orm import defer from sqlalchemy import text, and_, desc, asc -from .. import db +from ..app import db from ..auth import auth_required from ..auth.models import User, UserProfile from .forms import AccessPermissionForm diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index b7c70c2c..b9e07552 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -32,7 +32,7 @@ from gevent import sleep import base64 from werkzeug.exceptions import HTTPException -from .. import db +from ..app import db from ..auth import auth_required from ..auth.models import User from .models import ( @@ -86,7 +86,8 @@ from .errors import StorageLimitHit from ..utils import format_time_delta -push_triggered = signal("push_triggered") +push_finished = signal("push_finished") +# TODO: Move to database events to handle all commits to project versions project_version_created = signal("project_version_created") @@ -732,7 +733,6 @@ def project_push(namespace, project_name): if not ws: abort(404) - push_triggered.send(project) # fixme use get_latest pv = ProjectVersion.query.filter_by( project_id=project.id, name=project.latest_version @@ -874,6 +874,7 @@ def project_push(namespace, project_name): f"Transaction id: {upload.id}. No upload." ) project_version_created.send(pv) + push_finished.send(pv) return jsonify(ProjectSchema().dump(project)), 200 except IntegrityError as err: db.session.rollback() @@ -1084,6 +1085,7 @@ def push_finish(transaction_id): f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}." ) project_version_created.send(pv) + push_finished.send(pv) except (psycopg2.Error, FileNotFoundError, DataSyncError, IntegrityError) as err: db.session.rollback() logging.exception( diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 8e567238..59cdbb2d 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -8,7 +8,7 @@ from flask import abort, jsonify from flask_login import current_user -from mergin import db +from mergin.app import db from mergin.auth import auth_required from mergin.sync.models import Project from mergin.sync.permissions import ProjectPermissions, require_project_by_uuid diff --git a/server/mergin/sync/schemas.py b/server/mergin/sync/schemas.py index 081b2ab7..d8c766aa 100644 --- a/server/mergin/sync/schemas.py +++ b/server/mergin/sync/schemas.py @@ -7,11 +7,10 @@ from flask_login import current_user from flask import current_app -from .. import ma from .files import ProjectFileSchema, FileSchema from .permissions import ProjectPermissions from .models import Project, ProjectVersion, AccessRequest, FileHistory, PushChangeType -from ..app import DateTimeWithZ +from ..app import DateTimeWithZ, ma from ..auth.models import User diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index 29309241..4debb255 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -16,7 +16,7 @@ from result import Err, Ok, Result from .storage import ProjectStorage, FileNotFound, InitializationError -from ... import db +from ...app import db from ..utils import ( generate_checksum, is_versioned_file, diff --git a/server/mergin/sync/tasks.py b/server/mergin/sync/tasks.py index cfb1f645..ac9fdb1c 100644 --- a/server/mergin/sync/tasks.py +++ b/server/mergin/sync/tasks.py @@ -13,7 +13,7 @@ from .storages.disk import move_to_tmp from .config import Configuration from ..celery import celery -from .. import db +from ..app import db @celery.task diff --git a/server/mergin/sync/workspace.py b/server/mergin/sync/workspace.py index 1397a85d..5b5ac84e 100644 --- a/server/mergin/sync/workspace.py +++ b/server/mergin/sync/workspace.py @@ -18,7 +18,7 @@ ) from .permissions import projects_query, ProjectPermissions from .public_api_controller import parse_project_access_update_request -from .. import db +from ..app import db from ..auth.models import User from ..config import Configuration from .interfaces import AbstractWorkspace, WorkspaceHandler diff --git a/server/mergin/tests/fixtures.py b/server/mergin/tests/fixtures.py index bab7646e..6b83ecf9 100644 --- a/server/mergin/tests/fixtures.py +++ b/server/mergin/tests/fixtures.py @@ -12,7 +12,7 @@ from pygeodiff import GeoDiff import pytest -from .. import db, create_app +from ..app import db, create_app from ..sync.models import Project, ProjectVersion from ..stats.app import register from ..stats.models import MerginInfo diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index 43d61d7d..41f1065d 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -12,7 +12,7 @@ from ..auth.models import User, UserProfile, LoginHistory from ..auth.tasks import anonymize_removed_users -from .. import db +from ..app import db from ..sync.models import Project from . import ( test_workspace_id, diff --git a/server/mergin/tests/test_celery.py b/server/mergin/tests/test_celery.py index cc88e2d3..09b118c6 100644 --- a/server/mergin/tests/test_celery.py +++ b/server/mergin/tests/test_celery.py @@ -8,7 +8,7 @@ from flask_mail import Mail from unittest.mock import patch -from .. import db +from ..app import db from ..config import Configuration from ..sync.models import Project, AccessRequest, ProjectVersion from ..celery import send_email_async diff --git a/server/mergin/tests/test_db_hooks.py b/server/mergin/tests/test_db_hooks.py index 0f16710d..43186c18 100644 --- a/server/mergin/tests/test_db_hooks.py +++ b/server/mergin/tests/test_db_hooks.py @@ -20,7 +20,7 @@ ) from ..sync.files import UploadChanges from ..auth.models import User -from .. import db +from ..app import db from . import DEFAULT_USER from .utils import add_user, create_project, create_workspace, cleanup @@ -163,7 +163,7 @@ def test_remove_project(client, diff_project): LatestProjectFiles.query.filter_by(project_id=project_id) .first() .file_history_ids - is None + == [] ) # try to remove the deleted project diff --git a/server/mergin/tests/test_file_restore.py b/server/mergin/tests/test_file_restore.py index f189c9e4..278837d3 100644 --- a/server/mergin/tests/test_file_restore.py +++ b/server/mergin/tests/test_file_restore.py @@ -6,7 +6,7 @@ import shutil from sqlalchemy.orm.attributes import flag_modified -from .. import db +from ..app import db from ..auth.models import User from ..sync.models import ProjectVersion, Project, GeodiffActionHistory from . import test_project_dir, TMP_DIR diff --git a/server/mergin/tests/test_middleware.py b/server/mergin/tests/test_middleware.py new file mode 100644 index 00000000..09f4bf05 --- /dev/null +++ b/server/mergin/tests/test_middleware.py @@ -0,0 +1,33 @@ +# Copyright (C) Lutra Consulting Limited +# +# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + +import gevent +import pytest + +from ..app import create_simple_app, GeventTimeoutMiddleware +from ..config import Configuration + + +@pytest.mark.parametrize("use_middleware", [True, False]) +def test_use_middleware(use_middleware): + """Test using middleware""" + Configuration.GEVENT_WORKER = use_middleware + Configuration.GEVENT_REQUEST_TIMEOUT = 1 + application = create_simple_app() + + def ping(): + gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1) + return "pong" + + application.add_url_rule("/test", "ping", ping) + app_context = application.app_context() + app_context.push() + + assert isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware + # in case of gevent, dummy endpoint it set to time out + assert ( + application.test_client().get("/test").status_code == 504 + if use_middleware + else 200 + ) diff --git a/server/mergin/tests/test_permissions.py b/server/mergin/tests/test_permissions.py index 0e986de3..cfabbdc2 100644 --- a/server/mergin/tests/test_permissions.py +++ b/server/mergin/tests/test_permissions.py @@ -8,7 +8,7 @@ from ..sync.permissions import require_project, ProjectPermissions from ..sync.models import ProjectRole from ..auth.models import User -from .. import db +from ..app import db from ..config import Configuration from .utils import add_user, create_project, create_workspace diff --git a/server/mergin/tests/test_private_project_api.py b/server/mergin/tests/test_private_project_api.py index 361fe59e..d8c3145c 100644 --- a/server/mergin/tests/test_private_project_api.py +++ b/server/mergin/tests/test_private_project_api.py @@ -6,15 +6,19 @@ import json import os -import pytest from flask import url_for -from .. import db +from ..app import db from ..sync.models import AccessRequest, Project, ProjectRole, RequestStatus from ..auth.models import User from ..config import Configuration from . import json_headers -from .utils import add_user, login, create_project, create_workspace +from .utils import ( + add_user, + login, + create_project, + create_workspace, +) def test_project_unsubscribe(client, diff_project): diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index f0f3b64d..4371ab8b 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -18,12 +18,13 @@ import re from flask_login import current_user +from unittest.mock import patch from pygeodiff import GeoDiff from flask import url_for, current_app import tempfile from sqlalchemy import desc -from .. import db +from ..app import db from ..sync.models import ( Project, Upload, @@ -58,6 +59,7 @@ login, file_info, login_as_admin, + upload_file_to_project, ) from ..config import Configuration from ..sync.config import Configuration as SyncConfiguration @@ -1791,6 +1793,8 @@ def test_clone_project(client, data, username, expected): # cleanup shutil.rmtree(project.storage.project_dir) + Configuration.GLOBAL_STORAGE = 104857600 + def test_optimize_storage(app, client, diff_project): """Test optimize storage for geopackages which could be restored from diffs @@ -2461,3 +2465,32 @@ def test_delete_diff_file(client): change=PushChangeType.DELETE.value, ).first() assert fh.path == "base.gpkg" and fh.diff is None + + +def test_cache_files_ids(client): + """Test caching latest project files when it is None""" + user = User.query.filter_by(username="mergin").first() + test_workspace = create_workspace() + project = create_project("no_file_history", test_workspace, user) + db.session.commit() + assert project.latest_project_files.file_history_ids is not None + project.latest_project_files.file_history_ids = None + db.session.commit() + assert project.latest_project_files.file_history_ids is None + # uploading to project caches + filename = "test.txt" + upload_file_to_project(project, filename, client) + fp = ProjectFilePath.query.filter_by(project_id=project.id, path=filename).first() + fh = FileHistory.query.filter_by(file_path_id=fp.id).first() + assert project.latest_project_files.file_history_ids == [fh.id] + + +def test_signals(client): + workspace = create_workspace() + user = User.query.filter(User.username == "mergin").first() + project = create_project("test-project", workspace, user) + with patch( + "mergin.sync.public_api_controller.push_finished.send" + ) as push_finished_mock: + upload_file_to_project(project, "test.txt", client) + push_finished_mock.assert_called_once() diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 4a2e1547..4bdb65a2 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -from .. import db +from ..app import db from mergin.sync.models import Project from tests import test_project, test_workspace_id diff --git a/server/mergin/tests/test_statistics.py b/server/mergin/tests/test_statistics.py index f076d882..2c8e930c 100644 --- a/server/mergin/tests/test_statistics.py +++ b/server/mergin/tests/test_statistics.py @@ -6,7 +6,7 @@ from unittest.mock import patch import requests -from .. import db +from ..app import db from ..stats.tasks import send_statistics from ..stats.models import MerginInfo from .utils import Response diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index a18c6f2c..6141e573 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -10,7 +10,7 @@ from sqlalchemy import desc from unittest.mock import MagicMock -from .. import db +from ..app import db from ..sync.utils import parse_gpkgb_header_size, gpkg_wkb_to_wkt, is_name_allowed from ..auth.models import LoginHistory, User from . import json_headers diff --git a/server/mergin/tests/test_workspace.py b/server/mergin/tests/test_workspace.py index e273d2f1..a10eadb9 100644 --- a/server/mergin/tests/test_workspace.py +++ b/server/mergin/tests/test_workspace.py @@ -6,7 +6,7 @@ from sqlalchemy import null -from .. import db +from ..app import db from ..config import Configuration from ..sync.models import FileHistory, ProjectVersion, PushChangeType, ProjectFilePath from ..sync.workspace import GlobalWorkspaceHandler diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index 4e6905ef..debc096e 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -4,6 +4,7 @@ import json import shutil +from typing import Tuple import pysqlite3 import uuid import math @@ -21,7 +22,7 @@ from ..sync.models import Project, ProjectAccess, ProjectVersion, FileHistory from ..sync.files import UploadChanges, ChangesSchema from ..sync.workspace import GlobalWorkspace -from .. import db +from ..app import db from . import json_headers, DEFAULT_USER, test_project, test_project_dir, TMP_DIR CHUNK_SIZE = 1024 @@ -213,10 +214,7 @@ def file_info(project_dir, path, chunk_size=1024): } -def upload_file_to_project(project, filename, client): - """Add test file to project - start, upload and finish push process""" - file = os.path.join(test_project_dir, filename) - assert os.path.exists(file) +def mock_changes_data(project, filename) -> dict: changes = { "added": [file_info(test_project_dir, filename)], "updated": [], @@ -226,13 +224,33 @@ def upload_file_to_project(project, filename, client): "version": ProjectVersion.to_v_name(project.latest_version), "changes": changes, } + return data + + +def push_file_start( + project: Project, filename: str, client, mocked_changes_data=None +) -> dict: + """ + Initiate the process of pushing a file to a project by calling /push endpoint. + """ + file = os.path.join(test_project_dir, filename) + assert os.path.exists(file) + data = mocked_changes_data or mock_changes_data(project, filename) resp = client.post( f"/v1/project/push/{project.workspace.name}/{project.name}", data=json.dumps(data, cls=DateTimeEncoder).encode("utf-8"), headers=json_headers, ) + return resp + + +def upload_file_to_project(project: Project, filename: str, client) -> dict: + """Add test file to project - start, upload and finish push process""" + file = os.path.join(test_project_dir, filename) + data = mock_changes_data(project, filename) + changes = data.get("changes") + resp = push_file_start(project, filename, client, data) upload_id = resp.json["transaction"] - changes = data["changes"] file_meta = changes["added"][0] for chunk_id in file_meta["chunks"]: url = f"/v1/project/push/chunk/{upload_id}/{chunk_id}" @@ -241,7 +259,10 @@ def upload_file_to_project(project, filename, client): client.post( url, data=f_data, headers={"Content-Type": "application/octet-stream"} ) - assert client.post(f"/v1/project/push/finish/{upload_id}").status_code == 200 + push_finish = client.post(f"/v1/project/push/finish/{upload_id}") + assert resp.status_code == 200 + assert push_finish.status_code == 200 + return push_finish def gpkgs_are_equal(file1, file2): diff --git a/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue b/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue index fa114c9a..8ac2ea4f 100644 --- a/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue +++ b/web-app/packages/admin-lib/src/modules/admin/components/AdminProjectsTable.vue @@ -70,9 +70,9 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial :sortable="header.sortable" >