diff --git a/development.md b/development.md index 9e92f0d0..d1731221 100644 --- a/development.md +++ b/development.md @@ -60,7 +60,7 @@ Watching the type definitions is also useful to pick up any changes to imports o ## Running locally in a docker composition -If you want to run the whole stack locally, you can use the docker. Docker will build the images from yout local files and run the services. +If you want to run the whole stack locally, you can use the docker. Docker will build the images from your local files and run the services. ```shell # Run the docker composition with the current Dockerfiles diff --git a/docker-compose.yml b/docker-compose.yml index 68748caa..c697278b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -79,12 +79,13 @@ services: restart: always depends_on: - server-gunicorn + user: 101:999 links: - db networks: - merginmaps proxy: - image: nginxinc/nginx-unprivileged:1.25.5 + image: nginxinc/nginx-unprivileged:1.27 container_name: merginmaps-proxy restart: always # run nginx as built-in user but with group mergin-family for files permissions diff --git a/nginx.conf b/nginx.conf index ed4512ad..598b8e3e 100644 --- a/nginx.conf +++ b/nginx.conf @@ -17,7 +17,7 @@ server { # we don't want nginx trying to do something clever with # redirects, we set the Host: header above already. proxy_redirect off; - proxy_pass http://merginmaps-web; + proxy_pass http://merginmaps-web:8080; } # proxy to backend diff --git a/server/Dockerfile b/server/Dockerfile index 00d691a6..52e03636 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -16,7 +16,7 @@ RUN apt-get update -y && \ python3-pip \ python3-setuptools \ iputils-ping \ - gcc build-essential binutils cmake extra-cmake-modules libsqlite3-mod-spatialite && \ + gcc build-essential binutils cmake extra-cmake-modules libsqlite3-mod-spatialite libmagic1 && \ rm -rf /var/lib/apt/lists/* diff --git a/server/Pipfile b/server/Pipfile index 62622bfc..448405ca 100644 --- a/server/Pipfile +++ b/server/Pipfile @@ -40,6 +40,7 @@ shapely = "==2.0.6" psycogreen = "==1.0.2" importlib-metadata = "==8.4.0" # https://github.com/pallets/flask/issues/4502 typing_extensions = "==4.12.2" +python-magic = "==0.4.27" # requirements for development on windows colorama = "==0.4.5" diff --git a/server/Pipfile.lock b/server/Pipfile.lock index be32d083..ee769e4c 100644 --- a/server/Pipfile.lock +++ b/server/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ade41edf9691b6241ca491897a6a989d61f5ed253b800ba068e921c5c574435a" + "sha256": "1a3dbebb1c3c4acae26310595c759edaaa9d86f2ab2984d3268ffc73a01def72" }, "pipfile-spec": 6, "requires": { @@ -917,6 +917,15 @@ "markers": "python_version >= '3.5'", "version": "==0.20.0" }, + "python-magic": { + "hashes": [ + "sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b", + "sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3" + ], + "index": "pypi", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==0.4.27" + }, "pytz": { "hashes": [ "sha256:220f481bdafa09c3955dfbdddb7b57780e9a94f5127e35456a48589b9e0c0197", @@ -1590,71 +1599,66 @@ "toml" ], "hashes": [ - "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9", - "sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f", - "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273", - "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994", - "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e", - "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50", - "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e", - "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e", - "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c", - "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853", - "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8", - "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8", - "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe", - "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165", - "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb", - "sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59", - "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609", - "sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18", - "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098", - "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd", - "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3", - "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43", - "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d", - "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359", - "sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90", - "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78", - "sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a", - "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99", - "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988", - "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2", - "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0", - "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694", - "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377", - "sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d", - "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23", - "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312", - "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf", - "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6", - "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b", - "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c", - "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690", - "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a", - "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f", - "sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4", - "sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25", - "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd", - "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852", - "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0", - "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244", - "sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315", - "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078", - "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0", - "sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27", - "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132", - "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5", - "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247", - "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022", - "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b", - "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3", - "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18", - "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5", - "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f" + "sha256:050172741de03525290e67f0161ae5f7f387c88fca50d47fceb4724ceaa591d2", + "sha256:08e5fb93576a6b054d3d326242af5ef93daaac9bb52bc25f12ccbc3fa94227cd", + "sha256:09d03f48d9025b8a6a116cddcb6c7b8ce80e4fb4c31dd2e124a7c377036ad58e", + "sha256:0d03c9452d9d1ccfe5d3a5df0427705022a49b356ac212d529762eaea5ef97b4", + "sha256:13100f98497086b359bf56fc035a762c674de8ef526daa389ac8932cb9bff1e0", + "sha256:25575cd5a7d2acc46b42711e8aff826027c0e4f80fb38028a74f31ac22aae69d", + "sha256:27700d859be68e4fb2e7bf774cf49933dcac6f81a9bc4c13bd41735b8d26a53b", + "sha256:2c81e53782043b323bd34c7de711ed9b4673414eb517eaf35af92185b873839c", + "sha256:397489c611b76302dfa1d9ea079e138dddc4af80fc6819d5f5119ec8ca6c0e47", + "sha256:476f29a258b9cd153f2be5bf5f119d670d2806363595263917bddc167d6e5cce", + "sha256:4bda710139ea646890d1c000feb533caff86904a0e0638f85e967c28cb8eec50", + "sha256:4cf96beb05d004e4c51cd846fcdf9eee9eb2681518524b66b2e7610507944c2f", + "sha256:4f21e3617f48d683f30cf2a6c8b739c838e600cb1454fe6b2eb486ac2bce8fbd", + "sha256:5128f3ba694c0a1bde55fc480090392c336236c3e1a10dad40dc1ab17c7675ff", + "sha256:532fe139691af134aa8b54ed60dd3c806aa81312d93693bd2883c7b61592c840", + "sha256:5a3f7cbbcb4ad95067a6525f83a6fc78d9cbc1e70f8abaeeaeaa72ef34f48fc3", + "sha256:5b48db06f53d1864fea6dbd855e6d51d41c0f06c212c3004511c0bdc6847b297", + "sha256:5e7ac966ab110bd94ee844f2643f196d78fde1cd2450399116d3efdd706e19f5", + "sha256:5edc16712187139ab635a2e644cc41fc239bc6d245b16124045743130455c652", + "sha256:60d4ad09dfc8c36c4910685faafcb8044c84e4dae302e86c585b3e2e7778726c", + "sha256:61c834cbb80946d6ebfddd9b393a4c46bec92fcc0fa069321fcb8049117f76ea", + "sha256:6ba27a0375c5ef4d2a7712f829265102decd5ff78b96d342ac2fa555742c4f4f", + "sha256:6c96a142057d83ee993eaf71629ca3fb952cda8afa9a70af4132950c2bd3deb9", + "sha256:6d60577673ba48d8ae8e362e61fd4ad1a640293ffe8991d11c86f195479100b7", + "sha256:7eb0504bb307401fd08bc5163a351df301438b3beb88a4fa044681295bbefc67", + "sha256:8e433b6e3a834a43dae2889adc125f3fa4c66668df420d8e49bc4ee817dd7a70", + "sha256:8fa4fffd90ee92f62ff7404b4801b59e8ea8502e19c9bf2d3241ce745b52926c", + "sha256:90de4e9ca4489e823138bd13098af9ac8028cc029f33f60098b5c08c675c7bda", + "sha256:a165b09e7d5f685bf659063334a9a7b1a2d57b531753d3e04bd442b3cfe5845b", + "sha256:a46d56e99a31d858d6912d31ffa4ede6a325c86af13139539beefca10a1234ce", + "sha256:ac476e6d0128fb7919b3fae726de72b28b5c9644cb4b579e4a523d693187c551", + "sha256:ac5d92e2cc121a13270697e4cb37e1eb4511ac01d23fe1b6c097facc3b46489e", + "sha256:adc2d941c0381edfcf3897f94b9f41b1e504902fab78a04b1677f2f72afead4b", + "sha256:b6ff5be3b1853e0862da9d349fe87f869f68e63a25f7c37ce1130b321140f963", + "sha256:bb35ae9f134fbd9cf7302a9654d5a1e597c974202678082dcc569eb39a8cde03", + "sha256:be05bde21d5e6eefbc3a6de6b9bee2b47894b8945342e8663192809c4d1f08ce", + "sha256:c27df03730059118b8a923cfc8b84b7e9976742560af528242f201880879c1da", + "sha256:c7719a5e1dc93883a6b319bc0374ecd46fb6091ed659f3fbe281ab991634b9b0", + "sha256:c86f4c7a6d1a54a24d804d9684d96e36a62d3ef7c0d7745ae2ea39e3e0293251", + "sha256:ca95d40900cf614e07f00cee8c2fad0371df03ca4d7a80161d84be2ec132b7a4", + "sha256:cd4839813b09ab1dd1be1bbc74f9a7787615f931f83952b6a9af1b2d3f708bf7", + "sha256:db4b1a69976b1b02acda15937538a1d3fe10b185f9d99920b17a740a0a102e06", + "sha256:dbb1a822fd858d9853333a7c95d4e70dde9a79e65893138ce32c2ec6457d7a36", + "sha256:de6b079b39246a7da9a40cfa62d5766bd52b4b7a88cf5a82ec4c45bf6e152306", + "sha256:df6ff122a0a10a30121d9f0cb3fbd03a6fe05861e4ec47adb9f25e9245aabc19", + "sha256:e0b0f272901a5172090c0802053fbc503cdc3fa2612720d2669a98a7384a7bec", + "sha256:e2778be4f574b39ec9dcd9e5e13644f770351ee0990a0ecd27e364aba95af89b", + "sha256:e3b746fa0ffc5b6b8856529de487da8b9aeb4fb394bb58de6502ef45f3434f12", + "sha256:e642e6a46a04e992ebfdabed79e46f478ec60e2c528e1e1a074d63800eda4286", + "sha256:eafea49da254a8289bed3fab960f808b322eda5577cb17a3733014928bbfbebd", + "sha256:f0f334ae844675420164175bf32b04e18a81fe57ad8eb7e0cfd4689d681ffed7", + "sha256:f382004fa4c93c01016d9226b9d696a08c53f6818b7ad59b4e96cb67e863353a", + "sha256:f4679fcc9eb9004fdd1b00231ef1ec7167168071bebc4d66327e28c1979b4449", + "sha256:fd2fffc8ce8692ce540103dff26279d2af22d424516ddebe2d7e4d6dbb3816b2", + "sha256:ff136607689c1c87f43d24203b6d2055b42030f352d5176f9c8b204d4235ef27", + "sha256:ff52b4e2ac0080c96e506819586c4b16cdbf46724bda90d308a7330a73cc8521", + "sha256:ff562952f15eff27247a4c4b03e45ce8a82e3fb197de6a7c54080f9d4ba07845" ], "markers": "python_version >= '3.9'", - "version": "==7.6.10" + "version": "==7.6.11" }, "dill": { "hashes": [ diff --git a/server/mergin/auth/api.yaml b/server/mergin/auth/api.yaml index df924109..095b0d26 100644 --- a/server/mergin/auth/api.yaml +++ b/server/mergin/auth/api.yaml @@ -201,14 +201,10 @@ paths: schema: type: object required: - - username - email - password - confirm properties: - username: - type: string - example: john.doe email: type: string format: email @@ -686,6 +682,7 @@ paths: description: User info content: application/json: + # TODO: fix this to match the ma.SQLAlchemy schema or use UserDetail schema schema: $ref: "#/components/schemas/UserInfo" "401": diff --git a/server/mergin/auth/controller.py b/server/mergin/auth/controller.py index 86fa5610..980bc14c 100644 --- a/server/mergin/auth/controller.py +++ b/server/mergin/auth/controller.py @@ -383,7 +383,8 @@ def register_user(): # pylint: disable=W0613,W0612 from ..celery import send_email_async form = UserRegistrationForm() - if form.validate(): + form.username.data = User.generate_username(form.email.data) + if form.validate_on_submit(): user = User.create(form.username.data, form.email.data, form.password.data) user_created.send(user, source="admin") token = generate_confirmation_token( diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index 4f6d2f7e..a5def163 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -27,12 +27,16 @@ def username_validation(form, field): is_reserved_word, ) - errors = [ - has_valid_characters(field.data), - has_valid_first_character(field.data), - is_reserved_word(field.data), - check_filename(field.data), - ] + errors = ( + [ + has_valid_characters(field.data), + has_valid_first_character(field.data), + is_reserved_word(field.data), + check_filename(field.data), + ] + if field.data + else [] + ) for error in errors: if error: raise ValidationError(error) diff --git a/server/mergin/auth/models.py b/server/mergin/auth/models.py index b57631cb..b54336ce 100644 --- a/server/mergin/auth/models.py +++ b/server/mergin/auth/models.py @@ -12,7 +12,7 @@ from ..app import db from ..sync.models import ProjectUser -from ..sync.utils import get_user_agent, get_ip, get_device_id +from ..sync.utils import get_user_agent, get_ip, get_device_id, is_reserved_word class User(db.Model): @@ -200,7 +200,9 @@ def generate_username(cls, email: str) -> Optional[str]: # remove forbidden chars username = re.sub( r"[\@\#\$\%\^\&\*\(\)\{\}\[\]\?\'\"`,;\:\+\=\~\\\/\|\<\>]", "", username - ) + ).ljust(4, "0") + # additional check for reserved words + username = f"{username}0" if is_reserved_word(username) else username # check if we already do not have existing usernames suffix = db.session.execute( text( diff --git a/server/mergin/commands.py b/server/mergin/commands.py index 47c0600f..3b2b16d4 100644 --- a/server/mergin/commands.py +++ b/server/mergin/commands.py @@ -109,7 +109,7 @@ def _check_server(): # pylint: disable=W0612 "No contact email set. Please set CONTACT_EMAIL environment variable", ) else: - click.secho(f"Base URL of server is {base_url}", fg="green") + click.secho(f"Your contact email is {contact_email}.", fg="green") tables = db.engine.table_names() if not tables: diff --git a/server/mergin/sync/forms.py b/server/mergin/sync/forms.py index d2c2ad0a..d5a52db3 100644 --- a/server/mergin/sync/forms.py +++ b/server/mergin/sync/forms.py @@ -16,7 +16,8 @@ def project_name_validation(name: str) -> str | None: """Check whether project name is valid""" - if not name.strip(): + name = name.strip() if name is not None else name + if not name: return "Project name cannot be empty" errors = [ has_valid_characters(name), diff --git a/server/mergin/sync/models.py b/server/mergin/sync/models.py index e353f68a..06a2d1e4 100644 --- a/server/mergin/sync/models.py +++ b/server/mergin/sync/models.py @@ -1121,7 +1121,7 @@ class SyncFailuresHistory(db.Model): error_type = db.Column( db.String, index=True ) # e.g. push_start, push_finish, push_lost - error_details = db.Column(db.String, index=True) + error_details = db.Column(db.String) timestamp = db.Column(db.DateTime(), default=datetime.utcnow, index=True) user_id = db.Column( db.Integer, db.ForeignKey("user.id", ondelete="SET NULL"), nullable=True diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 9f4eca3d..5aada5b6 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -5,7 +5,6 @@ import binascii import functools import json -import mimetypes import os import logging from dataclasses import asdict @@ -13,6 +12,7 @@ from urllib.parse import quote import uuid from datetime import datetime + import psycopg2 from blinker import signal from connexion import NoContent, request @@ -87,6 +87,9 @@ get_project_path, get_device_id, is_valid_path, + is_supported_type, + is_supported_extension, + get_mimetype, ) from .errors import StorageLimitHit from ..utils import format_time_delta @@ -406,7 +409,7 @@ def download_project_file( if not is_binary(abs_path): mime_type = "text/plain" else: - mime_type = mimetypes.guess_type(abs_path)[0] + mime_type = get_mimetype(abs_path) resp.headers["Content-Type"] = mime_type resp.headers["Content-Disposition"] = "attachment; filename={}".format( quote(os.path.basename(file).encode("utf-8")) @@ -813,7 +816,16 @@ def project_push(namespace, project_name): if not all(ele.path != item.path for ele in project.files): abort(400, f"File {item.path} has been already uploaded") if not is_valid_path(item.path): - abort(400, f"File {item.path} contains invalid characters.") + abort( + 400, + f"Unsupported file name detected: {item.path}. Please remove the invalid characters.", + ) + if not is_supported_extension(item.path): + abort( + 400, + f"Unsupported file type detected: {item.path}. " + f"Please remove the file or try compressing it into a ZIP file before uploading", + ) # changes' files must be unique changes_files = [ @@ -1042,6 +1054,9 @@ def push_finish(transaction_id): ) corrupted_files.append(f.path) continue + if not is_supported_type(dest_file): + logging.info(f"Rejecting blacklisted file: {dest_file}") + abort(400, f"Unsupported file type detected: {f.path}") if expected_size != os.path.getsize(dest_file): logging.error( diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index f844d6b6..38e2fd68 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -21,6 +21,7 @@ is_valid_filepath, is_valid_filename, ) +import magic def generate_checksum(file, chunk_size=4096): @@ -368,3 +369,180 @@ def is_valid_path(filepath: str) -> bool: os.path.basename(filepath) ) # invalid characters in filename, reserved filenames ) + + +def is_supported_extension(filepath) -> bool: + """Check whether file's extension is supported.""" + ext = os.path.splitext(filepath)[1].lower() + return ext and ext not in FORBIDDEN_EXTENSIONS + + +FORBIDDEN_EXTENSIONS = { + ".ade", + ".adp", + ".app", + ".appcontent-ms", + ".application", + ".appref-ms", + ".asp", + ".aspx", + ".asx", + ".bas", + ".bat", + ".bgi", + ".cab", + ".cdxml", + ".cer", + ".chm", + ".cmd", + ".cnt", + ".com", + ".cpl", + ".crt", + ".csh", + ".der", + ".diagcab", + ".dll", + ".drv", + ".exe", + ".fxp", + ".gadget", + ".grp", + ".hlp", + ".hpj", + ".hta", + ".htc", + ".htaccess", + ".htpasswd", + ".inf", + ".ins", + ".iso", + ".isp", + ".its", + ".jar", + ".jnlp", + ".js", + ".jse", + ".jsp", + ".ksh", + ".lnk", + ".mad", + ".maf", + ".mag", + ".mam", + ".maq", + ".mar", + ".mas", + ".mat", + ".mau", + ".mav", + ".maw", + ".mcf", + ".mda", + ".mdb", + ".mde", + ".mdt", + ".mdw", + ".mdz", + ".msc", + ".mht", + ".mhtml", + ".msh", + ".msh1", + ".msh2", + ".mshxml", + ".msh1xml", + ".msh2xml", + ".msi", + ".msp", + ".mst", + ".msu", + ".ops", + ".osd", + ".pcd", + ".pif", + ".pl", + ".plg", + ".prf", + ".prg", + ".printerexport", + ".ps1", + ".ps1xml", + ".ps2", + ".ps2xml", + ".psc1", + ".psc2", + ".psd1", + ".psdm1", + ".pssc", + ".pst", + ".py", + ".pyc", + ".pyo", + ".pyw", + ".pyz", + ".pyzw", + ".reg", + ".scf", + ".scr", + ".sct", + ".settingcontent-ms", + ".sh", + ".shb", + ".shs", + ".sys", + ".theme", + ".tmp", + ".torrent", + ".url", + ".vb", + ".vbe", + ".vbp", + ".vbs", + ".vhd", + ".vhdx", + ".vsmacros", + ".vsw", + ".webpnp", + ".website", + ".ws", + ".wsb", + ".wsc", + ".wsf", + ".wsh", + ".xbap", + ".xll", + ".xnk", +} + +FORBIDDEN_MIME_TYPES = { + "application/x-msdownload", + "application/x-sh", + "application/x-bat", + "application/x-msdos-program", + "application/x-dosexec", + "application/x-csh", + "application/x-perl", + "application/javascript", + "application/x-python-code", + "application/x-ruby", + "application/java-archive", + "application/vnd.ms-cab-compressed", + "application/x-ms-shortcut", + "application/vnd.microsoft.portable-executable", + "application/x-ms-installer", + "application/x-ms-application", + "application/x-ms-wim", + "text/x-shellscript", +} + + +def is_supported_type(filepath) -> bool: + """Check whether the file mimetype is supported.""" + mime_type = get_mimetype(filepath) + return mime_type.startswith("image/") or mime_type not in FORBIDDEN_MIME_TYPES + + +def get_mimetype(filepath: str) -> str: + """Identifies file types by checking their headers""" + return magic.from_file(filepath, mime=True) diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index 18b50ffb..167fe5f3 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -88,65 +88,47 @@ def test_logout(client): # user registration tests test_user_reg_data = [ - ("test", "test@test.com", "#pwd1234", 201), # success + ("test@test.com", "#pwd1234", 201), # success ( - "TesTUser", "test@test.com", "#pwd1234", 201, ), # tests with upper case, but user does not exist ( - "TesTUser2", "test2@test.com", "#pwd1234", 201, ), # tests with upper case, but user does not exist - ("bob", "test@test.com", "#pwd1234", 400), # invalid (short) username - ("test", "test.com", "#pwd1234", 400), # invalid email - ("mergin", "test@test.com", "#pwd1234", 400), # existing user + ("test.com", "#pwd1234", 400), # invalid email + ("admin@example.com", "#pwd1234", 201), # some random admin with diff email, ( - "MerGin", - "tests@test.com", - "#pwd1234", - 400, - ), # tests with upper case but mergin already exists - ( - " mergin ", - "tests@test.com", - "#pwd1234", - 400, - ), # tests with blank spaces, but mergin user already exists - ( - "XmerginX", " tests@test.com ", "#pwd1234", 201, ), # tests with blank spaces, whitespace to be removed ( - "mergin2", " mergin@mergin.com ", "#pwd1234", 400, ), # tests with blank spaces, but email already exists ( - "mergin3", " merGIN@mergin.com ", "#pwd1234", 400, ), # tests with upper case, but email already exists - ("XmerginX", " mergin@mergin.com ", "#pwd123", 400), # invalid password + (" mergin@mergin.com ", "#pwd123", 400), # invalid password ] -@pytest.mark.parametrize("username,email,pwd,expected", test_user_reg_data) -def test_user_register(client, username, email, pwd, expected): +@pytest.mark.parametrize("email,pwd,expected", test_user_reg_data) +def test_user_register(client, email, pwd, expected): login_as_admin(client) url = url_for("/.mergin_auth_controller_register_user") - data = {"username": username, "email": email, "password": pwd, "confirm": pwd} + data = {"email": email, "password": pwd, "confirm": pwd} resp = client.post(url, data=json.dumps(data), headers=json_headers) assert resp.status_code == expected if expected == 201: - user = User.query.filter_by(username=username).first() + user = User.query.filter_by(email=email.strip()).first() assert user assert user.active assert not user.verified_email # awaits user confirmation @@ -862,6 +844,14 @@ def test_username_generation(client): # generate username from email containing invalid chars for username, e.g. + assert User.generate_username("tralala+test@example.com") == "tralalatest" + # generate username from short email + user = add_user("t000", "user") + assert User.generate_username("t@example.com") == "t0001" + assert User.generate_username("t11@example.com") == "t110" + + user = add_user("support1", "user") + assert User.generate_username("support@example.com") == "support0" + def test_server_usage(client): """Test server usage endpoint""" diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 9d8671e8..ac839d95 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -801,7 +801,7 @@ def test_large_project_download_fail(client, diff_project): (test_project, "test.txt", "text/plain", 200), (test_project, "logo.pdf", "application/pdf", 200), (test_project, "logo.jpeg", "image/jpeg", 200), - (test_project, "base.gpkg", "application/geopackage+sqlite3", 200), + (test_project, "base.gpkg", "application/vnd.sqlite3", 200), (test_project, "json.json", "text/plain", 200), (test_project, "foo.txt", None, 404), ("bar", "test.txt", None, 404), @@ -2514,8 +2514,8 @@ def test_signals(client): push_finished_mock.assert_called_once() -def test_upload_validation(client): - """Test filepath and filename validation during file upload""" +def test_filepath_manipulation(client): + """Test filepath validation during file upload""" push_start_url = url_for( f"/v1.mergin_sync_public_api_controller_project_push", namespace=test_workspace_name, @@ -2530,7 +2530,7 @@ def test_upload_validation(client): "removed": [], } # Manipulate the path by prepending ../../ - manipulated_path = "../../image.png" + manipulated_path = "../../" + filename changes["added"][0]["path"] = manipulated_path # Block script upload in push_start because of the invalid path resp = client.post( @@ -2542,5 +2542,75 @@ def test_upload_validation(client): ) assert resp.status_code == 400 assert ( - resp.json["detail"] == f"File {manipulated_path} contains invalid characters." + resp.json["detail"] + == f"Unsupported file name detected: {manipulated_path}. Please remove the invalid characters." ) + + +def test_supported_file_upload(client): + """Test rejecting unsupported file based on extension and its mime type""" + push_start_url = url_for( + f"/v1.mergin_sync_public_api_controller_project_push", + namespace=test_workspace_name, + project_name=test_project, + ) + content = """#!/bin/bash + echo "Hello Mergin!" + """ + script_filename = "script.sh" + with open(os.path.join(TMP_DIR, script_filename), "w") as f: + f.write(content) + changes = { + "added": [file_info(TMP_DIR, script_filename, chunk_size=CHUNK_SIZE)], + "updated": [], + "removed": [], + } + # Block script upload during push_start because of the unsupported extension + resp = client.post( + push_start_url, + data=json.dumps( + {"version": "v1", "changes": changes}, cls=DateTimeEncoder + ).encode("utf-8"), + headers=json_headers, + ) + assert resp.status_code == 400 + assert ( + resp.json["detail"] + == f"Unsupported file type detected: {script_filename}. Please remove the file or try compressing it into a ZIP file before uploading" + ) + # Extension spoofing to trick the validator + spoof_name = "script.gpkg" + os.rename(os.path.join(TMP_DIR, script_filename), os.path.join(TMP_DIR, spoof_name)) + changes = { + "added": [file_info(TMP_DIR, spoof_name, chunk_size=CHUNK_SIZE)], + "updated": [], + "removed": [], + } + # File passes the extension check in push_start + resp = client.post( + push_start_url, + data=json.dumps( + {"version": "v1", "changes": changes}, cls=DateTimeEncoder + ).encode("utf-8"), + headers=json_headers, + ) + assert resp.status_code == 200 + upload = Upload.query.get(resp.json["transaction"]) + assert upload + # Even chunks are correctly uploaded + for file in changes["added"]: + for chunk_id in file["chunks"]: + url = "/v1/project/push/chunk/{}/{}".format(upload.id, chunk_id) + with open(os.path.join(TMP_DIR, file["path"]), "rb") as f: + data = f.read(CHUNK_SIZE) + checksum = hashlib.sha1() + checksum.update(data) + resp = client.post( + url, data=data, headers={"Content-Type": "application/octet-stream"} + ) + assert resp.status_code == 200 + assert resp.json["checksum"] == checksum.hexdigest() + # Unsupported file type is revealed when reconstructed from chunks - based on the mime type - and upload is refused + resp = client.post(f"/v1/project/push/finish/{upload.id}") + assert resp.status_code == 400 + assert resp.json["detail"] == f"Unsupported file type detected: {spoof_name}" diff --git a/server/migrations/community/2686074eff45_ce_base.py b/server/migrations/community/2686074eff45_ce_base.py index 35803ba7..91d269e0 100644 --- a/server/migrations/community/2686074eff45_ce_base.py +++ b/server/migrations/community/2686074eff45_ce_base.py @@ -2,10 +2,10 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -""" Base migration for CE, reflects the state of release 2021.6.1 +"""Base migration for CE, reflects the state of release 2021.6.1 Revision ID: 2686074eff45 -Revises: +Revises: Create Date: 2022-12-15 10:40:42.234210 """ diff --git a/server/migrations/community/5ad13be6f7ef_drop_error_details_index.py b/server/migrations/community/5ad13be6f7ef_drop_error_details_index.py new file mode 100644 index 00000000..ceb1d225 --- /dev/null +++ b/server/migrations/community/5ad13be6f7ef_drop_error_details_index.py @@ -0,0 +1,38 @@ +"""Drop error details index + +Revision ID: 5ad13be6f7ef +Revises: ba5051218de4 +Create Date: 2025-02-20 07:52:36.670158 + +""" + +from alembic import op +import sqlalchemy as sa +from alembic import context + + +# revision identifiers, used by Alembic. +revision = "5ad13be6f7ef" +down_revision = "ba5051218de4" +branch_labels = None +depends_on = None + + +def upgrade(): + conn = op.get_bind() + conn.execute( + sa.text( + """ + DROP INDEX IF EXISTS ix_sync_failures_history_error_details; + """ + ) + ) + + +def downgrade(): + op.create_index( + op.f("ix_sync_failures_history_error_details"), + "sync_failures_history", + ["error_details"], + unique=False, + ) diff --git a/web-app/Dockerfile b/web-app/Dockerfile index dfa28896..38bc206a 100644 --- a/web-app/Dockerfile +++ b/web-app/Dockerfile @@ -9,7 +9,7 @@ RUN yarn link:dependencies RUN yarn build:all RUN PUBLIC_PATH=/admin/ yarn build:all:admin -FROM nginxinc/nginx-unprivileged:1.25.5 +FROM nginxinc/nginx-unprivileged:1.27 MAINTAINER Martin Varga "martin.varga@lutraconsulting.co.uk" WORKDIR /usr/share/nginx/html # client app @@ -18,4 +18,5 @@ COPY --from=builder /mergin/web-app/packages/app/dist ./app COPY --from=builder /mergin/web-app/packages/admin-app/dist ./admin # basic nginx config to serve static files COPY ./nginx.proxy.conf /etc/nginx/conf.d/default.conf +EXPOSE 8080 ENTRYPOINT ["nginx", "-g", "daemon off;"] diff --git a/web-app/nginx.proxy.conf b/web-app/nginx.proxy.conf index 43c9beef..05c4bd06 100644 --- a/web-app/nginx.proxy.conf +++ b/web-app/nginx.proxy.conf @@ -1,5 +1,6 @@ server { - listen 80; + listen 8080; + listen [::]:8080; client_max_body_size 4G; location / { diff --git a/web-app/packages/admin-lib/src/modules/admin/components/CreateUserForm.vue b/web-app/packages/admin-lib/src/modules/admin/components/CreateUserForm.vue index a65aa841..8de69926 100644 --- a/web-app/packages/admin-lib/src/modules/admin/components/CreateUserForm.vue +++ b/web-app/packages/admin-lib/src/modules/admin/components/CreateUserForm.vue @@ -6,22 +6,6 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial