Initial version for diagnostic logs:

deployment/community/.env.template

@@ -208,3 +208,7 @@ GEVENT_WORKER=True
 # Deprecated from 2024.7.0, replacement is to set GEVENT_WORKER=True
 NO_MONKEY_PATCH=False
 
+# Diagnostic logs
+
+DIAGNOSTIC_LOGS_DIR=/diagnostic_logs
+

deployment/community/README.md

@@ -13,4 +13,6 @@ cp .env.template .prod.env
 
 Next step is to create data directory for mergin maps `projects` with proper permissions. Should you prefer a different location, please do search and replace it in config files (`.prod.env`, `docker-compose.yml`). Make sure your volume is large enough since mergin maps keeps all projects files, their history and also needs some space for temporary processing.
 
+If you want to persist diagnostic logs, create data directory `diagnostic_logs` with proper permissions.
+
 For more details about deployment please check [docs](https://merginmaps.com/docs/server/install/#deployment).

deployment/community/docker-compose.yml

@@ -29,6 +29,7 @@ services:
     user: 901:999
     volumes:
       - ./projects:/data
+      - ./diagnostic_logs:/diagnostic_logs
       - ../common/entrypoint.sh:/app/entrypoint.sh
     env_file:
       - .prod.env

deployment/enterprise/.env.template

@@ -215,3 +215,6 @@ VECTOR_TILES_STYLE_URL=https://tiles-ee.merginmaps.com//styles/default.json
 #QGIS_EXTRACTOR_TIMEOUT=60
 
 #OVERVIEW_MAX_FILE_SIZE=1048576 # 1MB
+
+# Diagnostic logs from Mobile and QGIS Plugin
+DIAGNOSTIC_LOGS_DIR=/diagnostic_logs

deployment/enterprise/docker-compose.yml

@@ -12,6 +12,7 @@ services:
     command: ["gunicorn -w 4 --config config.py application:application"]
     volumes:
       - ./data:/data  # map data dir to host
+      - ./diagnostic_logs:/diagnostic_logs # diagnostic logs dir
       - ../common/entrypoint.sh:/app/entrypoint.sh
     env_file:
       - .prod.env

server/.test.env

@@ -23,3 +23,4 @@ GEODIFF_WORKING_DIR=/tmp/geodiff
 SECURITY_BEARER_SALT='bearer'
 SECURITY_EMAIL_SALT='email'
 SECURITY_PASSWORD_SALT='password'
+DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs

server/application.py

@@ -46,6 +46,7 @@
         "GLOBAL_READ",
         "GLOBAL_WRITE",
         "ENABLE_SUPERADMIN_ASSIGNMENT",
+        "DIAGNOSTIC_LOGS_URL",
     ]
 )
 register_stats(application)

server/mergin/api.yaml

@@ -0,0 +1,84 @@
+openapi: 3.0.0
+info:
+  description: Common Mergin Maps API
+  version: "0.1"
+  title: Common Mergin Maps API
+servers:
+  - url: /
+paths:
+  /v1/latest-version:
+    get:
+      summary: Fetch latest available server version
+      operationId: get_latest_version
+      responses:
+        "200":
+          description: Latest version info
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ServerVersion"
+        "400":
+          $ref: "#/components/responses/BadRequestResp"
+        "404":
+          $ref: "#/components/responses/NotFoundResp"
+      x-openapi-router-controller: mergin.controller
+  /v2/diagnostic-logs:
+    post:
+      summary: Save diagnostic log to the server
+      description: This endpoint allows users to upload diagnostic logs for troubleshooting purposes from mobile and plugin.
+      operationId: save_diagnostic_log
+      x-openapi-router-controller: mergin.controller
+      parameters:
+        - name: app
+          in: query
+          description: Application name (e.g., "input-android-0.9.0")
+          required: true
+          schema:
+            type: string
+      requestBody:
+        required: true
+        content:
+          text/plain:
+            schema:
+              type: string
+              description: Log content in plain text
+      responses:
+        "200":
+          description: Log saved successfully
+        "400":
+          $ref: "#/components/responses/BadRequestResp"
+        "404":
+          $ref: "#/components/responses/NotFoundResp"
+        "413":
+          $ref: "#/components/responses/RequestTooLarge"
+components:
+  responses:
+    UnauthorizedError:
+      description: Authentication information is missing or invalid.
+    NotFoundResp:
+      description: Not found
+    BadRequestResp:
+      description: Invalid request.
+    RequestTooLarge:
+      description: Request Entity Too Large.
+  schemas:
+    ServerVersion:
+      type: object
+      properties:
+        version:
+          type: string
+          example: 2023.1.1
+        major:
+          type: integer
+          example: 2023
+        minor:
+          type: integer
+          example: 1
+        fix:
+          nullable: true
+          type: integer
+          example: 1
+        info_url:
+          nullable: true
+          type: string
+          example: "https://github.com/MerginMaps/mergin/releases/tag/2023.1"

server/mergin/app.py

@@ -177,6 +177,12 @@ def create_app(public_keys: List[str] = None) -> Flask:
         options={"swagger_ui": False, "serve_spec": False},
         validate_responses=True,
     )
+    app.add_api(
+        "api.yaml",
+        arguments={"title": "Mergin"},
+        options={"swagger_ui": False, "serve_spec": False},
+        validate_responses=True,
+    )
 
     app.app.config.from_object(SyncConfig)
     app.app.connexion_app = app

server/mergin/config.py

@@ -7,6 +7,8 @@
 
 from .version import get_version
 
+config_dir = os.path.abspath(os.path.dirname(__file__))
+
 
 class Configuration(object):
     # flask/connexion variables
@@ -107,3 +109,16 @@ class Configuration(object):
     # using gevent type of worker impose some requirements on code, e.g. to be greenlet safe, custom timeouts
     GEVENT_WORKER = config("GEVENT_WORKER", default=False, cast=bool)
     GEVENT_REQUEST_TIMEOUT = config("GEVENT_REQUEST_TIMEOUT", default=30, cast=int)
+    DIAGNOSTIC_LOGS_URL = config(
+        "DIAGNOSTIC_LOGS_URL",
+        default="",
+    )
+    DIAGNOSTIC_LOGS_DIR = config(
+        "DIAGNOSTIC_LOGS_DIR",
+        default=os.path.join(
+            config_dir, os.pardir, os.pardir, os.pardir, "diagnostic_logs"
+        ),
+    )
+    DIAGNOSTIC_LOGS_MAX_SIZE = config(
+        "DIAGNOSTIC_LOGS_MAX_SIZE", default=1024 * 1024, cast=int
+    )

server/mergin/controller.py

@@ -0,0 +1,61 @@
+import json
+import logging
+import os
+from flask import abort, current_app, request
+from flask_login import current_user
+from magic import from_buffer
+import time
+
+import requests
+
+from .utils import save_diagnostic_log_file
+from .app import parse_version_string, db
+
+
+def get_latest_version():
+    """Parse information about available server updates from 3rd party service"""
+    try:
+        req = requests.get(current_app.config["STATISTICS_URL"] + "/latest-versions")
+    except requests.exceptions.RequestException:
+        abort(400, "Updates information not available")
+
+    if not req.ok:
+        abort(400, "Updates information not available")
+
+    data = req.json().get(current_app.config["SERVER_TYPE"].lower(), None)
+    if not data:
+        abort(400, "Updates information not available")
+
+    parsed_version = parse_version_string(data.get("version", ""))
+    if not parsed_version:
+        abort(400, "Updates information not available")
+
+    data = {**data, **parsed_version}
+    return data, 200
+
+
+def save_diagnostic_log():
+    """Save diagnostic logs"""
+    # if server is using external storage, we don't want to save logs
+    if current_app.config.get("DIAGNOSTIC_LOGS_URL"):
+        abort(404)
+
+    # check if plain text body is not larger than 1MB
+    max_size = current_app.config.get("DIAGNOSTIC_LOGS_MAX_SIZE")
+    if request.content_length > max_size:
+        abort(413)
+    # get body from request
+    body = request.get_data()
+    if not body:
+        abort(400)
+    if len(body) > max_size:
+        abort(413)
+    mime_type = from_buffer(body, mime=True)
+    if mime_type != "text/plain":
+        abort(400)
+
+    app = request.args.get("app")
+    username = current_user.username if current_user.is_authenticated else "anonymous"
+    save_diagnostic_log_file(app, username, body)
+
+    return "Log saved successfully", 200

server/mergin/stats/api.yaml

@@ -6,22 +6,6 @@ info:
 servers:
   - url: /
 paths:
-  /v1/latest-version:
-    get:
-      summary: Fetch latest available server version
-      operationId: get_latest_version
-      responses:
-        "200":
-          description: Latest version info
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ServerVersion"
-        "400":
-          $ref: "#/components/responses/BadStatusResp"
-        "404":
-          $ref: "#/components/responses/NotFoundResp"
-      x-openapi-router-controller: mergin.stats.controller
   /app/admin/report:
     get:
       summary: Download statistics for server
@@ -50,7 +34,7 @@ paths:
               schema:
                 type: string
         "400":
-          $ref: "#/components/responses/BadStatusResp"
+          $ref: "#/components/responses/BadRequestResp"
         "404":
           $ref: "#/components/responses/NotFoundResp"
 components:
@@ -59,26 +43,5 @@ components:
       description: Authentication information is missing or invalid.
     NotFoundResp:
       description: Project not found.
-    BadStatusResp:
+    BadRequestResp:
       description: Invalid request.
-  schemas:
-    ServerVersion:
-      type: object
-      properties:
-        version:
-          type: string
-          example: 2023.1.1
-        major:
-          type: integer
-          example: 2023
-        minor:
-          type: integer
-          example: 1
-        fix:
-          nullable: true
-          type: integer
-          example: 1
-        info_url:
-          nullable: true
-          type: string
-          example: "https://github.com/MerginMaps/mergin/releases/tag/2023.1"

server/mergin/stats/config.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
+import os
 from decouple import config
 
 

server/mergin/stats/controller.py

@@ -2,7 +2,6 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
-from dataclasses import asdict
 import requests
 from flask import abort, current_app, make_response
 from datetime import datetime, time
@@ -12,7 +11,7 @@
 from mergin.stats.models import MerginStatistics, ServerCallhomeData
 
 from .config import Configuration
-from ..app import parse_version_string, db
+from ..app import db
 
 
 class CsvTextBuilder(object):
@@ -27,28 +26,6 @@ def write(self, row):
         self.data.append(row)
 
 
-def get_latest_version():
-    """Parse information about available server updates from 3rd party service"""
-    try:
-        req = requests.get(Configuration.STATISTICS_URL + "/latest-versions")
-    except requests.exceptions.RequestException:
-        abort(400, "Updates information not available")
-
-    if not req.ok:
-        abort(400, "Updates information not available")
-
-    data = req.json().get(current_app.config["SERVER_TYPE"].lower(), None)
-    if not data:
-        abort(400, "Updates information not available")
-
-    parsed_version = parse_version_string(data.get("version", ""))
-    if not parsed_version:
-        abort(400, "Updates information not available")
-
-    data = {**data, **parsed_version}
-    return data, 200
-
-
 @auth_required(permissions=["admin"])
 def download_report(date_from: str, date_to: str):
     """Download statistics from server instance"""

server/mergin/tests/fixtures.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
 import os
+import shutil
 import sys
 import uuid
 from copy import deepcopy
@@ -80,6 +81,9 @@ def teardown():
                 if p.storage is not None
             ]
             cleanup(flask_app.test_client(), dirs)
+            diagnostic_logs_dir = flask_app.config.get("DIAGNOSTIC_LOGS_DIR")
+            if os.path.exists(diagnostic_logs_dir):
+                shutil.rmtree(diagnostic_logs_dir)
 
     request.addfinalizer(teardown)
     return flask_app