diff --git a/server/application.py b/server/application.py index 2a4220b6..db46e12e 100644 --- a/server/application.py +++ b/server/application.py @@ -26,7 +26,7 @@ from mergin.sync.tasks import remove_temp_files, remove_projects_backups from mergin.celery import celery, configure_celery from mergin.stats.config import Configuration -from mergin.stats.tasks import send_statistics +from mergin.stats.tasks import save_statistics, send_statistics from mergin.stats.app import register as register_stats Configuration.SERVER_TYPE = "ce" @@ -65,6 +65,11 @@ def setup_periodic_tasks(sender, **kwargs): remove_projects_backups, name="remove old project backups", ) + sender.add_periodic_task( + crontab(hour="*/12"), + save_statistics, + name="Save usage statistics to database", + ) if Configuration.COLLECT_STATISTICS: sender.add_periodic_task( crontab(hour=randint(0, 5), minute=randint(0, 60)), diff --git a/server/mergin/stats/api.yaml b/server/mergin/stats/api.yaml index dd67d34b..382da567 100644 --- a/server/mergin/stats/api.yaml +++ b/server/mergin/stats/api.yaml @@ -22,6 +22,37 @@ paths: "404": $ref: "#/components/responses/NotFoundResp" x-openapi-router-controller: mergin.stats.controller + /app/admin/report: + get: + summary: Download statistics for server + operationId: download_report + x-openapi-router-controller: mergin.stats.controller + parameters: + - name: date_from + in: query + description: Start date for statistics (YYYY-MM-DD) + required: true + schema: + type: string + format: date + - name: date_to + in: query + description: End date for statistics (YYYY-MM-DD) + required: true + schema: + type: string + format: date + responses: + "200": + description: CSV file with statistics + content: + text/csv: + schema: + type: string + "400": + $ref: "#/components/responses/BadStatusResp" + "404": + $ref: "#/components/responses/NotFoundResp" components: responses: UnauthorizedError: diff --git a/server/mergin/stats/controller.py b/server/mergin/stats/controller.py index 6fe255aa..4606f1e5 100644 --- a/server/mergin/stats/controller.py +++ b/server/mergin/stats/controller.py @@ -2,11 +2,29 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +from dataclasses import asdict import requests -from flask import abort, current_app +from flask import abort, current_app, make_response +from datetime import datetime, time +from csv import DictWriter + +from mergin.auth.app import auth_required +from mergin.stats.models import MerginStatistics, ServerCallhomeData from .config import Configuration -from ..app import parse_version_string +from ..app import parse_version_string, db + + +class CsvTextBuilder(object): + """ + Mock csv writer that writes to text buffer + """ + + def __init__(self): + self.data = [] + + def write(self, row): + self.data.append(row) def get_latest_version(): @@ -29,3 +47,44 @@ def get_latest_version(): data = {**data, **parsed_version} return data, 200 + + +@auth_required(permissions=["admin"]) +def download_report(date_from: str, date_to: str): + """Download statistics from server instance""" + try: + # try to validate dates to prevent unhandled date formats + # add start of the day time and end of the day time to prevent bad filtering in db + parsed_from = datetime.combine( + datetime.strptime(date_from, "%Y-%m-%d"), time.min + ) + parsed_to = datetime.combine(datetime.strptime(date_to, "%Y-%m-%d"), time.max) + except ValueError: + abort(400, "Invalid date format") + + stats = ( + db.session.query(MerginStatistics.created_at, MerginStatistics.data) + .filter(MerginStatistics.created_at.between(parsed_from, parsed_to)) + .order_by(MerginStatistics.created_at.desc()) + .all() + ) + created_column = "created_at" + data = [ + { + **stat.data, + "created_at": datetime.isoformat(stat.created_at), + } + for stat in stats + ] + columns = list(ServerCallhomeData.__dataclass_fields__.keys()) + [created_column] + # get columns for data, this is usefull when we will update data json format (removing columns, adding new ones) + + builder = CsvTextBuilder() + writer = DictWriter(builder, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + writer.writerows(data) + csv_data = "".join(builder.data) + response = make_response(csv_data) + response.headers["Content-Disposition"] = f"attachment; filename=usage-report.csv" + response.mimetype = "text/csv" + return response diff --git a/server/mergin/stats/models.py b/server/mergin/stats/models.py index 320e0137..4fbc0af3 100644 --- a/server/mergin/stats/models.py +++ b/server/mergin/stats/models.py @@ -2,12 +2,30 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +from dataclasses import dataclass +from typing import Optional import uuid -from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.dialects.postgresql import UUID, JSONB +from datetime import datetime, timezone from ..app import db +@dataclass +class ServerCallhomeData: + service_uuid: Optional[str] + url: Optional[str] + contact_email: Optional[str] + licence: Optional[str] + projects_count: Optional[int] + users_count: Optional[int] + workspaces_count: Optional[int] + last_change: Optional[str] + server_version: Optional[str] + monthly_contributors: Optional[int] + editors: Optional[int] + + class MerginInfo(db.Model): """Information about deployment""" @@ -19,3 +37,14 @@ def __init__(self, service_id: str = None): self.service_id = uuid.UUID(service_id) else: self.service_id = uuid.uuid4() + + +class MerginStatistics(db.Model): + """Information about deployment""" + + id = db.Column(db.Integer, primary_key=True, autoincrement=True) + created_at = db.Column( + db.DateTime, index=True, nullable=False, server_default="now()" + ) + # data with statistics + data = db.Column(JSONB, nullable=False) diff --git a/server/mergin/stats/tasks.py b/server/mergin/stats/tasks.py index da7a9cb6..9812340d 100644 --- a/server/mergin/stats/tasks.py +++ b/server/mergin/stats/tasks.py @@ -2,20 +2,57 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +from dataclasses import asdict import requests -import datetime +from datetime import datetime, timedelta, timezone import json import logging from flask import current_app from sqlalchemy.sql.operators import is_ -from .models import MerginInfo +from .models import MerginInfo, MerginStatistics, ServerCallhomeData from ..celery import celery from ..app import db from ..auth.models import User from ..sync.models import Project +def get_callhome_data(info: MerginInfo | None = None) -> ServerCallhomeData: + """ + Get data about server to send to callhome service + """ + last_change_item = ( + db.session.query(Project.updated).order_by(Project.updated.desc()).first() + ) + service_uuid = str(info.service_id) if info else None + data = ServerCallhomeData( + service_uuid=service_uuid, + url=current_app.config["MERGIN_BASE_URL"], + contact_email=current_app.config["CONTACT_EMAIL"], + licence=current_app.config["SERVER_TYPE"], + projects_count=Project.query.filter(Project.removed_at.is_(None)).count(), + users_count=User.query.filter( + is_(User.username.ilike("deleted_%"), False) + ).count(), + workspaces_count=current_app.ws_handler.workspace_count(), + last_change=str(last_change_item.updated) + "Z" if last_change_item else "", + server_version=current_app.config["VERSION"], + monthly_contributors=current_app.ws_handler.monthly_contributors_count(), + editors=current_app.ws_handler.server_editors_count(), + ) + return data + + +@celery.task(ignore_result=True) +def save_statistics(): + """Save statistics about usage.""" + info = MerginInfo.query.first() + data = get_callhome_data(info) + stat = MerginStatistics(data=data) + db.session.add(stat) + db.session.commit() + + @celery.task(ignore_result=True) def send_statistics(): """Send statistics about usage.""" @@ -45,32 +82,12 @@ def send_statistics(): db.session.add(info) db.session.commit() - if ( - info.last_reported - and datetime.datetime.utcnow() - < info.last_reported + datetime.timedelta(hours=12) + if info.last_reported and datetime.utcnow() < info.last_reported + timedelta( + hours=12 ): return - last_change_item = ( - db.session.query(Project.updated).order_by(Project.updated.desc()).first() - ) - - data = { - "service_uuid": str(info.service_id), - "url": current_app.config["MERGIN_BASE_URL"], - "contact_email": current_app.config["CONTACT_EMAIL"], - "licence": current_app.config["SERVER_TYPE"], - "projects_count": Project.query.filter(Project.removed_at.is_(None)).count(), - "users_count": User.query.filter( - is_(User.username.ilike("deleted_%"), False) - ).count(), - "workspaces_count": current_app.ws_handler.workspace_count(), - "last_change": str(last_change_item.updated) + "Z" if last_change_item else "", - "server_version": current_app.config["VERSION"], - "monthly_contributors": current_app.ws_handler.monthly_contributors_count(), - "editors": current_app.ws_handler.server_editors_count(), - } + data = asdict(get_callhome_data(info)) try: resp = requests.post( @@ -78,7 +95,7 @@ def send_statistics(): data=json.dumps(data), ) if resp.ok: - info.last_reported = datetime.datetime.utcnow() + info.last_reported = datetime.utcnow() db.session.commit() else: logging.warning("Statistics error: " + str(resp.text)) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 4e3901d5..4305a15f 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -192,7 +192,7 @@ def is_active_workspace(workspace): return workspace.is_active or is_admin -def require_project(ws, project_name, permission): +def require_project(ws, project_name, permission) -> Project: workspace = current_app.ws_handler.get_by_name(ws) if not workspace: abort(404) diff --git a/server/mergin/tests/test_statistics.py b/server/mergin/tests/test_statistics.py index b73cb042..a87ba08c 100644 --- a/server/mergin/tests/test_statistics.py +++ b/server/mergin/tests/test_statistics.py @@ -2,6 +2,9 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +import csv +from dataclasses import asdict +from datetime import timedelta, timezone, datetime import json from unittest.mock import patch import requests @@ -11,8 +14,8 @@ from mergin.sync.models import Project, ProjectRole from ..app import db -from ..stats.tasks import send_statistics -from ..stats.models import MerginInfo +from ..stats.tasks import get_callhome_data, save_statistics, send_statistics +from ..stats.models import MerginInfo, MerginStatistics, ServerCallhomeData from .utils import Response, add_user, create_project, create_workspace @@ -152,3 +155,66 @@ def test_server_updates(client): mock.side_effect = requests.exceptions.RequestException("Some failure") resp = client.get(url) assert resp.status_code == 400 + + +def test_save_statistics(app, client): + """Test save statistics celery job""" + info = MerginInfo.query.first() + app.config["CONTACT_EMAIL"] = "test@example.com" + assert MerginStatistics.query.count() == 0 + save_statistics.s().apply() + assert MerginStatistics.query.count() == 1 + stats = MerginStatistics.query.order_by(MerginStatistics.created_at.desc()).first() + stats_json_data = get_callhome_data(info) + assert stats.created_at + assert stats.data == asdict(stats_json_data) + + +def test_download_report(app, client): + """Test download report endpoint""" + url = "/app/admin/report" + resp = client.get(url) + resp.status_code == 400 + + # bad date format + resp = client.get(f"{url}?date_from=2021-01-01T00:00:00&date_to=2021-01-01") + assert resp.status_code == 400 + + app.config["CONTACT_EMAIL"] = "test@example.com" + save_statistics.s().apply() + resp = client.get( + f"{url}?date_from=2021-01-01&date_to={datetime.now(timezone.utc).strftime('%Y-%m-%d')}" + ) + assert resp.status_code == 200 + assert resp.mimetype == "text/csv" + lines = resp.data.splitlines() + assert len(lines) == 2 + + stat = MerginStatistics.query.first() + keys = list(asdict(ServerCallhomeData(**stat.data)).keys()) + ["created_at"] + assert lines[0].decode("UTF-8") == ",".join(keys) + + # test same day + stat.created_at = datetime(2021, 1, 1, tzinfo=timezone.utc) + db.session.commit() + + resp = client.get( + f"{url}?date_from=2021-01-01&date_to={datetime.now(timezone.utc).strftime('%Y-%m-%d')}" + ) + assert resp.status_code == 200 + assert resp.mimetype == "text/csv" + lines = resp.data.splitlines() + assert len(lines) == 2 + + # empty response + stat.created_at = datetime(2020, 1, 1, tzinfo=timezone.utc) + db.session.commit() + resp = client.get( + f"{url}?date_from=2021-01-01&date_to={datetime.now(timezone.utc).strftime('%Y-%m-%d')}" + ) + assert resp.status_code == 200 + assert resp.mimetype == "text/csv" + lines = resp.data.splitlines() + empty_file = f"{','.join(keys)}\r\n" + assert resp.data.decode("UTF-8") == empty_file + assert len(lines) == 1 diff --git a/server/migrations/community/ba5051218de4_add_mergin_statistics_table.py b/server/migrations/community/ba5051218de4_add_mergin_statistics_table.py new file mode 100644 index 00000000..069bfb83 --- /dev/null +++ b/server/migrations/community/ba5051218de4_add_mergin_statistics_table.py @@ -0,0 +1,41 @@ +"""add mergin statistics table + +Revision ID: ba5051218de4 +Revises: d02961c7416c +Create Date: 2025-01-24 12:41:23.714579 + +""" + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "ba5051218de4" +down_revision = "d02961c7416c" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "mergin_statistics", + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column("created_at", sa.DateTime(), server_default="now()", nullable=False), + sa.Column("data", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_mergin_statistics")), + ) + op.create_index( + op.f("ix_mergin_statistics_created_at"), + "mergin_statistics", + ["created_at"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + op.drop_index( + op.f("ix_mergin_statistics_created_at"), table_name="mergin_statistics" + ) + op.drop_table("mergin_statistics") diff --git a/web-app/packages/admin-lib/components.d.ts b/web-app/packages/admin-lib/components.d.ts index 89889c60..bf16d936 100644 --- a/web-app/packages/admin-lib/components.d.ts +++ b/web-app/packages/admin-lib/components.d.ts @@ -9,6 +9,7 @@ declare module 'vue' { export interface GlobalComponents { PAvatar: typeof import('primevue/avatar')['default'] PButton: typeof import('primevue/button')['default'] + PCalendar: typeof import('primevue/calendar')['default'] PColumn: typeof import('primevue/column')['default'] PDataTable: typeof import('primevue/datatable')['default'] PDivider: typeof import('primevue/divider')['default'] diff --git a/web-app/packages/admin-lib/src/modules/admin/adminApi.ts b/web-app/packages/admin-lib/src/modules/admin/adminApi.ts index 6380ef6c..08114297 100644 --- a/web-app/packages/admin-lib/src/modules/admin/adminApi.ts +++ b/web-app/packages/admin-lib/src/modules/admin/adminApi.ts @@ -18,7 +18,8 @@ import { CreateUserData, PaginatedAdminProjectsResponse, PaginatedAdminProjectsParams, - ServerUsageResponse + ServerUsageResponse, + DownloadReportParams } from '@/modules/admin/types' export const AdminApi = { @@ -99,6 +100,23 @@ export const AdminApi = { }, async getServerUsage(): Promise> { - return AdminModule.httpService.get('/app/admin/usage', ) + return AdminModule.httpService.get('/app/admin/usage') + }, + + /** + * Create url for csv file with server statistics + */ + contructDownloadStatisticsUrl(): string { + return AdminModule.httpService.absUrl('/app/admin/report') + }, + + async downloadStatistics( + url: string, + params: DownloadReportParams + ): Promise> { + return AdminModule.httpService.get(url, { + responseType: 'blob', + params + }) } } diff --git a/web-app/packages/admin-lib/src/modules/admin/components/ReportDownloadDialog.vue b/web-app/packages/admin-lib/src/modules/admin/components/ReportDownloadDialog.vue new file mode 100644 index 00000000..6ba72cc8 --- /dev/null +++ b/web-app/packages/admin-lib/src/modules/admin/components/ReportDownloadDialog.vue @@ -0,0 +1,86 @@ + + + diff --git a/web-app/packages/admin-lib/src/modules/admin/store.ts b/web-app/packages/admin-lib/src/modules/admin/store.ts index 7b9281ff..1f76d4b1 100644 --- a/web-app/packages/admin-lib/src/modules/admin/store.ts +++ b/web-app/packages/admin-lib/src/modules/admin/store.ts @@ -13,6 +13,7 @@ import { useNotificationStore, UserResponse } from '@mergin/lib' +import FileSaver from 'file-saver' import { defineStore, getActivePinia } from 'pinia' import Cookies from 'universal-cookie' @@ -22,10 +23,12 @@ import { AdminApi } from '@/modules/admin/adminApi' import { LatestServerVersionResponse, PaginatedAdminProjectsParams, - PaginatedAdminProjectsResponse, ServerUsageResponse, + PaginatedAdminProjectsResponse, + ServerUsageResponse, UpdateUserPayload, UsersResponse } from '@/modules/admin/types' +import axios from 'axios' export interface AdminState { loading: boolean @@ -325,6 +328,43 @@ export const useAdminStore = defineStore('adminModule', { } catch (e) { notificationStore.error({ text: errorUtils.getErrorMessage(e) }) } + }, + + async downloadReport(payload: { from: Date; to: Date }) { + const notificationStore = useNotificationStore() + + try { + const { from, to } = payload + const url = AdminApi.contructDownloadStatisticsUrl() + const date = new Date() + // we need to sanitize hours from custom range in Calendar picker, because it's 00:00:00 by default. If you convert it to UTC -> it could be previous day, which is not what you selected. + from.setHours(date.getHours(), date.getMinutes(), date.getSeconds()) + to.setHours(date.getHours(), date.getMinutes(), date.getSeconds()) + // toISOString converts date to iso format and UTC zone + const date_from = payload.from.toISOString().split('T')[0] + const date_to = payload.to.toISOString().split('T')[0] + const resp = await AdminApi.downloadStatistics(url, { + date_from, + date_to + }) + const fileName = + resp.headers['content-disposition'].split('filename=')[1] + const extension = fileName.split('.')[1] + new FileSaver.saveAs( + resp.data, + `usage-report-${date.toISOString().split('T')[0]}.${extension}` + ) + } catch (e) { + // parse error details from blob + if (axios.isAxiosError(e)) { + let resp + const blob = new Blob([e.response.data], { type: 'text/plain' }) + blob.text().then((text) => { + resp = JSON.parse(text) + notificationStore.error({ text: resp.detail }) + }) + } + } } } }) diff --git a/web-app/packages/admin-lib/src/modules/admin/types.ts b/web-app/packages/admin-lib/src/modules/admin/types.ts index 2c3bc54d..e443b0a7 100644 --- a/web-app/packages/admin-lib/src/modules/admin/types.ts +++ b/web-app/packages/admin-lib/src/modules/admin/types.ts @@ -80,4 +80,9 @@ export interface ServerUsage { editors: number } +export interface DownloadReportParams { + date_from: string + date_to: string +} + /* eslint-enable camelcase */ diff --git a/web-app/packages/admin-lib/src/modules/admin/views/SettingsViewTemplate.vue b/web-app/packages/admin-lib/src/modules/admin/views/SettingsViewTemplate.vue index ee4880c7..3956ffbc 100644 --- a/web-app/packages/admin-lib/src/modules/admin/views/SettingsViewTemplate.vue +++ b/web-app/packages/admin-lib/src/modules/admin/views/SettingsViewTemplate.vue @@ -16,12 +16,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial - + + + @@ -33,9 +40,11 @@ import { AppContainer, AppSection, AppSettings, - AppSettingsItemConfig + AppSettingsItemConfig, + useDialogStore } from '@mergin/lib' +import ReportDownloadDialog from '../components/ReportDownloadDialog.vue' import { useAdminStore } from '../store' import AdminLayout from '@/modules/admin/components/AdminLayout.vue' @@ -46,16 +55,31 @@ withDefaults(defineProps<{ settingsItems?: AppSettingsItemConfig[] }>(), { title: 'Check for updates', description: 'Let Mergin Maps automatically check for new updates', key: 'checkForUpdates' + }, + { + title: 'Server usage report', + description: 'Download usage statistics for your server deployment.', + key: 'downloadReport' } ] }) const adminStore = useAdminStore() +const dialogStore = useDialogStore() function switchCheckForUpdates() { const value = !adminStore.checkForUpdates adminStore.setCheckUpdatesToCookies({ value }) } + +function downloadReport() { + dialogStore.show({ + component: ReportDownloadDialog, + params: { + dialog: { header: 'Download report' } + } + }) +} diff --git a/web-app/packages/lib/src/common/components/app-settings/AppSettings.vue b/web-app/packages/lib/src/common/components/app-settings/AppSettings.vue index 4627492d..29cd1800 100644 --- a/web-app/packages/lib/src/common/components/app-settings/AppSettings.vue +++ b/web-app/packages/lib/src/common/components/app-settings/AppSettings.vue @@ -5,6 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -->