diff --git a/deployment/community/.env.template b/deployment/community/.env.template index b7def629..741e0e9c 100644 --- a/deployment/community/.env.template +++ b/deployment/community/.env.template @@ -112,11 +112,23 @@ MAIL_SUPPRESS_SEND=0 #MAX_CHUNK_SIZE=10 * 1024 * 1024 # 10485760 in bytes -#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 # max total files size for archive download +# data download + +#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10 # max total files size in bytes for archive download - 10 GB #USE_X_ACCEL=False # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/) USE_X_ACCEL=1 +#PARTIAL_ZIP_EXPIRATION=600 # in seconds + +#PROJECTS_ARCHIVES_DIR=LOCAL_PROJECTS/projects_archives # where to store archives for download + +# days for which archive is ready to download +# PROJECTS_ARCHIVES_EXPIRATION=7 # in days + +# If use x-accel buffering by download (no/yes) +# PROJECTS_ARCHIVES_X_ACCEL_BUFFERING="no" + # geodif related # where geodiff lib copies working files diff --git a/deployment/enterprise/.env.template b/deployment/enterprise/.env.template index 62553a19..f6bda021 100644 --- a/deployment/enterprise/.env.template +++ b/deployment/enterprise/.env.template @@ -106,10 +106,22 @@ MAIL_USERNAME=fix-me #MAX_CHUNK_SIZE=10 * 1024 * 1024 # 10485760 in bytes -#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 # max total files size for archive download +# data download + +#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 # max total files size in bytes for archive download #USE_X_ACCEL=False # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/) -USE_X_ACCEL=True +USE_X_ACCEL=1 + +#PARTIAL_ZIP_EXPIRATION=600 # in seconds + +#PROJECTS_ARCHIVES_DIR=LOCAL_PROJECTS/projects_archives # where to store archives for download + +# days for which archive is ready to download +# PROJECTS_ARCHIVES_EXPIRATION=7 # in days + +# If use x-accel buffering by download (no/yes) +# PROJECTS_ARCHIVES_X_ACCEL_BUFFERING="no" # celery diff --git a/server/mergin/sync/config.py b/server/mergin/sync/config.py index 26a9d14c..b182da6d 100644 --- a/server/mergin/sync/config.py +++ b/server/mergin/sync/config.py @@ -43,8 +43,8 @@ class Configuration(object): ) # max total files size for archive download MAX_DOWNLOAD_ARCHIVE_SIZE = config( - "MAX_DOWNLOAD_ARCHIVE_SIZE", default=1024 * 1024 * 1024 * 20, cast=int - ) # 20 GB + "MAX_DOWNLOAD_ARCHIVE_SIZE", default=1024 * 1024 * 1024 * 10, cast=int + ) # 10 GB PROJECT_ACCESS_REQUEST = config( "PROJECT_ACCESS_REQUEST", default=7 * 24 * 3600, cast=int ) @@ -63,4 +63,4 @@ class Configuration(object): default=os.path.join(LOCAL_PROJECTS, "geodiff_tmp"), ) # in seconds, older unfinished zips are moved to temp - PARTIAL_ZIP_EXPIRATION = config("PARTIAL_ZIP_EXPIRATION", default=300, cast=int) + PARTIAL_ZIP_EXPIRATION = config("PARTIAL_ZIP_EXPIRATION", default=600, cast=int) diff --git a/server/mergin/sync/private_api.yaml b/server/mergin/sync/private_api.yaml index 389a6d7b..8c7c8491 100644 --- a/server/mergin/sync/private_api.yaml +++ b/server/mergin/sync/private_api.yaml @@ -410,6 +410,8 @@ paths: description: Accepted "400": $ref: "#/components/responses/BadStatusResp" + "413": + $ref: "#/components/responses/FileTooLargeResp" "403": $ref: "#/components/responses/Forbidden" "404": @@ -423,6 +425,8 @@ components: description: Project not found. BadStatusResp: description: Invalid request. + FileTooLargeResp: + description: File is too large. InvalidDataResp: description: Invalid/unprocessable data. Success: diff --git a/server/mergin/sync/private_api_controller.py b/server/mergin/sync/private_api_controller.py index 88f84c35..8e0180d4 100644 --- a/server/mergin/sync/private_api_controller.py +++ b/server/mergin/sync/private_api_controller.py @@ -337,11 +337,7 @@ def download_project(id: str, version=None): # noqa: E501 # pylint: disable=W06 ).first_or_404("Project version does not exist") if project_version.project_size > current_app.config["MAX_DOWNLOAD_ARCHIVE_SIZE"]: - abort( - 400, - "The total size of requested files is too large to download as a single zip, " - "please use different method/client for download", - ) + abort(400) # check zip is already created if os.path.exists(project_version.zip_path): diff --git a/server/mergin/tests/test_private_project_api.py b/server/mergin/tests/test_private_project_api.py index 0b72952c..27021ecb 100644 --- a/server/mergin/tests/test_private_project_api.py +++ b/server/mergin/tests/test_private_project_api.py @@ -472,7 +472,6 @@ def test_large_project_download_fail(client, diff_project): ) ) assert resp.status_code == 400 - assert "The total size of requested files is too large" in resp.json["detail"] @patch("mergin.sync.tasks.create_project_version_zip.delay") diff --git a/web-app/packages/admin-lib/src/modules/admin/views/ProjectView.vue b/web-app/packages/admin-lib/src/modules/admin/views/ProjectView.vue index 5da36817..4a975d9a 100644 --- a/web-app/packages/admin-lib/src/modules/admin/views/ProjectView.vue +++ b/web-app/packages/admin-lib/src/modules/admin/views/ProjectView.vue @@ -101,6 +101,7 @@ + @@ -111,6 +112,7 @@ import { useProjectStore, ProjectApi, DownloadProgress, + DownloadFileLarge } from '@mergin/lib' import { computed, watch, defineProps } from 'vue' import { useRouter, useRoute } from 'vue-router' diff --git a/web-app/packages/lib/src/modules/project/components/DownloadFileLarge.vue b/web-app/packages/lib/src/modules/project/components/DownloadFileLarge.vue new file mode 100644 index 00000000..4865dc66 --- /dev/null +++ b/web-app/packages/lib/src/modules/project/components/DownloadFileLarge.vue @@ -0,0 +1,33 @@ + + + + + + + diff --git a/web-app/packages/lib/src/modules/project/components/DownloadProgress.vue b/web-app/packages/lib/src/modules/project/components/DownloadProgress.vue index ee9c60d8..61f99d48 100644 --- a/web-app/packages/lib/src/modules/project/components/DownloadProgress.vue +++ b/web-app/packages/lib/src/modules/project/components/DownloadProgress.vue @@ -30,8 +30,8 @@ export default defineComponent({ this.$toast.add({ group: 'download-progress', severity: 'info', - summary: `Downloading ${this.project?.name}`, - detail: 'Please wait while your project is being downloaded.', + summary: `Preparing archive`, + detail: `Your project ${this.project?.name} is being prepared for download.`, life: undefined }) }, diff --git a/web-app/packages/lib/src/modules/project/components/index.ts b/web-app/packages/lib/src/modules/project/components/index.ts index 2255fc00..87086c9a 100644 --- a/web-app/packages/lib/src/modules/project/components/index.ts +++ b/web-app/packages/lib/src/modules/project/components/index.ts @@ -26,3 +26,4 @@ export { default as FilesTable } from './FilesTable.vue' export { default as ProjectVersionsTable } from './ProjectVersionsTable.vue' export { default as ProjectVersionChanges } from './ProjectVersionChanges.vue' export { default as DownloadProgress } from './DownloadProgress.vue' +export { default as DownloadFileLarge } from './DownloadFileLarge.vue' diff --git a/web-app/packages/lib/src/modules/project/store.ts b/web-app/packages/lib/src/modules/project/store.ts index 0fc201d2..401c264d 100644 --- a/web-app/packages/lib/src/modules/project/store.ts +++ b/web-app/packages/lib/src/modules/project/store.ts @@ -8,7 +8,7 @@ import keyBy from 'lodash/keyBy' import omit from 'lodash/omit' import { defineStore, getActivePinia } from 'pinia' -import { DropdownOption, permissionUtils } from '@/common' +import { DropdownOption, errorUtils, permissionUtils } from '@/common' import { getErrorMessage } from '@/common/error_utils' import { waitCursor } from '@/common/html_utils' import { filesDiff } from '@/common/mergin_utils' @@ -708,14 +708,21 @@ export const useProjectStore = defineStore('projectModule', { const notificationStore = useNotificationStore() this.cancelDownloadArchive() this.projectDownloading = true + const errorMessage = + 'Failed to download project archive. Please try again later.' + const exceedMessage = + 'It seems like preparing your ZIP file is taking longer than expected. Please try again in a little while to download your file.' + const fileTooLargeMessage = + 'The requested archive is too large to download. Please use direct download with python client or plugin instead.' const delays = [...Array(3).fill(1000), ...Array(3).fill(3000), 5000] let retryCount = 0 const pollDownloadArchive = async () => { try { - if (retryCount > 100) { - notificationStore.error({ - text: 'Failed to download project. Please try again.' + if (retryCount > 125) { + notificationStore.warn({ + text: exceedMessage, + life: 6000 }) this.cancelDownloadArchive() return @@ -736,8 +743,18 @@ export const useProjectStore = defineStore('projectModule', { FileSaver.saveAs(payload.url) notificationStore.closeNotification() this.cancelDownloadArchive() - } catch { - notificationStore.error({ text: 'Failed to download project' }) + } catch (e) { + if (axios.isAxiosError(e) && e.response?.status === 400) { + notificationStore.error({ + group: 'download-large-error', + text: '', + life: 6000 + }) + } else { + notificationStore.error({ + text: errorMessage + }) + } this.cancelDownloadArchive() } } diff --git a/web-app/packages/lib/src/modules/project/views/ProjectViewTemplate.vue b/web-app/packages/lib/src/modules/project/views/ProjectViewTemplate.vue index 9ce0b3f5..16535528 100644 --- a/web-app/packages/lib/src/modules/project/views/ProjectViewTemplate.vue +++ b/web-app/packages/lib/src/modules/project/views/ProjectViewTemplate.vue @@ -113,6 +113,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + @@ -120,6 +121,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial import { mapActions, mapState } from 'pinia' import { defineComponent, PropType } from 'vue' +import DownloadFileLarge from '../components/DownloadFileLarge.vue' import DownloadProgress from '../components/DownloadProgress.vue' import { AppContainer, AppSection } from '@/common' @@ -149,7 +151,8 @@ export default defineComponent({ UploadDialog, AppContainer, AppSection, - DownloadProgress + DownloadProgress, + DownloadFileLarge }, props: { namespace: String,