Release tweaks

deployment/community/.env.template

@@ -112,11 +112,23 @@ MAIL_SUPPRESS_SEND=0
 
 #MAX_CHUNK_SIZE=10 * 1024 * 1024  # 10485760 in bytes
 
-#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024  # max total files size for archive download
+# data download
+
+#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024 * 10  # max total files size in bytes for archive download - 10 GB
 
 #USE_X_ACCEL=False  # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/)
 USE_X_ACCEL=1
 
+#PARTIAL_ZIP_EXPIRATION=600 # in seconds
+
+#PROJECTS_ARCHIVES_DIR=LOCAL_PROJECTS/projects_archives  # where to store archives for download
+
+# days for which archive is ready to download
+# PROJECTS_ARCHIVES_EXPIRATION=7 # in days
+
+# If use x-accel buffering by download (no/yes)
+# PROJECTS_ARCHIVES_X_ACCEL_BUFFERING="no"
+
 # geodif related
 
 # where geodiff lib copies working files

deployment/enterprise/.env.template

@@ -106,10 +106,22 @@ MAIL_USERNAME=fix-me
 
 #MAX_CHUNK_SIZE=10 * 1024 * 1024  # 10485760 in bytes
 
-#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024  # max total files size for archive download
+# data download
+
+#MAX_DOWNLOAD_ARCHIVE_SIZE=1024 * 1024 * 1024  # max total files size in bytes for archive download
 
 #USE_X_ACCEL=False  # use nginx (in front of gunicorn) to serve files (https://www.nginx.com/resources/wiki/start/topics/examples/x-accel/)
-USE_X_ACCEL=True
+USE_X_ACCEL=1
+
+#PARTIAL_ZIP_EXPIRATION=600 # in seconds
+
+#PROJECTS_ARCHIVES_DIR=LOCAL_PROJECTS/projects_archives  # where to store archives for download
+
+# days for which archive is ready to download
+# PROJECTS_ARCHIVES_EXPIRATION=7 # in days
+
+# If use x-accel buffering by download (no/yes)
+# PROJECTS_ARCHIVES_X_ACCEL_BUFFERING="no"
 
 # celery
 

server/mergin/sync/config.py

@@ -43,8 +43,8 @@ class Configuration(object):
     )
     # max total files size for archive download
     MAX_DOWNLOAD_ARCHIVE_SIZE = config(
-        "MAX_DOWNLOAD_ARCHIVE_SIZE", default=1024 * 1024 * 1024 * 20, cast=int
-    )  # 20 GB
+        "MAX_DOWNLOAD_ARCHIVE_SIZE", default=1024 * 1024 * 1024 * 10, cast=int
+    )  # 10 GB
     PROJECT_ACCESS_REQUEST = config(
         "PROJECT_ACCESS_REQUEST", default=7 * 24 * 3600, cast=int
     )
@@ -63,4 +63,4 @@ class Configuration(object):
         default=os.path.join(LOCAL_PROJECTS, "geodiff_tmp"),
     )
     # in seconds, older unfinished zips are moved to temp
-    PARTIAL_ZIP_EXPIRATION = config("PARTIAL_ZIP_EXPIRATION", default=300, cast=int)
+    PARTIAL_ZIP_EXPIRATION = config("PARTIAL_ZIP_EXPIRATION", default=600, cast=int)

server/mergin/sync/private_api.yaml

@@ -410,6 +410,8 @@ paths:
           description: Accepted
         "400":
           $ref: "#/components/responses/BadStatusResp"
+        "413":
+          $ref: "#/components/responses/FileTooLargeResp"
         "403":
           $ref: "#/components/responses/Forbidden"
         "404":
@@ -423,6 +425,8 @@ components:
       description: Project not found.
     BadStatusResp:
       description: Invalid request.
+    FileTooLargeResp:
+      description: File is too large.
     InvalidDataResp:
       description: Invalid/unprocessable data.
     Success:

server/mergin/sync/private_api_controller.py

@@ -337,11 +337,7 @@ def download_project(id: str, version=None):  # noqa: E501 # pylint: disable=W06
     ).first_or_404("Project version does not exist")
 
     if project_version.project_size > current_app.config["MAX_DOWNLOAD_ARCHIVE_SIZE"]:
-        abort(
-            400,
-            "The total size of requested files is too large to download as a single zip, "
-            "please use different method/client for download",
-        )
+        abort(400)
 
     # check zip is already created
     if os.path.exists(project_version.zip_path):

server/mergin/tests/test_private_project_api.py

@@ -472,7 +472,6 @@ def test_large_project_download_fail(client, diff_project):
         )
     )
     assert resp.status_code == 400
-    assert "The total size of requested files is too large" in resp.json["detail"]
 
 
 @patch("mergin.sync.tasks.create_project_version_zip.delay")

web-app/packages/admin-lib/src/modules/admin/views/ProjectView.vue

@@ -101,6 +101,7 @@
       <router-view />
     </app-container>
     <download-progress />
+    <download-file-large />
   </admin-layout>
 </template>
 
@@ -111,6 +112,7 @@ import {
   useProjectStore,
   ProjectApi,
   DownloadProgress,
+  DownloadFileLarge
 } from '@mergin/lib'
 import { computed, watch, defineProps } from 'vue'
 import { useRouter, useRoute } from 'vue-router'

web-app/packages/lib/src/modules/project/components/DownloadFileLarge.vue

@@ -0,0 +1,33 @@
+<!--
+Copyright (C) Lutra Consulting Limited
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+-->
+
+<template>
+  <PToast group="download-large-error" position="top-center">
+    <template #message>
+      <p>
+        The project is too large to download. Please use direct download with
+        our Python API Client or QGIS plugin instead.
+        <a
+          href="https://merginmaps.com/blog/support-tip-downloading-a-previous-version-of-your-project"
+          target="_blank"
+          class="font-semibold"
+          >Read our blog for more details</a
+        >.
+      </p>
+    </template>
+  </PToast>
+</template>
+
+<script lang="ts">
+import { defineComponent } from 'vue'
+
+// we need to use options api here , because useToast is not properly handled multiple toast services
+export default defineComponent({
+  name: 'DownloadFileLarge'
+})
+</script>
+
+<style lang="scss" scoped></style>

web-app/packages/lib/src/modules/project/components/DownloadProgress.vue

@@ -30,8 +30,8 @@ export default defineComponent({
       this.$toast.add({
         group: 'download-progress',
         severity: 'info',
-        summary: `Downloading ${this.project?.name}`,
-        detail: 'Please wait while your project is being downloaded.',
+        summary: `Preparing archive`,
+        detail: `Your project ${this.project?.name} is being prepared for download.`,
         life: undefined
       })
     },

web-app/packages/lib/src/modules/project/components/index.ts

@@ -26,3 +26,4 @@ export { default as FilesTable } from './FilesTable.vue'
 export { default as ProjectVersionsTable } from './ProjectVersionsTable.vue'
 export { default as ProjectVersionChanges } from './ProjectVersionChanges.vue'
 export { default as DownloadProgress } from './DownloadProgress.vue'
+export { default as DownloadFileLarge } from './DownloadFileLarge.vue'

web-app/packages/lib/src/modules/project/store.ts

@@ -8,7 +8,7 @@
 import omit from 'lodash/omit'
 import { defineStore, getActivePinia } from 'pinia'
 
-import { DropdownOption, permissionUtils } from '@/common'
+import { DropdownOption, errorUtils, permissionUtils } from '@/common'
 import { getErrorMessage } from '@/common/error_utils'
 import { waitCursor } from '@/common/html_utils'
 import { filesDiff } from '@/common/mergin_utils'
@@ -708,14 +708,21 @@
       const notificationStore = useNotificationStore()
       this.cancelDownloadArchive()
       this.projectDownloading = true
+      const errorMessage =
+        'Failed to download project archive. Please try again later.'
+      const exceedMessage =
+        'It seems like preparing your ZIP file is taking longer than expected. Please try again in a little while to download your file.'
+      const fileTooLargeMessage =
+        'The requested archive is too large to download. Please use direct download with python client or plugin instead.'
 
       const delays = [...Array(3).fill(1000), ...Array(3).fill(3000), 5000]
       let retryCount = 0
       const pollDownloadArchive = async () => {
         try {
-          if (retryCount > 100) {
-            notificationStore.error({
-              text: 'Failed to download project. Please try again.'
+          if (retryCount > 125) {
+            notificationStore.warn({
+              text: exceedMessage,
+              life: 6000
             })
             this.cancelDownloadArchive()
             return
@@ -736,8 +743,18 @@
           FileSaver.saveAs(payload.url)
           notificationStore.closeNotification()
           this.cancelDownloadArchive()
-        } catch {
-          notificationStore.error({ text: 'Failed to download project' })
+        } catch (e) {
+          if (axios.isAxiosError(e) && e.response?.status === 400) {
+            notificationStore.error({
+              group: 'download-large-error',
+              text: '',
+              life: 6000
+            })
+          } else {
+            notificationStore.error({
+              text: errorMessage
+            })
+          }
           this.cancelDownloadArchive()
         }
       }

web-app/packages/lib/src/modules/project/views/ProjectViewTemplate.vue

@@ -113,13 +113,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
       <upload-dialog v-if="upload" :namespace="namespace" />
     </slot>
     <DownloadProgress />
+    <DownloadFileLarge />
   </div>
 </template>
 
 <script lang="ts">
 import { mapActions, mapState } from 'pinia'
 import { defineComponent, PropType } from 'vue'
 
+import DownloadFileLarge from '../components/DownloadFileLarge.vue'
 import DownloadProgress from '../components/DownloadProgress.vue'
 
 import { AppContainer, AppSection } from '@/common'
@@ -149,7 +151,8 @@ export default defineComponent({
     UploadDialog,
     AppContainer,
     AppSection,
-    DownloadProgress
+    DownloadProgress,
+    DownloadFileLarge
   },
   props: {
     namespace: String,