From 4726593a2479a0297ef7c56e43dc2c99c3a4021f Mon Sep 17 00:00:00 2001 From: Hudell Date: Tue, 20 Feb 2018 18:08:28 -0300 Subject: [PATCH 01/14] Implemented basic JSON generation --- .../rocketchat-lib/server/models/Messages.js | 8 ++ .../client/accountPreferences.html | 9 ++ .../client/accountPreferences.js | 15 ++++ server/methods/downloadMyData.js | 84 +++++++++++++++++++ 4 files changed, 116 insertions(+) create mode 100644 server/methods/downloadMyData.js diff --git a/packages/rocketchat-lib/server/models/Messages.js b/packages/rocketchat-lib/server/models/Messages.js index 90f8bf068765d..1b6b4d3d775c8 100644 --- a/packages/rocketchat-lib/server/models/Messages.js +++ b/packages/rocketchat-lib/server/models/Messages.js @@ -284,6 +284,14 @@ RocketChat.models.Messages = new class extends RocketChat.models._Base { return this.findOne(query); } + findByRoomId(roomId, options) { + const query = { + rid: roomId + }; + + return this.find(query, options); + } + getLastVisibleMessageSentWithNoTypeByRoomId(rid, messageId) { const query = { rid, diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html index 20e2efaf6b254..e87721d1088d5 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.html +++ b/packages/rocketchat-ui-account/client/accountPreferences.html @@ -291,6 +291,15 @@

{{_ "Sound"}}

+
+

{{_ "My Data"}}

+
+
+ +
+ +
+
diff --git a/packages/rocketchat-ui-account/client/accountPreferences.js b/packages/rocketchat-ui-account/client/accountPreferences.js index 35b42f53b689d..a622d1ca9ab95 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.js +++ b/packages/rocketchat-ui-account/client/accountPreferences.js @@ -203,6 +203,18 @@ Template.accountPreferences.onCreated(function() { } }); }; + + this.downloadMyData = function() { + Meteor.call('downloadMyData', {}, function(error, results) { + if (results) { + + } + + if (error) { + return handleError(error); + } + }); + }; }); Template.accountPreferences.onRendered(function() { @@ -222,6 +234,9 @@ Template.accountPreferences.events({ 'click .enable-notifications'() { KonchatNotification.getDesktopPermission(); }, + 'click .download-my-data'(e, t) { + t.downloadMyData(); + }, 'click .test-notifications'() { KonchatNotification.notify({ duration: $('input[name=desktopNotificationDuration]').val(), diff --git a/server/methods/downloadMyData.js b/server/methods/downloadMyData.js new file mode 100644 index 0000000000000..636de20afad86 --- /dev/null +++ b/server/methods/downloadMyData.js @@ -0,0 +1,84 @@ +import fs from 'fs'; +import path from 'path'; + +const startFile = function(fileName, content) { + fs.writeFileSync(fileName, content); +}; + +const writeToFile = function(fileName, content) { + fs.appendFileSync(fileName, content); +}; + +const createDir = function(folderName) { + if (!fs.existsSync(folderName)) { + fs.mkdirSync(folderName); + } +}; + +Meteor.methods({ + downloadMyData() { + const currentUserData = Meteor.user(); + const subscriptions = RocketChat.models.Subscriptions.findByUserId(currentUserData._id).fetch(); + + subscriptions.forEach((subscription) => { + const roomId = subscription._room._id; + const roomData = subscription._room; + const roomName = roomData.name ? roomData.name : roomId; + + const folderName = path.join('/tmp/', currentUserData._id); + + createDir(folderName); + const fileName = `${ roomName }.json`; + const filePath = path.join(folderName, fileName); + + startFile(filePath, '['); + + let loadedAllMessages = false; + let needsComma = false; + const limit = 20; + let skip = 0; + + while (!loadedAllMessages) { + const messages = RocketChat.models.Messages.findByRoomId(roomId, { limit, skip }).fetch(); + if (messages.length === 0) { + loadedAllMessages = true; + break; + } + + skip += limit; + + messages.forEach((msg) => { + const attachments = []; + + if (msg.attachments) { + msg.attachments.forEach((attachment) => { + attachments.push({ + type : attachment.type, + title : attachment.title, + url : attachment.image_url || attachment.audio_url + }); + }); + } + + const messageObject = { + msg: msg.msg, + username: msg.u.username, + name: msg.u.name || '', + attachments + }; + + let messageString = JSON.stringify(messageObject); + if (needsComma) { + messageString = `,\n${ messageString }`; + } + + writeToFile(filePath, messageString); + needsComma = true; + }); + } + + writeToFile(filePath, ']'); + }); + } + +}); From 0945340801f75b5282b975f317654f92fb7589a8 Mon Sep 17 00:00:00 2001 From: Hudell Date: Wed, 21 Feb 2018 11:01:05 -0300 Subject: [PATCH 02/14] Added new data to message json --- server/methods/downloadMyData.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/server/methods/downloadMyData.js b/server/methods/downloadMyData.js index 636de20afad86..666366f1759ac 100644 --- a/server/methods/downloadMyData.js +++ b/server/methods/downloadMyData.js @@ -55,18 +55,26 @@ Meteor.methods({ attachments.push({ type : attachment.type, title : attachment.title, - url : attachment.image_url || attachment.audio_url + url : attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url, }); }); } const messageObject = { msg: msg.msg, - username: msg.u.username, - name: msg.u.name || '', - attachments + username: msg.u.username }; + if (attachments && attachments.length > 0) { + messageObject.attachments = attachments; + } + if (msg.t) { + messageObject.type = msg.t; + } + if (msg.u.name) { + messageObject.name = msg.u.name; + } + let messageString = JSON.stringify(messageObject); if (needsComma) { messageString = `,\n${ messageString }`; From 5ad3755b8675042ca4d2eca8f727957adec9ef82 Mon Sep 17 00:00:00 2001 From: Hudell Date: Wed, 21 Feb 2018 13:07:32 -0300 Subject: [PATCH 03/14] Changed export operation structure --- server/methods/downloadMyData.js | 186 ++++++++++++++++++++----------- 1 file changed, 118 insertions(+), 68 deletions(-) diff --git a/server/methods/downloadMyData.js b/server/methods/downloadMyData.js index 666366f1759ac..4b9cc3f128904 100644 --- a/server/methods/downloadMyData.js +++ b/server/methods/downloadMyData.js @@ -15,78 +15,128 @@ const createDir = function(folderName) { } }; +const loadUserSubscriptions = function(exportOperation) { + exportOperation.roomList = []; + + const subscriptions = RocketChat.models.Subscriptions.findByUserId(exportOperation.userId).fetch(); + subscriptions.forEach((subscription) => { + const roomId = subscription._room._id; + const roomData = subscription._room; + const roomName = roomData.name ? roomData.name : roomId; + + const fileName = `${ roomName }.json`; + + exportOperation.roomList.push({ + roomId, + roomName, + exportedCount: 0, + status: 'pending', + targetFile: fileName + }); + }); +}; + +const continueExportingRoom = function(exportOperation, exportOpRoomData) { + const filePath = path.join(exportOperation.exportPath, exportOpRoomData.targetFile); + + if (exportOpRoomData.status == 'pending') { + exportOpRoomData.status = 'exporting'; + startFile(filePath, '['); + } + + const limit = 20; + let skip = exportOpRoomData.exportedCount; + + const messages = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }).fetch(); + if (messages.length === 0) { + writeToFile(filePath, ']'); + exportOpRoomData.status = 'completed'; + return; + } + + messages.forEach((msg) => { + const attachments = []; + const needsComma = exportOpRoomData.exportedCount > 0; + + if (msg.attachments) { + msg.attachments.forEach((attachment) => { + attachments.push({ + type : attachment.type, + title : attachment.title, + url : attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url, + }); + }); + } + + const messageObject = { + msg: msg.msg, + username: msg.u.username + }; + + if (attachments && attachments.length > 0) { + messageObject.attachments = attachments; + } + if (msg.t) { + messageObject.type = msg.t; + } + if (msg.u.name) { + messageObject.name = msg.u.name; + } + + let messageString = JSON.stringify(messageObject); + if (needsComma) { + messageString = `,\n${ messageString }`; + } + + writeToFile(filePath, messageString); + exportOpRoomData.exportedCount++; + }); + +}; + +const continueExportOperation = function(exportOperation) { + if (exportOperation.status == 'completed') { + return true; + } + + if (!exportOperation.roomList) { + loadUserSubscriptions(exportOperation); + } + + let nextRoom = false; + exportOperation.roomList.forEach((exportOpRoomData) => { + if (exportOpRoomData.status == 'completed') { + return; + } + + nextRoom = exportOpRoomData; + return false; + }); + + if (nextRoom) { + continueExportingRoom(exportOperation, nextRoom); + return false; + } + + exportOperation.status = 'completed'; + return true; +}; + Meteor.methods({ downloadMyData() { const currentUserData = Meteor.user(); - const subscriptions = RocketChat.models.Subscriptions.findByUserId(currentUserData._id).fetch(); - - subscriptions.forEach((subscription) => { - const roomId = subscription._room._id; - const roomData = subscription._room; - const roomName = roomData.name ? roomData.name : roomId; - - const folderName = path.join('/tmp/', currentUserData._id); - - createDir(folderName); - const fileName = `${ roomName }.json`; - const filePath = path.join(folderName, fileName); - - startFile(filePath, '['); - - let loadedAllMessages = false; - let needsComma = false; - const limit = 20; - let skip = 0; - - while (!loadedAllMessages) { - const messages = RocketChat.models.Messages.findByRoomId(roomId, { limit, skip }).fetch(); - if (messages.length === 0) { - loadedAllMessages = true; - break; - } - - skip += limit; - - messages.forEach((msg) => { - const attachments = []; - - if (msg.attachments) { - msg.attachments.forEach((attachment) => { - attachments.push({ - type : attachment.type, - title : attachment.title, - url : attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url, - }); - }); - } - - const messageObject = { - msg: msg.msg, - username: msg.u.username - }; - - if (attachments && attachments.length > 0) { - messageObject.attachments = attachments; - } - if (msg.t) { - messageObject.type = msg.t; - } - if (msg.u.name) { - messageObject.name = msg.u.name; - } - - let messageString = JSON.stringify(messageObject); - if (needsComma) { - messageString = `,\n${ messageString }`; - } - - writeToFile(filePath, messageString); - needsComma = true; - }); - } - writeToFile(filePath, ']'); - }); + const folderName = path.join('/tmp/', currentUserData._id); + const exportOperation = { + userId : currentUserData._id, + roomList: null, + status: 'pending', + exportPath: folderName + }; + + while (exportOperation.status != 'completed') { + continueExportOperation(exportOperation); + } } }); From e3c8f0a3c8be3d9cfb50a1e9d83f8ca663059ca4 Mon Sep 17 00:00:00 2001 From: Hudell Date: Wed, 21 Feb 2018 14:46:23 -0300 Subject: [PATCH 04/14] Changed code to use forEach instead of fetch --- server/methods/downloadMyData.js | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/server/methods/downloadMyData.js b/server/methods/downloadMyData.js index 4b9cc3f128904..665843f41d257 100644 --- a/server/methods/downloadMyData.js +++ b/server/methods/downloadMyData.js @@ -18,8 +18,8 @@ const createDir = function(folderName) { const loadUserSubscriptions = function(exportOperation) { exportOperation.roomList = []; - const subscriptions = RocketChat.models.Subscriptions.findByUserId(exportOperation.userId).fetch(); - subscriptions.forEach((subscription) => { + const cursor = RocketChat.models.Subscriptions.findByUserId(exportOperation.userId); + cursor.forEach((subscription) => { const roomId = subscription._room._id; const roomData = subscription._room; const roomName = roomData.name ? roomData.name : roomId; @@ -47,14 +47,16 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { const limit = 20; let skip = exportOpRoomData.exportedCount; - const messages = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }).fetch(); - if (messages.length === 0) { + const cursor = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }); + const count = cursor.count(); + + if (count <= exportOpRoomData.exportedCount) { writeToFile(filePath, ']'); exportOpRoomData.status = 'completed'; return; } - messages.forEach((msg) => { + cursor.forEach((msg) => { const attachments = []; const needsComma = exportOpRoomData.exportedCount > 0; @@ -104,13 +106,13 @@ const continueExportOperation = function(exportOperation) { } let nextRoom = false; - exportOperation.roomList.forEach((exportOpRoomData) => { + exportOperation.roomList.some((exportOpRoomData) => { if (exportOpRoomData.status == 'completed') { - return; + return false; } nextRoom = exportOpRoomData; - return false; + return true; }); if (nextRoom) { @@ -135,7 +137,9 @@ Meteor.methods({ }; while (exportOperation.status != 'completed') { - continueExportOperation(exportOperation); + if (continueExportOperation(exportOperation)) { + break; + } } } From cdec24ac01bf622c5d26f9268a99f73675a1df24 Mon Sep 17 00:00:00 2001 From: Hudell Date: Wed, 21 Feb 2018 16:53:31 -0300 Subject: [PATCH 05/14] Split the operation into two independent commands --- packages/rocketchat-lib/package.js | 1 + .../server/models/ExportOperations.js | 61 ++++++++++++++ .../client/accountPreferences.html | 1 + .../client/accountPreferences.js | 17 +++- ...nloadMyData.js => processDataDownloads.js} | 80 +++++++++---------- server/methods/requestDataDownload.js | 17 ++++ 6 files changed, 136 insertions(+), 41 deletions(-) create mode 100644 packages/rocketchat-lib/server/models/ExportOperations.js rename server/methods/{downloadMyData.js => processDataDownloads.js} (67%) create mode 100644 server/methods/requestDataDownload.js diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js index 6cdea174f1f7a..4cbf37eadcf08 100644 --- a/packages/rocketchat-lib/package.js +++ b/packages/rocketchat-lib/package.js @@ -126,6 +126,7 @@ Package.onUse(function(api) { api.addFiles('server/models/Subscriptions.js', 'server'); api.addFiles('server/models/Uploads.js', 'server'); api.addFiles('server/models/Users.js', 'server'); + api.addFiles('server/models/ExportOperations.js', 'server'); api.addFiles('server/oauth/oauth.js', 'server'); api.addFiles('server/oauth/google.js', 'server'); diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js new file mode 100644 index 0000000000000..e14ed03ecd8fd --- /dev/null +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -0,0 +1,61 @@ +import _ from 'underscore'; + +RocketChat.models.ExportOperations = new class ModelExportOperations extends RocketChat.models._Base { + constructor() { + super('exportOperations'); + + this.tryEnsureIndex({ 'userId': 1 }); + this.tryEnsureIndex({ 'status': 1 }); + } + + // FIND + findById(id) { + const query = {_id: id}; + + return this.find(query); + } + + findPendingByUser(userId) { + const query = {userId}; + + return this.find(query); + } + + findAllPending(options) { + const query = { + status: { $in: ['pending', 'exporting'] } + }; + + return this.find(query, options); + } + + // UPDATE + updateOperation(data) { + const update = { + $set: { + roomList: data.roomList, + status: data.status + } + }; + + return this.update(data._id, update); + } + + + // INSERT + create(data) { + const exportOperation = { + createdAt: new Date + }; + + _.extend(exportOperation, data); + + return this.insert(exportOperation); + } + + + // REMOVE + removeById(_id) { + return this.remove(_id); + } +}; diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html index 6ebbf6e03c94e..274c83a753d17 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.html +++ b/packages/rocketchat-ui-account/client/accountPreferences.html @@ -296,6 +296,7 @@

{{_ "My Data"}}

+
diff --git a/packages/rocketchat-ui-account/client/accountPreferences.js b/packages/rocketchat-ui-account/client/accountPreferences.js index 00b09e71ba217..6b1ec68c47ef5 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.js +++ b/packages/rocketchat-ui-account/client/accountPreferences.js @@ -197,7 +197,19 @@ Template.accountPreferences.onCreated(function() { }; this.downloadMyData = function() { - Meteor.call('downloadMyData', {}, function(error, results) { + Meteor.call('requestDataDownload', {}, function(error, results) { + if (results) { + + } + + if (error) { + return handleError(error); + } + }); + }; + + this.processDataDownload = function() { + Meteor.call('processDataDownloads', {}, function(error, results) { if (results) { } @@ -229,6 +241,9 @@ Template.accountPreferences.events({ 'click .download-my-data'(e, t) { t.downloadMyData(); }, + 'click .process-data-download'(e, t) { + t.processDataDownload(); + }, 'click .test-notifications'(e) { e.preventDefault(); KonchatNotification.notify({ diff --git a/server/methods/downloadMyData.js b/server/methods/processDataDownloads.js similarity index 67% rename from server/methods/downloadMyData.js rename to server/methods/processDataDownloads.js index 665843f41d257..ed0be6c739d6e 100644 --- a/server/methods/downloadMyData.js +++ b/server/methods/processDataDownloads.js @@ -34,45 +34,42 @@ const loadUserSubscriptions = function(exportOperation) { targetFile: fileName }); }); + + exportOperation.status = 'exporting'; }; const continueExportingRoom = function(exportOperation, exportOpRoomData) { + createDir(exportOperation.exportPath); const filePath = path.join(exportOperation.exportPath, exportOpRoomData.targetFile); - if (exportOpRoomData.status == 'pending') { + if (exportOpRoomData.status === 'pending') { exportOpRoomData.status = 'exporting'; - startFile(filePath, '['); + startFile(filePath, ''); } - const limit = 20; - let skip = exportOpRoomData.exportedCount; + const limit = 50; + const skip = exportOpRoomData.exportedCount; const cursor = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }); const count = cursor.count(); - if (count <= exportOpRoomData.exportedCount) { - writeToFile(filePath, ']'); - exportOpRoomData.status = 'completed'; - return; - } - cursor.forEach((msg) => { const attachments = []; - const needsComma = exportOpRoomData.exportedCount > 0; if (msg.attachments) { msg.attachments.forEach((attachment) => { attachments.push({ type : attachment.type, title : attachment.title, - url : attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url, + url : attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url }); }); } const messageObject = { msg: msg.msg, - username: msg.u.username + username: msg.u.username, + ts: msg.ts }; if (attachments && attachments.length > 0) { @@ -85,19 +82,30 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { messageObject.name = msg.u.name; } - let messageString = JSON.stringify(messageObject); - if (needsComma) { - messageString = `,\n${ messageString }`; - } + const messageString = JSON.stringify(messageObject); - writeToFile(filePath, messageString); + writeToFile(filePath, `${ messageString }\n`); exportOpRoomData.exportedCount++; }); + if (count <= exportOpRoomData.exportedCount) { + exportOpRoomData.status = 'completed'; + return true; + } + + return false; +}; + +const isOperationFinished = function(exportOperation) { + const incomplete = exportOperation.roomList.some((exportOpRoomData) => { + return exportOpRoomData.status !== 'completed'; + }); + + return !incomplete; }; const continueExportOperation = function(exportOperation) { - if (exportOperation.status == 'completed') { + if (exportOperation.status === 'completed') { return true; } @@ -107,7 +115,7 @@ const continueExportOperation = function(exportOperation) { let nextRoom = false; exportOperation.roomList.some((exportOpRoomData) => { - if (exportOpRoomData.status == 'completed') { + if (exportOpRoomData.status === 'completed') { return false; } @@ -117,30 +125,22 @@ const continueExportOperation = function(exportOperation) { if (nextRoom) { continueExportingRoom(exportOperation, nextRoom); - return false; } - exportOperation.status = 'completed'; - return true; + if (isOperationFinished(exportOperation)) { + exportOperation.status = 'completed'; + return true; + } + + return false; }; Meteor.methods({ - downloadMyData() { - const currentUserData = Meteor.user(); - - const folderName = path.join('/tmp/', currentUserData._id); - const exportOperation = { - userId : currentUserData._id, - roomList: null, - status: 'pending', - exportPath: folderName - }; - - while (exportOperation.status != 'completed') { - if (continueExportOperation(exportOperation)) { - break; - } - } + processDataDownloads() { + const cursor = RocketChat.models.ExportOperations.findAllPending({limit: 1}); + cursor.forEach((exportOperation) => { + continueExportOperation(exportOperation); + RocketChat.models.ExportOperations.updateOperation(exportOperation); + }); } - }); diff --git a/server/methods/requestDataDownload.js b/server/methods/requestDataDownload.js new file mode 100644 index 0000000000000..6414630932f42 --- /dev/null +++ b/server/methods/requestDataDownload.js @@ -0,0 +1,17 @@ +import path from 'path'; + +Meteor.methods({ + requestDataDownload() { + const currentUserData = Meteor.user(); + + const folderName = path.join('/tmp/', currentUserData._id); + const exportOperation = { + userId : currentUserData._id, + roomList: null, + status: 'pending', + exportPath: folderName + }; + + RocketChat.models.ExportOperations.create(exportOperation); + } +}); From b41d67c76f40f156a66149773bcd37c70d876c7d Mon Sep 17 00:00:00 2001 From: Hudell Date: Fri, 23 Feb 2018 17:21:31 -0300 Subject: [PATCH 06/14] File download, zip generation, admin settings --- .meteor/packages | 1 + .meteor/versions | 1 + package.json | 1 + .../server/config/FileSystem.js | 16 ++ .../server/config/GridFS.js | 17 +- .../server/lib/FileUpload.js | 21 ++- packages/rocketchat-i18n/i18n/en.i18n.json | 4 + .../server/models/ExportOperations.js | 3 +- .../client/accountPreferences.html | 19 +- .../client/accountPreferences.js | 7 +- .../rocketchat-user-data-download/package.js | 16 ++ .../server/startup/settings.js | 38 ++++ server/methods/processDataDownloads.js | 178 ++++++++++++++---- server/methods/requestDataDownload.js | 15 +- 14 files changed, 280 insertions(+), 57 deletions(-) create mode 100644 packages/rocketchat-user-data-download/package.js create mode 100644 packages/rocketchat-user-data-download/server/startup/settings.js diff --git a/.meteor/packages b/.meteor/packages index f6da7829fd40d..7c0bdaa397344 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -146,6 +146,7 @@ rocketchat:ui-master rocketchat:ui-message rocketchat:ui-sidenav rocketchat:ui-vrecord +rocketchat:user-data-download rocketchat:version rocketchat:videobridge rocketchat:webrtc diff --git a/.meteor/versions b/.meteor/versions index fe345b2e9cbd0..c42509c71d4a2 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -235,6 +235,7 @@ rocketchat:ui-master@0.1.0 rocketchat:ui-message@0.1.0 rocketchat:ui-sidenav@0.1.0 rocketchat:ui-vrecord@0.0.1 +rocketchat:user-data-download@1.0.0 rocketchat:version@1.0.0 rocketchat:version-check@0.0.1 rocketchat:videobridge@0.2.0 diff --git a/package.json b/package.json index d8d6bfa0958da..4562f3dc34aa7 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "@google-cloud/storage": "^1.6.0", "@google-cloud/vision": "^0.15.2", "adm-zip": "^0.4.7", + "archiver": "^2.1.1", "atlassian-crowd": "^0.5.0", "autolinker": "^1.6.2", "aws-sdk": "^2.195.0", diff --git a/packages/rocketchat-file-upload/server/config/FileSystem.js b/packages/rocketchat-file-upload/server/config/FileSystem.js index 096df87e4dee0..804d2bc6d4af3 100644 --- a/packages/rocketchat-file-upload/server/config/FileSystem.js +++ b/packages/rocketchat-file-upload/server/config/FileSystem.js @@ -28,6 +28,22 @@ const FileSystemUploads = new FileUploadClass({ res.end(); return; } + }, + + copy(file, out) { + const filePath = this.store.getFilePath(file._id, file); + try { + const stat = Meteor.wrapAsync(fs.stat)(filePath); + + if (stat && stat.isFile()) { + file = FileUpload.addExtensionTo(file); + + this.store.getReadStream(file._id, file).pipe(out); + } + } catch (e) { + out.end(); + return; + } } }); diff --git a/packages/rocketchat-file-upload/server/config/GridFS.js b/packages/rocketchat-file-upload/server/config/GridFS.js index 54bb40712cacd..cce94c6d1c042 100644 --- a/packages/rocketchat-file-upload/server/config/GridFS.js +++ b/packages/rocketchat-file-upload/server/config/GridFS.js @@ -62,7 +62,6 @@ const getByteRange = function(header) { return null; }; - // code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310 const readFromGridFS = function(storeName, fileId, file, req, res) { const store = UploadFS.getStore(storeName); @@ -123,6 +122,18 @@ const readFromGridFS = function(storeName, fileId, file, req, res) { } }; +const copyFromGridFS = function(storeName, fileId, file, out) { + const store = UploadFS.getStore(storeName); + const rs = store.getReadStream(fileId, file); + + [rs, out].forEach(stream => stream.on('error', function(err) { + store.onReadError.call(store, err, fileId, file); + out.end(); + })); + + rs.pipe(out); +}; + FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', { collectionName: 'rocketchat_uploads' }); @@ -147,6 +158,10 @@ new FileUploadClass({ res.setHeader('Content-Length', file.size); return readFromGridFS(file.store, file._id, file, req, res); + }, + + copy(file, out) { + copyFromGridFS(file.store, file._id, file, out); } }); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index a056e991fba5e..201a99e88086a 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -1,6 +1,7 @@ /* globals UploadFS */ import fs from 'fs'; +import path from 'path'; import stream from 'stream'; import mime from 'mime-type/with-db'; import Future from 'fibers/future'; @@ -229,16 +230,32 @@ Object.assign(FileUpload, { } res.writeHead(404); res.end(); + }, + + copy(file, targetFolder) { + const store = this.getStoreByName(file.store); + const targetFile = path.join(targetFolder, `${ file._id }-${ file.name }`); + + const out = fs.createWriteStream(targetFile); + + file = FileUpload.addExtensionTo(file); + + if (store.copy) { + store.copy(file, out); + return true; + } + + return false; } }); - export class FileUploadClass { - constructor({ name, model, store, get, insert, getStore }) { + constructor({ name, model, store, get, insert, getStore, copy }) { this.name = name; this.model = model || this.getModelFromName(); this._store = store || UploadFS.getStore(name); this.get = get; + this.copy = copy; if (insert) { this.insert = insert; diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index 743ec575c97e0..370e5a38e0a0d 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -2068,6 +2068,10 @@ "User_uploaded_file": "Uploaded a file", "User_uploaded_image": "Uploaded an image", "User_Presence": "User Presence", + "UserDataDownload" : "User Data Download", + "UserData_EnableDownload" : "Enable User Data Download", + "UserData_FileSystemPath" : "System Path (Exported Files)", + "UserData_FileSystemZipPath" : "System Path (Compressed File)", "Username": "Username", "Username_and_message_must_not_be_empty": "Username and message must not be empty.", "Username_cant_be_empty": "The username cannot be empty", diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js index e14ed03ecd8fd..b5d5cb6fd948d 100644 --- a/packages/rocketchat-lib/server/models/ExportOperations.js +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -34,7 +34,8 @@ RocketChat.models.ExportOperations = new class ModelExportOperations extends Roc const update = { $set: { roomList: data.roomList, - status: data.status + status: data.status, + fileList: data.fileList } }; diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html index 274c83a753d17..d51a3e43bb790 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.html +++ b/packages/rocketchat-ui-account/client/accountPreferences.html @@ -291,16 +291,19 @@

{{_ "Sound"}}

-
-

{{_ "My Data"}}

-
-
- - + + {{#if userDataDownloadEnabled}} +
+

{{_ "My Data"}}

+
+
+ + +
+
-
-
+ {{/if}}
diff --git a/packages/rocketchat-ui-account/client/accountPreferences.js b/packages/rocketchat-ui-account/client/accountPreferences.js index 6b1ec68c47ef5..d1d5fe742a21c 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.js +++ b/packages/rocketchat-ui-account/client/accountPreferences.js @@ -87,6 +87,9 @@ Template.accountPreferences.helpers({ showRoles() { return RocketChat.settings.get('UI_DisplayRoles'); }, + userDataDownloadEnabled() { + return RocketChat.settings.get('UserData_EnableDownload') !== false; + }, notificationsSoundVolume() { return RocketChat.getUserPreference(Meteor.user(), 'notificationsSoundVolume'); } @@ -199,7 +202,7 @@ Template.accountPreferences.onCreated(function() { this.downloadMyData = function() { Meteor.call('requestDataDownload', {}, function(error, results) { if (results) { - + return true; } if (error) { @@ -211,7 +214,7 @@ Template.accountPreferences.onCreated(function() { this.processDataDownload = function() { Meteor.call('processDataDownloads', {}, function(error, results) { if (results) { - + return true; } if (error) { diff --git a/packages/rocketchat-user-data-download/package.js b/packages/rocketchat-user-data-download/package.js new file mode 100644 index 0000000000000..7d74c84afdd53 --- /dev/null +++ b/packages/rocketchat-user-data-download/package.js @@ -0,0 +1,16 @@ +Package.describe({ + name: 'rocketchat:user-data-download', + version: '1.0.0', + summary: 'Adds setting to allow the user to download all their data stored in the servers.', + git: '' +}); + +Package.onUse(function(api) { + api.use([ + 'ecmascript', + 'rocketchat:file', + 'rocketchat:lib' + ]); + + api.addFiles('server/startup/settings.js', 'server'); +}); diff --git a/packages/rocketchat-user-data-download/server/startup/settings.js b/packages/rocketchat-user-data-download/server/startup/settings.js new file mode 100644 index 0000000000000..d1b9f1b600ac7 --- /dev/null +++ b/packages/rocketchat-user-data-download/server/startup/settings.js @@ -0,0 +1,38 @@ +RocketChat.settings.addGroup('UserDataDownload', function() { + + this.add('UserData_EnableDownload', true, { + type: 'boolean', + public: true, + i18nLabel: 'UserData_EnableDownload' + }); + + this.add('UserData_Storage_Type', 'FileSystem', { + type: 'select', + public: true, + values: [{ + key: 'FileSystem', + i18nLabel: 'FileSystem' + }], + i18nLabel: 'FileUpload_Storage_Type' + }); + + this.add('UserData_FileSystemPath', '', { + type: 'string', + public: true, + enableQuery: { + _id: 'UserData_Storage_Type', + value: 'FileSystem' + }, + i18nLabel: 'UserData_FileSystemPath' + }); + + this.add('UserData_FileSystemZipPath', '', { + type: 'string', + public: true, + enableQuery: { + _id: 'UserData_Storage_Type', + value: 'FileSystem' + }, + i18nLabel: 'UserData_FileSystemZipPath' + }); +}); diff --git a/server/methods/processDataDownloads.js b/server/methods/processDataDownloads.js index ed0be6c739d6e..ae6b255e217ac 100644 --- a/server/methods/processDataDownloads.js +++ b/server/methods/processDataDownloads.js @@ -1,5 +1,13 @@ import fs from 'fs'; import path from 'path'; +import archiver from 'archiver'; + +let zipFolder = '~/zipFiles'; +if (RocketChat.settings.get('UserData_FileSystemZipPath') != null) { + if (RocketChat.settings.get('UserData_FileSystemZipPath').trim() !== '') { + zipFolder = RocketChat.settings.get('UserData_FileSystemZipPath'); + } +} const startFile = function(fileName, content) { fs.writeFileSync(fileName, content); @@ -38,8 +46,100 @@ const loadUserSubscriptions = function(exportOperation) { exportOperation.status = 'exporting'; }; +const getAttachmentData = function(attachment) { + const attachmentData = { + type : attachment.type, + title: attachment.title, + title_link: attachment.title_link, + image_url: attachment.image_url, + audio_url: attachment.audio_url, + video_url: attachment.video_url, + message_link: attachment.message_link, + image_type: attachment.image_type, + image_size: attachment.image_size, + video_size: attachment.video_size, + video_type: attachment.video_type, + audio_size: attachment.audio_size, + audio_type: attachment.audio_type + }; + + return attachmentData; +}; + +const addToFileList = function(exportOperation, attachment) { + const url = attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url || attachment.message_link; + if (!url) { + return; + } + + const attachmentData = { + url, + copied: false + }; + + exportOperation.fileList.push(attachmentData); +}; + +const getMessageData = function(msg, exportOperation) { + const attachments = []; + + if (msg.attachments) { + msg.attachments.forEach((attachment) => { + const attachmentData = getAttachmentData(attachment); + + attachments.push(attachmentData); + addToFileList(exportOperation, attachment); + }); + } + + const messageObject = { + msg: msg.msg, + username: msg.u.username, + ts: msg.ts + }; + + if (attachments && attachments.length > 0) { + messageObject.attachments = attachments; + } + if (msg.t) { + messageObject.type = msg.t; + } + if (msg.u.name) { + messageObject.name = msg.u.name; + } + + return messageObject; +}; + +const copyFile = function(exportOperation, attachmentData) { + if (attachmentData.copied) { + return; + } + + //If it is an URL, just mark as downloaded + const urlMatch = /\:\/\//.exec(attachmentData.url); + if (urlMatch && urlMatch.length > 0) { + attachmentData.copied = true; + return; + } + + const match = /^\/([^\/]+)\/([^\/]+)\/(.*)/.exec(attachmentData.url); + + if (match && match[2]) { + const file = RocketChat.models.Uploads.findOneById(match[2]); + + if (file) { + if (FileUpload.copy(file, exportOperation.assetsPath)) { + attachmentData.copied = true; + } + } + } +}; + const continueExportingRoom = function(exportOperation, exportOpRoomData) { createDir(exportOperation.exportPath); + createDir(exportOperation.assetsPath); + const filePath = path.join(exportOperation.exportPath, exportOpRoomData.targetFile); if (exportOpRoomData.status === 'pending') { @@ -47,41 +147,14 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { startFile(filePath, ''); } - const limit = 50; + const limit = 100; const skip = exportOpRoomData.exportedCount; const cursor = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }); const count = cursor.count(); cursor.forEach((msg) => { - const attachments = []; - - if (msg.attachments) { - msg.attachments.forEach((attachment) => { - attachments.push({ - type : attachment.type, - title : attachment.title, - url : attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url - }); - }); - } - - const messageObject = { - msg: msg.msg, - username: msg.u.username, - ts: msg.ts - }; - - if (attachments && attachments.length > 0) { - messageObject.attachments = attachments; - } - if (msg.t) { - messageObject.type = msg.t; - } - if (msg.u.name) { - messageObject.name = msg.u.name; - } - + const messageObject = getMessageData(msg, exportOperation); const messageString = JSON.stringify(messageObject); writeToFile(filePath, `${ messageString }\n`); @@ -101,7 +174,29 @@ const isOperationFinished = function(exportOperation) { return exportOpRoomData.status !== 'completed'; }); - return !incomplete; + const anyDownloadPending = exportOperation.fileList.some((fileData) => { + return !fileData.copied; + }); + + return !incomplete && !anyDownloadPending; +}; + +const makeZipFile = function(exportOperation) { + const targetFile = path.join(zipFolder, `${ exportOperation.userId }.zip`); + const output = fs.createWriteStream(targetFile); + + const archive = archiver('zip'); + + output.on('close', () => { + }); + + archive.on('error', (err) => { + throw err; + }); + + archive.pipe(output); + archive.directory(exportOperation.exportPath, false); + archive.finalize(); }; const continueExportOperation = function(exportOperation) { @@ -113,21 +208,22 @@ const continueExportOperation = function(exportOperation) { loadUserSubscriptions(exportOperation); } - let nextRoom = false; - exportOperation.roomList.some((exportOpRoomData) => { - if (exportOpRoomData.status === 'completed') { - return false; - } - - nextRoom = exportOpRoomData; - return true; - }); + try { + //Run every room on every request, to avoid missing new messages on the rooms that finished first. + exportOperation.roomList.forEach((exportOpRoomData) => { + continueExportingRoom(exportOperation, exportOpRoomData); + }); - if (nextRoom) { - continueExportingRoom(exportOperation, nextRoom); + exportOperation.fileList.forEach((attachmentData) => { + copyFile(exportOperation, attachmentData); + }); + } catch (e) { + console.error(e); + return false; } if (isOperationFinished(exportOperation)) { + makeZipFile(exportOperation); exportOperation.status = 'completed'; return true; } diff --git a/server/methods/requestDataDownload.js b/server/methods/requestDataDownload.js index 6414630932f42..be70a68e78bac 100644 --- a/server/methods/requestDataDownload.js +++ b/server/methods/requestDataDownload.js @@ -1,15 +1,26 @@ import path from 'path'; +let tempFolder = '~/userData'; +if (RocketChat.settings.get('UserData_FileSystemPath') != null) { + if (RocketChat.settings.get('UserData_FileSystemPath').trim() !== '') { + tempFolder = RocketChat.settings.get('UserData_FileSystemPath'); + } +} + Meteor.methods({ requestDataDownload() { const currentUserData = Meteor.user(); - const folderName = path.join('/tmp/', currentUserData._id); + const folderName = path.join(tempFolder, currentUserData._id); + const assetsFolder = path.join(folderName, 'assets'); + const exportOperation = { userId : currentUserData._id, roomList: null, status: 'pending', - exportPath: folderName + exportPath: folderName, + assetsPath: assetsFolder, + fileList: [] }; RocketChat.models.ExportOperations.create(exportOperation); From 41fa0dbdb2dc4b1e6b4949485844cc57c84c5dad Mon Sep 17 00:00:00 2001 From: Hudell Date: Mon, 26 Feb 2018 14:11:59 -0300 Subject: [PATCH 07/14] Use syncedcron to process data downloads --- packages/rocketchat-i18n/i18n/en.i18n.json | 7 +++ .../server/models/ExportOperations.js | 20 +++++++-- .../client/accountPreferences.html | 1 - .../client/accountPreferences.js | 45 +++++++++++++------ .../rocketchat-user-data-download/package.js | 1 + .../server/cronProcessDownloads.js | 36 +++++++++++---- .../server/startup/settings.js | 14 ++++++ server/methods/requestDataDownload.js | 22 ++++++++- 8 files changed, 120 insertions(+), 26 deletions(-) rename server/methods/processDataDownloads.js => packages/rocketchat-user-data-download/server/cronProcessDownloads.js (86%) diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index 370e5a38e0a0d..bac04ee67c252 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -556,6 +556,7 @@ "Domain_removed": "Domain Removed", "Domains": "Domains", "Domains_allowed_to_embed_the_livechat_widget": "Comma-separated list of domains allowed to embed the livechat widget. Leave blank to allow all domains.", + "Download_My_Data" : "Download My Data", "Download_Snippet": "Download", "Drop_to_upload_file": "Drop to upload file", "Dry_run": "Dry run", @@ -2072,6 +2073,12 @@ "UserData_EnableDownload" : "Enable User Data Download", "UserData_FileSystemPath" : "System Path (Exported Files)", "UserData_FileSystemZipPath" : "System Path (Compressed File)", + "UserData_ProcessingFrequency" : "Processing Frequency (Minutes)", + "UserData_MessageLimitPerRequest" : "Message Limit per Request", + "UserDataDownload_Requested" : "Download File Requested", + "UserDataDownload_Requested_Text" : "Your data file will be generated. A link to download it will be sent to your email address when ready.", + "UserDataDownload_RequestExisted_Text" : "Your data file is already being generated. A link to download it will be sent to your email address when ready.", + "UserDataDownload_CompletedRequestExisted_Text" : "Your data file was already generated. Check your email account for the download link.", "Username": "Username", "Username_and_message_must_not_be_empty": "Username and message must not be empty.", "Username_cant_be_empty": "The username cannot be empty", diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js index b5d5cb6fd948d..9be75bb0b4174 100644 --- a/packages/rocketchat-lib/server/models/ExportOperations.js +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -15,10 +15,24 @@ RocketChat.models.ExportOperations = new class ModelExportOperations extends Roc return this.find(query); } - findPendingByUser(userId) { - const query = {userId}; + findLastOperationByUser(userId, options = {}) { + const query = { + userId + }; - return this.find(query); + options.sort = {'createdAt' : -1}; + return this.findOne(query, options); + } + + findPendingByUser(userId, options) { + const query = { + userId, + status: { + $in: ['pending', 'exporting'] + } + }; + + return this.find(query, options); } findAllPending(options) { diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html index d51a3e43bb790..15d6c477592f8 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.html +++ b/packages/rocketchat-ui-account/client/accountPreferences.html @@ -298,7 +298,6 @@

{{_ "My Data"}}

-
diff --git a/packages/rocketchat-ui-account/client/accountPreferences.js b/packages/rocketchat-ui-account/client/accountPreferences.js index d1d5fe742a21c..f856c469be258 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.js +++ b/packages/rocketchat-ui-account/client/accountPreferences.js @@ -202,18 +202,39 @@ Template.accountPreferences.onCreated(function() { this.downloadMyData = function() { Meteor.call('requestDataDownload', {}, function(error, results) { if (results) { - return true; - } + if (results.requested) { + modal.open({ + title: t('UserDataDownload_Requested'), + text: t('UserDataDownload_Requested_Text'), + type: 'success' + }); - if (error) { - return handleError(error); - } - }); - }; + return true; + } - this.processDataDownload = function() { - Meteor.call('processDataDownloads', {}, function(error, results) { - if (results) { + if (results.exportOperation) { + if (results.exportOperation.status === 'completed') { + modal.open({ + title: t('UserDataDownload_Requested'), + text: t('UserDataDownload_CompletedRequestExisted_Text'), + type: 'success' + }); + + return true; + } + + modal.open({ + title: t('UserDataDownload_Requested'), + text: t('UserDataDownload_RequestExisted_Text'), + type: 'success' + }); + return true; + } + + modal.open({ + title: t('UserDataDownload_Requested'), + type: 'success' + }); return true; } @@ -242,11 +263,9 @@ Template.accountPreferences.events({ KonchatNotification.getDesktopPermission(); }, 'click .download-my-data'(e, t) { + e.preventDefault(); t.downloadMyData(); }, - 'click .process-data-download'(e, t) { - t.processDataDownload(); - }, 'click .test-notifications'(e) { e.preventDefault(); KonchatNotification.notify({ diff --git a/packages/rocketchat-user-data-download/package.js b/packages/rocketchat-user-data-download/package.js index 7d74c84afdd53..e5efbf831117e 100644 --- a/packages/rocketchat-user-data-download/package.js +++ b/packages/rocketchat-user-data-download/package.js @@ -13,4 +13,5 @@ Package.onUse(function(api) { ]); api.addFiles('server/startup/settings.js', 'server'); + api.addFiles('server/cronProcessDownloads.js', 'server'); }); diff --git a/server/methods/processDataDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js similarity index 86% rename from server/methods/processDataDownloads.js rename to packages/rocketchat-user-data-download/server/cronProcessDownloads.js index ae6b255e217ac..4954718fe23b3 100644 --- a/server/methods/processDataDownloads.js +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -1,3 +1,4 @@ +/* globals SyncedCron */ import fs from 'fs'; import path from 'path'; import archiver from 'archiver'; @@ -9,6 +10,11 @@ if (RocketChat.settings.get('UserData_FileSystemZipPath') != null) { } } +let processingFrequency = 15; +if (RocketChat.settings.get('UserData_ProcessingFrequency') > 0) { + processingFrequency = RocketChat.settings.get('UserData_ProcessingFrequency'); +} + const startFile = function(fileName, content) { fs.writeFileSync(fileName, content); }; @@ -147,7 +153,11 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { startFile(filePath, ''); } - const limit = 100; + let limit = 100; + if (RocketChat.settings.get('UserData_MessageLimitPerRequest') > 0) { + limit = RocketChat.settings.get('UserData_MessageLimitPerRequest'); + } + const skip = exportOpRoomData.exportedCount; const cursor = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }); @@ -231,12 +241,22 @@ const continueExportOperation = function(exportOperation) { return false; }; -Meteor.methods({ - processDataDownloads() { - const cursor = RocketChat.models.ExportOperations.findAllPending({limit: 1}); - cursor.forEach((exportOperation) => { - continueExportOperation(exportOperation); - RocketChat.models.ExportOperations.updateOperation(exportOperation); +function processDataDownloads() { + const cursor = RocketChat.models.ExportOperations.findAllPending({limit: 1}); + cursor.forEach((exportOperation) => { + continueExportOperation(exportOperation); + RocketChat.models.ExportOperations.updateOperation(exportOperation); + }); +} + +Meteor.startup(function() { + Meteor.defer(function() { + processDataDownloads(); + + SyncedCron.add({ + name: 'Generate download files for user data', + schedule: (parser) => parser.cron(`*/${ processingFrequency } * * * *`), + job: processDataDownloads }); - } + }); }); diff --git a/packages/rocketchat-user-data-download/server/startup/settings.js b/packages/rocketchat-user-data-download/server/startup/settings.js index d1b9f1b600ac7..979338215d6cf 100644 --- a/packages/rocketchat-user-data-download/server/startup/settings.js +++ b/packages/rocketchat-user-data-download/server/startup/settings.js @@ -35,4 +35,18 @@ RocketChat.settings.addGroup('UserDataDownload', function() { }, i18nLabel: 'UserData_FileSystemZipPath' }); + + this.add('UserData_ProcessingFrequency', 15, { + type: 'int', + public: true, + i18nLabel: 'UserData_ProcessingFrequency' + }); + + this.add('UserData_MessageLimitPerRequest', 100, { + type: 'int', + public: true, + i18nLabel: 'UserData_MessageLimitPerRequest' + }); + + }); diff --git a/server/methods/requestDataDownload.js b/server/methods/requestDataDownload.js index be70a68e78bac..a09a3c0f154f6 100644 --- a/server/methods/requestDataDownload.js +++ b/server/methods/requestDataDownload.js @@ -10,8 +10,23 @@ if (RocketChat.settings.get('UserData_FileSystemPath') != null) { Meteor.methods({ requestDataDownload() { const currentUserData = Meteor.user(); + const userId = currentUserData._id; - const folderName = path.join(tempFolder, currentUserData._id); + const lastOperation = RocketChat.models.ExportOperations.findLastOperationByUser(userId); + + if (lastOperation) { + const yesterday = new Date(); + yesterday.setUTCDate(yesterday.getUTCDate() - 1); + + if (lastOperation.createdAt > yesterday) { + return { + requested: false, + exportOperation: lastOperation + }; + } + } + + const folderName = path.join(tempFolder, userId); const assetsFolder = path.join(folderName, 'assets'); const exportOperation = { @@ -24,5 +39,10 @@ Meteor.methods({ }; RocketChat.models.ExportOperations.create(exportOperation); + + return { + requested: true, + exportOperation + }; } }); From e8e006eea75cf8814d6f3de0e227d9c0883a5ff2 Mon Sep 17 00:00:00 2001 From: Hudell Date: Mon, 26 Feb 2018 15:52:24 -0300 Subject: [PATCH 08/14] Added download URL --- .../server/models/ExportOperations.js | 3 +- .../rocketchat-lib/server/models/Users.js | 5 ++ .../rocketchat-user-data-download/package.js | 4 +- .../server/cronProcessDownloads.js | 2 + .../server/lib/requests.js | 62 +++++++++++++++++++ server/methods/requestDataDownload.js | 3 +- 6 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 packages/rocketchat-user-data-download/server/lib/requests.js diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js index 9be75bb0b4174..eca30caddbba4 100644 --- a/packages/rocketchat-lib/server/models/ExportOperations.js +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -49,7 +49,8 @@ RocketChat.models.ExportOperations = new class ModelExportOperations extends Roc $set: { roomList: data.roomList, status: data.status, - fileList: data.fileList + fileList: data.fileList, + generatedFile: data.generatedFile } }; diff --git a/packages/rocketchat-lib/server/models/Users.js b/packages/rocketchat-lib/server/models/Users.js index 41ed1fe2926a6..69cd05df75651 100644 --- a/packages/rocketchat-lib/server/models/Users.js +++ b/packages/rocketchat-lib/server/models/Users.js @@ -51,6 +51,11 @@ class ModelUsers extends RocketChat.models._Base { return this.findOne(query, options); } + findOneById(userId) { + const query = {_id: userId}; + + return this.findOne(query); + } // FIND findById(userId) { diff --git a/packages/rocketchat-user-data-download/package.js b/packages/rocketchat-user-data-download/package.js index e5efbf831117e..4e085ab0037bd 100644 --- a/packages/rocketchat-user-data-download/package.js +++ b/packages/rocketchat-user-data-download/package.js @@ -9,9 +9,11 @@ Package.onUse(function(api) { api.use([ 'ecmascript', 'rocketchat:file', - 'rocketchat:lib' + 'rocketchat:lib', + 'webapp' ]); api.addFiles('server/startup/settings.js', 'server'); + api.addFiles('server/lib/requests.js', 'server'); api.addFiles('server/cronProcessDownloads.js', 'server'); }); diff --git a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js index 4954718fe23b3..14883a1dedf88 100644 --- a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -195,6 +195,8 @@ const makeZipFile = function(exportOperation) { const targetFile = path.join(zipFolder, `${ exportOperation.userId }.zip`); const output = fs.createWriteStream(targetFile); + exportOperation.generatedFile = targetFile; + const archive = archiver('zip'); output.on('close', () => { diff --git a/packages/rocketchat-user-data-download/server/lib/requests.js b/packages/rocketchat-user-data-download/server/lib/requests.js new file mode 100644 index 0000000000000..2a631ceb37b6a --- /dev/null +++ b/packages/rocketchat-user-data-download/server/lib/requests.js @@ -0,0 +1,62 @@ +/* globals FileUpload, WebApp */ + +import fs from 'fs'; + +WebApp.connectHandlers.use(`${ __meteor_runtime_config__.ROOT_URL_PATH_PREFIX }/user-data-download/`, function(req, res) { + const match = /^\/(.*)\/?/.exec(req.url); + if (match[1]) { + const userId = match[1]; + const user = RocketChat.models.Users.findOneById(userId); + const userName = user ? user.name : userId; + + const operation = RocketChat.models.ExportOperations.findLastOperationByUser(userId); + + if (operation && operation.status === 'completed') { + if (operation.generatedFile !== null) { + + if (!Meteor.settings.public.sandstorm && !FileUpload.requestCanAccessFiles(req)) { + res.writeHead(403); + return res.end(); + } + + const filePath = operation.generatedFile; + + if (!filePath) { + res.writeHead(404); + return res.end(); + } + + try { + const stat = Meteor.wrapAsync(fs.stat)(filePath); + + if (stat && stat.isFile()) { + const utcDate = stat.mtime.toISOString().split('T')[0]; + const newFileName = encodeURIComponent(`${ utcDate }-${ userName }`); + + res.setHeader('Content-Security-Policy', 'default-src \'none\''); + res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ newFileName }.zip`); + res.setHeader('Last-Modified', stat.mtime.toUTCString()); + + res.setHeader('Content-Type', 'application/zip'); + res.setHeader('Content-Length', stat.size); + + const stream = fs.createReadStream(filePath); + stream.on('close', () => { + res.end(); + }); + stream.pipe(res); + } + + return; + } catch (e) { + res.writeHead(404); + res.end(); + return; + } + } + } + } + + res.writeHead(404); + res.end(); +}); diff --git a/server/methods/requestDataDownload.js b/server/methods/requestDataDownload.js index a09a3c0f154f6..752450b734238 100644 --- a/server/methods/requestDataDownload.js +++ b/server/methods/requestDataDownload.js @@ -35,7 +35,8 @@ Meteor.methods({ status: 'pending', exportPath: folderName, assetsPath: assetsFolder, - fileList: [] + fileList: [], + generatedFile: null }; RocketChat.models.ExportOperations.create(exportOperation); From a726286ef3d5ae7e9ad3fc1af7eb78bc96b3c17b Mon Sep 17 00:00:00 2001 From: Hudell Date: Mon, 26 Feb 2018 17:48:06 -0300 Subject: [PATCH 09/14] Sending emails when the download file is ready --- packages/rocketchat-i18n/i18n/en.i18n.json | 2 + .../server/cronProcessDownloads.js | 48 ++++++++++++++++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index bac04ee67c252..58c9693598350 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -2075,6 +2075,8 @@ "UserData_FileSystemZipPath" : "System Path (Compressed File)", "UserData_ProcessingFrequency" : "Processing Frequency (Minutes)", "UserData_MessageLimitPerRequest" : "Message Limit per Request", + "UserDataDownload_EmailSubject" : "Your Data File is Ready to Download", + "UserDataDownload_EmailBody" : "Your data file is now ready to download. Click here to download it.", "UserDataDownload_Requested" : "Download File Requested", "UserDataDownload_Requested_Text" : "Your data file will be generated. A link to download it will be sent to your email address when ready.", "UserDataDownload_RequestExisted_Text" : "Your data file is already being generated. A link to download it will be sent to your email address when ready.", diff --git a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js index 14883a1dedf88..f5b8198c77cdb 100644 --- a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -1,4 +1,5 @@ /* globals SyncedCron */ + import fs from 'fs'; import path from 'path'; import archiver from 'archiver'; @@ -191,6 +192,33 @@ const isOperationFinished = function(exportOperation) { return !incomplete && !anyDownloadPending; }; +const sendEmail = function(userId) { + const userData = RocketChat.models.Users.findOneById(userId); + + if (userData && userData.emails && userData.emails[0] && userData.emails[0].address) { + const emailAddress = `${ userData.name } <${ userData.emails[0].address }>`; + const fromAddress = RocketChat.settings.get('From_Email'); + const subject = TAPi18n.__('UserDataDownload_EmailSubject'); + const download_link = `${ __meteor_runtime_config__.ROOT_URL }${ __meteor_runtime_config__.ROOT_URL_PATH_PREFIX }/user-data-download/${ userId }`; + const body = TAPi18n.__('UserDataDownload_EmailBody', { download_link }); + + const rfcMailPatternWithName = /^(?:.*<)?([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)(?:>?)$/; + + if (rfcMailPatternWithName.test(emailAddress)) { + Meteor.defer(function() { + return Email.send({ + to: emailAddress, + from: fromAddress, + subject, + html: body + }); + }); + + return console.log(`Sending email to ${ emailAddress }`); + } + } +}; + const makeZipFile = function(exportOperation) { const targetFile = path.join(zipFolder, `${ exportOperation.userId }.zip`); const output = fs.createWriteStream(targetFile); @@ -229,25 +257,33 @@ const continueExportOperation = function(exportOperation) { exportOperation.fileList.forEach((attachmentData) => { copyFile(exportOperation, attachmentData); }); + + if (isOperationFinished(exportOperation)) { + makeZipFile(exportOperation); + exportOperation.status = 'completed'; + return true; + } } catch (e) { console.error(e); return false; } - if (isOperationFinished(exportOperation)) { - makeZipFile(exportOperation); - exportOperation.status = 'completed'; - return true; - } - return false; }; function processDataDownloads() { const cursor = RocketChat.models.ExportOperations.findAllPending({limit: 1}); cursor.forEach((exportOperation) => { + if (exportOperation.status === 'completed') { + return; + } + continueExportOperation(exportOperation); RocketChat.models.ExportOperations.updateOperation(exportOperation); + + if (exportOperation.status === 'completed') { + sendEmail(exportOperation.userId); + } }); } From 649f1acad4940e81530a87e3d632c4c2f49a9ab6 Mon Sep 17 00:00:00 2001 From: Hudell Date: Tue, 27 Feb 2018 18:28:12 -0300 Subject: [PATCH 10/14] Allow usage of GridFS as storage for the finished file. --- .../client/lib/fileUploadHandler.js | 4 + .../lib/FileUploadBase.js | 16 ++- .../server/config/FileSystem.js | 26 ++++ .../server/config/GridFS.js | 23 ++++ .../server/config/_configUploadStorage.js | 1 + .../server/lib/FileUpload.js | 19 +++ .../client/models/UserDataFiles.js | 6 + packages/rocketchat-lib/package.js | 3 +- .../server/models/ExportOperations.js | 4 +- .../server/models/UserDataFiles.js | 40 ++++++ .../rocketchat-user-data-download/package.js | 1 - .../server/cronProcessDownloads.js | 129 +++++++++++++----- .../server/lib/requests.js | 62 --------- .../server/startup/settings.js | 18 --- 14 files changed, 233 insertions(+), 119 deletions(-) create mode 100644 packages/rocketchat-lib/client/models/UserDataFiles.js create mode 100644 packages/rocketchat-lib/server/models/UserDataFiles.js delete mode 100644 packages/rocketchat-user-data-download/server/lib/requests.js diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index 73474ab5181f3..cbcf7c6df4f8f 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -17,6 +17,10 @@ new UploadFS.Store({ }) }); +// new UploadFS.Store({ +// collection: RocketChat.models.UserDataFiles.model, +// name: 'UserDataFiles' +// }); fileUploadHandler = (directive, meta, file) => { const store = UploadFS.getStore(directive); diff --git a/packages/rocketchat-file-upload/lib/FileUploadBase.js b/packages/rocketchat-file-upload/lib/FileUploadBase.js index 3e2b166547f59..97fd9ced52731 100644 --- a/packages/rocketchat-file-upload/lib/FileUploadBase.js +++ b/packages/rocketchat-file-upload/lib/FileUploadBase.js @@ -4,7 +4,21 @@ import _ from 'underscore'; UploadFS.config.defaultStorePermissions = new UploadFS.StorePermissions({ insert(userId, doc) { - return userId || (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0); // allow inserts from slackbridge (message_id = slack-timestamp-milli) + if (userId) { + return true; + } + + // allow inserts from slackbridge (message_id = slack-timestamp-milli) + if (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0) { + return true; + } + + // allow inserts to the UserDataFiles store + if (doc && doc.store && doc.store.split(':').pop() === 'UserDataFiles') { + return true; + } + + return false; }, update(userId, doc) { return RocketChat.authz.hasPermission(Meteor.userId(), 'delete-message', doc.rid) || (RocketChat.settings.get('Message_AllowDeleting') && userId === doc.userId); diff --git a/packages/rocketchat-file-upload/server/config/FileSystem.js b/packages/rocketchat-file-upload/server/config/FileSystem.js index 804d2bc6d4af3..fab473b5ab6cf 100644 --- a/packages/rocketchat-file-upload/server/config/FileSystem.js +++ b/packages/rocketchat-file-upload/server/config/FileSystem.js @@ -70,6 +70,31 @@ const FileSystemAvatars = new FileUploadClass({ } }); +const FileSystemUserDataFiles = new FileUploadClass({ + name: 'FileSystem:UserDataFiles', + + get(file, req, res) { + const filePath = this.store.getFilePath(file._id, file); + + try { + const stat = Meteor.wrapAsync(fs.stat)(filePath); + + if (stat && stat.isFile()) { + file = FileUpload.addExtensionTo(file); + res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`); + res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); + res.setHeader('Content-Type', file.type); + res.setHeader('Content-Length', file.size); + + this.store.getReadStream(file._id, file).pipe(res); + } + } catch (e) { + res.writeHead(404); + res.end(); + return; + } + } +}); const createFileSystemStore = _.debounce(function() { const options = { @@ -78,6 +103,7 @@ const createFileSystemStore = _.debounce(function() { FileSystemUploads.store = FileUpload.configureUploadsStore('Local', FileSystemUploads.name, options); FileSystemAvatars.store = FileUpload.configureUploadsStore('Local', FileSystemAvatars.name, options); + FileSystemUserDataFiles.store = FileUpload.configureUploadsStore('Local', FileSystemUserDataFiles.name, options); // DEPRECATED backwards compatibililty (remove) UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name]; diff --git a/packages/rocketchat-file-upload/server/config/GridFS.js b/packages/rocketchat-file-upload/server/config/GridFS.js index cce94c6d1c042..0ad63b5e86264 100644 --- a/packages/rocketchat-file-upload/server/config/GridFS.js +++ b/packages/rocketchat-file-upload/server/config/GridFS.js @@ -138,6 +138,10 @@ FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', { collectionName: 'rocketchat_uploads' }); +FileUpload.configureUploadsStore('GridFS', 'GridFS:UserDataFiles', { + collectionName: 'rocketchat_userDataFiles' +}); + // DEPRECATED: backwards compatibility (remove) UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; @@ -165,6 +169,25 @@ new FileUploadClass({ } }); +new FileUploadClass({ + name: 'GridFS:UserDataFiles', + + get(file, req, res) { + file = FileUpload.addExtensionTo(file); + + res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`); + res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); + res.setHeader('Content-Type', file.type); + res.setHeader('Content-Length', file.size); + + return readFromGridFS(file.store, file._id, file, req, res); + }, + + copy(file, out) { + copyFromGridFS(file.store, file._id, file, out); + } +}); + new FileUploadClass({ name: 'GridFS:Avatars', diff --git a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js index d2ba5505f2d4d..8855eb76fca8a 100644 --- a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js +++ b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js @@ -14,6 +14,7 @@ const configStore = _.debounce(() => { console.log('Setting default file store to', store); UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`); UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`); + UploadFS.getStores().UserDataFiles = UploadFS.getStore(`${ store }:UserDataFiles`); } }, 1000); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index 201a99e88086a..ec0cb41d1ef58 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -59,6 +59,25 @@ Object.assign(FileUpload, { }; }, + defaultUserDataFiles() { + return { + collection: RocketChat.models.UserDataFiles.model, + getPath(file) { + return `${ RocketChat.settings.get('uniqueID') }/uploads/userData/${ file.userId }`; + }, + onValidate: FileUpload.uploadsOnValidate, + onRead(fileId, file, req, res) { + if (!FileUpload.requestCanAccessFiles(req)) { + res.writeHead(403); + return false; + } + + res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`); + return true; + } + }; + }, + avatarsOnValidate(file) { if (RocketChat.settings.get('Accounts_AvatarResize') !== true) { return; diff --git a/packages/rocketchat-lib/client/models/UserDataFiles.js b/packages/rocketchat-lib/client/models/UserDataFiles.js new file mode 100644 index 0000000000000..80e7e22e162ed --- /dev/null +++ b/packages/rocketchat-lib/client/models/UserDataFiles.js @@ -0,0 +1,6 @@ +RocketChat.models.UserDataFiles = new class extends RocketChat.models._Base { + constructor() { + super(); + this._initModel('userDataFiles'); + } +}; diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js index 4cbf37eadcf08..8de0fa3a73a85 100644 --- a/packages/rocketchat-lib/package.js +++ b/packages/rocketchat-lib/package.js @@ -126,7 +126,8 @@ Package.onUse(function(api) { api.addFiles('server/models/Subscriptions.js', 'server'); api.addFiles('server/models/Uploads.js', 'server'); api.addFiles('server/models/Users.js', 'server'); - api.addFiles('server/models/ExportOperations.js', 'server'); + api.addFiles('server/models/ExportOperations.js', 'server'); + api.addFiles('server/models/UserDataFiles.js', 'server'); api.addFiles('server/oauth/oauth.js', 'server'); api.addFiles('server/oauth/google.js', 'server'); diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js index eca30caddbba4..1f7e029bc1684 100644 --- a/packages/rocketchat-lib/server/models/ExportOperations.js +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -28,7 +28,7 @@ RocketChat.models.ExportOperations = new class ModelExportOperations extends Roc const query = { userId, status: { - $in: ['pending', 'exporting'] + $nin: ['completed'] } }; @@ -37,7 +37,7 @@ RocketChat.models.ExportOperations = new class ModelExportOperations extends Roc findAllPending(options) { const query = { - status: { $in: ['pending', 'exporting'] } + status: { $nin: ['completed'] } }; return this.find(query, options); diff --git a/packages/rocketchat-lib/server/models/UserDataFiles.js b/packages/rocketchat-lib/server/models/UserDataFiles.js new file mode 100644 index 0000000000000..ba70a25226c33 --- /dev/null +++ b/packages/rocketchat-lib/server/models/UserDataFiles.js @@ -0,0 +1,40 @@ +import _ from 'underscore'; + +RocketChat.models.UserDataFiles = new class ModelUserDataFiles extends RocketChat.models._Base { + constructor() { + super('userDataFiles'); + + this.tryEnsureIndex({ 'userId': 1 }); + } + + // FIND + findById(id) { + const query = {_id: id}; + return this.find(query); + } + + findLastFileByUser(userId, options = {}) { + const query = { + userId + }; + + options.sort = {'_updatedAt' : -1}; + return this.findOne(query, options); + } + + // INSERT + create(data) { + const userDataFile = { + createdAt: new Date + }; + + _.extend(userDataFile, data); + + return this.insert(userDataFile); + } + + // REMOVE + removeById(_id) { + return this.remove(_id); + } +}; diff --git a/packages/rocketchat-user-data-download/package.js b/packages/rocketchat-user-data-download/package.js index 4e085ab0037bd..d3eacedee5023 100644 --- a/packages/rocketchat-user-data-download/package.js +++ b/packages/rocketchat-user-data-download/package.js @@ -14,6 +14,5 @@ Package.onUse(function(api) { ]); api.addFiles('server/startup/settings.js', 'server'); - api.addFiles('server/lib/requests.js', 'server'); api.addFiles('server/cronProcessDownloads.js', 'server'); }); diff --git a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js index f5b8198c77cdb..2f3d9ac30af3d 100644 --- a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -185,36 +185,44 @@ const isOperationFinished = function(exportOperation) { return exportOpRoomData.status !== 'completed'; }); + return !incomplete; +}; + +const isDownloadFinished = function(exportOperation) { const anyDownloadPending = exportOperation.fileList.some((fileData) => { return !fileData.copied; }); - return !incomplete && !anyDownloadPending; + return !anyDownloadPending; }; const sendEmail = function(userId) { - const userData = RocketChat.models.Users.findOneById(userId); - - if (userData && userData.emails && userData.emails[0] && userData.emails[0].address) { - const emailAddress = `${ userData.name } <${ userData.emails[0].address }>`; - const fromAddress = RocketChat.settings.get('From_Email'); - const subject = TAPi18n.__('UserDataDownload_EmailSubject'); - const download_link = `${ __meteor_runtime_config__.ROOT_URL }${ __meteor_runtime_config__.ROOT_URL_PATH_PREFIX }/user-data-download/${ userId }`; - const body = TAPi18n.__('UserDataDownload_EmailBody', { download_link }); - - const rfcMailPatternWithName = /^(?:.*<)?([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)(?:>?)$/; - - if (rfcMailPatternWithName.test(emailAddress)) { - Meteor.defer(function() { - return Email.send({ - to: emailAddress, - from: fromAddress, - subject, - html: body + const lastFile = RocketChat.models.UserDataFiles.findLastFileByUser(userId); + if (lastFile) { + const userData = RocketChat.models.Users.findOneById(userId); + + if (userData && userData.emails && userData.emails[0] && userData.emails[0].address) { + const emailAddress = `${ userData.name } <${ userData.emails[0].address }>`; + const fromAddress = RocketChat.settings.get('From_Email'); + const subject = TAPi18n.__('UserDataDownload_EmailSubject'); + + const download_link = lastFile.url; + const body = TAPi18n.__('UserDataDownload_EmailBody', { download_link }); + + const rfcMailPatternWithName = /^(?:.*<)?([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)(?:>?)$/; + + if (rfcMailPatternWithName.test(emailAddress)) { + Meteor.defer(function() { + return Email.send({ + to: emailAddress, + from: fromAddress, + subject, + html: body + }); }); - }); - return console.log(`Sending email to ${ emailAddress }`); + return console.log(`Sending email to ${ emailAddress }`); + } } } }; @@ -239,9 +247,43 @@ const makeZipFile = function(exportOperation) { archive.finalize(); }; +const uploadZipFile = function(exportOperation, callback) { + const userDataStore = FileUpload.getStore('UserDataFiles'); + const filePath = exportOperation.generatedFile; + + const stat = Meteor.wrapAsync(fs.stat)(filePath); + const stream = fs.createReadStream(filePath); + + const contentType = 'application/zip'; + const size = stat.size; + + const userId = exportOperation.userId; + const user = RocketChat.models.Users.findOneById(userId); + const userDisplayName = user ? user.name : userId; + const utcDate = new Date().toISOString().split('T')[0]; + + const newFileName = encodeURIComponent(`${ utcDate }-${ userDisplayName }.zip`); + + const details = { + userId: userId, + type: contentType, + size, + name: newFileName + }; + + userDataStore.insert(details, stream, (err) => { + if (err) { + logError({err}); + resolve(); + } else { + callback(); + } + }); +}; + const continueExportOperation = function(exportOperation) { if (exportOperation.status === 'completed') { - return true; + return; } if (!exportOperation.roomList) { @@ -250,25 +292,44 @@ const continueExportOperation = function(exportOperation) { try { //Run every room on every request, to avoid missing new messages on the rooms that finished first. - exportOperation.roomList.forEach((exportOpRoomData) => { - continueExportingRoom(exportOperation, exportOpRoomData); - }); + if (exportOperation.status == 'exporting') { + exportOperation.roomList.forEach((exportOpRoomData) => { + continueExportingRoom(exportOperation, exportOpRoomData); + }); + + if (isOperationFinished(exportOperation)) { + exportOperation.status = 'downloading'; + return; + } + } - exportOperation.fileList.forEach((attachmentData) => { - copyFile(exportOperation, attachmentData); - }); + if (exportOperation.status == 'downloading') { + exportOperation.fileList.forEach((attachmentData) => { + copyFile(exportOperation, attachmentData); + }); + + if (isDownloadFinished(exportOperation)) { + exportOperation.status = 'compressing'; + return; + } + } - if (isOperationFinished(exportOperation)) { + if (exportOperation.status == 'compressing') { makeZipFile(exportOperation); - exportOperation.status = 'completed'; - return true; + exportOperation.status = 'uploading'; + return; + } + + if (exportOperation.status == 'uploading') { + uploadZipFile(exportOperation, () => { + exportOperation.status = 'completed'; + RocketChat.models.ExportOperations.updateOperation(exportOperation); + }); + return; } } catch (e) { console.error(e); - return false; } - - return false; }; function processDataDownloads() { diff --git a/packages/rocketchat-user-data-download/server/lib/requests.js b/packages/rocketchat-user-data-download/server/lib/requests.js deleted file mode 100644 index 2a631ceb37b6a..0000000000000 --- a/packages/rocketchat-user-data-download/server/lib/requests.js +++ /dev/null @@ -1,62 +0,0 @@ -/* globals FileUpload, WebApp */ - -import fs from 'fs'; - -WebApp.connectHandlers.use(`${ __meteor_runtime_config__.ROOT_URL_PATH_PREFIX }/user-data-download/`, function(req, res) { - const match = /^\/(.*)\/?/.exec(req.url); - if (match[1]) { - const userId = match[1]; - const user = RocketChat.models.Users.findOneById(userId); - const userName = user ? user.name : userId; - - const operation = RocketChat.models.ExportOperations.findLastOperationByUser(userId); - - if (operation && operation.status === 'completed') { - if (operation.generatedFile !== null) { - - if (!Meteor.settings.public.sandstorm && !FileUpload.requestCanAccessFiles(req)) { - res.writeHead(403); - return res.end(); - } - - const filePath = operation.generatedFile; - - if (!filePath) { - res.writeHead(404); - return res.end(); - } - - try { - const stat = Meteor.wrapAsync(fs.stat)(filePath); - - if (stat && stat.isFile()) { - const utcDate = stat.mtime.toISOString().split('T')[0]; - const newFileName = encodeURIComponent(`${ utcDate }-${ userName }`); - - res.setHeader('Content-Security-Policy', 'default-src \'none\''); - res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ newFileName }.zip`); - res.setHeader('Last-Modified', stat.mtime.toUTCString()); - - res.setHeader('Content-Type', 'application/zip'); - res.setHeader('Content-Length', stat.size); - - const stream = fs.createReadStream(filePath); - stream.on('close', () => { - res.end(); - }); - stream.pipe(res); - } - - return; - } catch (e) { - res.writeHead(404); - res.end(); - return; - } - } - } - } - - res.writeHead(404); - res.end(); -}); diff --git a/packages/rocketchat-user-data-download/server/startup/settings.js b/packages/rocketchat-user-data-download/server/startup/settings.js index 979338215d6cf..3f4cc24ba157c 100644 --- a/packages/rocketchat-user-data-download/server/startup/settings.js +++ b/packages/rocketchat-user-data-download/server/startup/settings.js @@ -6,33 +6,15 @@ RocketChat.settings.addGroup('UserDataDownload', function() { i18nLabel: 'UserData_EnableDownload' }); - this.add('UserData_Storage_Type', 'FileSystem', { - type: 'select', - public: true, - values: [{ - key: 'FileSystem', - i18nLabel: 'FileSystem' - }], - i18nLabel: 'FileUpload_Storage_Type' - }); - this.add('UserData_FileSystemPath', '', { type: 'string', public: true, - enableQuery: { - _id: 'UserData_Storage_Type', - value: 'FileSystem' - }, i18nLabel: 'UserData_FileSystemPath' }); this.add('UserData_FileSystemZipPath', '', { type: 'string', public: true, - enableQuery: { - _id: 'UserData_Storage_Type', - value: 'FileSystem' - }, i18nLabel: 'UserData_FileSystemZipPath' }); From b013ff164e096a82241da39f69155517250174ee Mon Sep 17 00:00:00 2001 From: Hudell Date: Tue, 27 Feb 2018 18:41:23 -0300 Subject: [PATCH 11/14] Lint --- packages/rocketchat-lib/package.js | 4 ++-- .../server/cronProcessDownloads.js | 19 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js index 8de0fa3a73a85..ccbd3e06c1b74 100644 --- a/packages/rocketchat-lib/package.js +++ b/packages/rocketchat-lib/package.js @@ -126,8 +126,8 @@ Package.onUse(function(api) { api.addFiles('server/models/Subscriptions.js', 'server'); api.addFiles('server/models/Uploads.js', 'server'); api.addFiles('server/models/Users.js', 'server'); - api.addFiles('server/models/ExportOperations.js', 'server'); - api.addFiles('server/models/UserDataFiles.js', 'server'); + api.addFiles('server/models/ExportOperations.js', 'server'); + api.addFiles('server/models/UserDataFiles.js', 'server'); api.addFiles('server/oauth/oauth.js', 'server'); api.addFiles('server/oauth/google.js', 'server'); diff --git a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js index 2f3d9ac30af3d..eb4bc791bdaea 100644 --- a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -252,7 +252,7 @@ const uploadZipFile = function(exportOperation, callback) { const filePath = exportOperation.generatedFile; const stat = Meteor.wrapAsync(fs.stat)(filePath); - const stream = fs.createReadStream(filePath); + const stream = fs.createReadStream(filePath); const contentType = 'application/zip'; const size = stat.size; @@ -265,7 +265,7 @@ const uploadZipFile = function(exportOperation, callback) { const newFileName = encodeURIComponent(`${ utcDate }-${ userDisplayName }.zip`); const details = { - userId: userId, + userId, type: contentType, size, name: newFileName @@ -273,8 +273,7 @@ const uploadZipFile = function(exportOperation, callback) { userDataStore.insert(details, stream, (err) => { if (err) { - logError({err}); - resolve(); + throw new Meteor.Error('invalid-file', 'Invalid Zip File', { method: 'cronProcessDownloads.uploadZipFile' }); } else { callback(); } @@ -292,35 +291,35 @@ const continueExportOperation = function(exportOperation) { try { //Run every room on every request, to avoid missing new messages on the rooms that finished first. - if (exportOperation.status == 'exporting') { + if (exportOperation.status === 'exporting') { exportOperation.roomList.forEach((exportOpRoomData) => { continueExportingRoom(exportOperation, exportOpRoomData); }); - + if (isOperationFinished(exportOperation)) { exportOperation.status = 'downloading'; return; } } - if (exportOperation.status == 'downloading') { + if (exportOperation.status === 'downloading') { exportOperation.fileList.forEach((attachmentData) => { copyFile(exportOperation, attachmentData); }); - + if (isDownloadFinished(exportOperation)) { exportOperation.status = 'compressing'; return; } } - if (exportOperation.status == 'compressing') { + if (exportOperation.status === 'compressing') { makeZipFile(exportOperation); exportOperation.status = 'uploading'; return; } - if (exportOperation.status == 'uploading') { + if (exportOperation.status === 'uploading') { uploadZipFile(exportOperation, () => { exportOperation.status = 'completed'; RocketChat.models.ExportOperations.updateOperation(exportOperation); From a606f73ffeb5cc51a11a6d88b7c3b76ecc981ab1 Mon Sep 17 00:00:00 2001 From: Hudell Date: Wed, 28 Feb 2018 13:04:46 -0300 Subject: [PATCH 12/14] Added support for Google and Amazon as storage types --- .../server/config/AmazonS3.js | 25 ++++++++++++++-- .../server/config/GoogleStorage.js | 29 +++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/AmazonS3.js b/packages/rocketchat-file-upload/server/config/AmazonS3.js index bc80d64b233dc..32141e73b7499 100644 --- a/packages/rocketchat-file-upload/server/config/AmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/AmazonS3.js @@ -23,15 +23,35 @@ const get = function(file, req, res) { } }; +const copy = function(file, out) { + const fileUrl = this.store.getRedirectURL(file); + + if (fileUrl) { + const request = /^https:/.test(fileUrl) ? https : http; + request.get(fileUrl, fileRes => fileRes.pipe(out)); + } else { + out.end(); + } +}; + const AmazonS3Uploads = new FileUploadClass({ name: 'AmazonS3:Uploads', - get + get, + copy // store setted bellow }); const AmazonS3Avatars = new FileUploadClass({ name: 'AmazonS3:Avatars', - get + get, + copy + // store setted bellow +}); + +const AmazonS3UserDataFiles = new FileUploadClass({ + name: 'AmazonS3:UserDataFiles', + get, + copy // store setted bellow }); @@ -72,6 +92,7 @@ const configure = _.debounce(function() { AmazonS3Uploads.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Uploads.name, config); AmazonS3Avatars.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Avatars.name, config); + AmazonS3UserDataFiles.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3UserDataFiles.name, config); }, 500); RocketChat.settings.get(/^FileUpload_S3_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/GoogleStorage.js b/packages/rocketchat-file-upload/server/config/GoogleStorage.js index fd571ea6ec8f6..8933d3201179f 100644 --- a/packages/rocketchat-file-upload/server/config/GoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/GoogleStorage.js @@ -27,15 +27,39 @@ const get = function(file, req, res) { }); }; +const copy = function(file, out) { + this.store.getRedirectURL(file, (err, fileUrl) => { + if (err) { + console.error(err); + } + + if (fileUrl) { + const request = /^https:/.test(fileUrl) ? https : http; + request.get(fileUrl, fileRes => fileRes.pipe(out)); + } else { + out.end(); + } + }); +}; + const GoogleCloudStorageUploads = new FileUploadClass({ name: 'GoogleCloudStorage:Uploads', - get + get, + copy // store setted bellow }); const GoogleCloudStorageAvatars = new FileUploadClass({ name: 'GoogleCloudStorage:Avatars', - get + get, + copy + // store setted bellow +}); + +const GoogleCloudStorageUserDataFiles = new FileUploadClass({ + name: 'GoogleCloudStorage:UserDataFiles', + get, + copy // store setted bellow }); @@ -62,6 +86,7 @@ const configure = _.debounce(function() { GoogleCloudStorageUploads.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUploads.name, config); GoogleCloudStorageAvatars.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageAvatars.name, config); + GoogleCloudStorageUserDataFiles.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUserDataFiles.name, config); }, 500); RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure); From 4f10a4f168284ef28351dfba682e0d3c47c37a9f Mon Sep 17 00:00:00 2001 From: Hudell Date: Mon, 5 Mar 2018 17:01:48 -0300 Subject: [PATCH 13/14] Split the options to download and export data --- .../server/lib/FileUpload.js | 5 +- packages/rocketchat-i18n/i18n/en.i18n.json | 1 + .../server/models/ExportOperations.js | 5 +- .../client/accountPreferences.html | 1 + .../client/accountPreferences.js | 12 +- .../server/cronProcessDownloads.js | 186 ++++++++++++++---- server/methods/requestDataDownload.js | 24 ++- 7 files changed, 186 insertions(+), 48 deletions(-) diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index ec0cb41d1ef58..3194dc8999a82 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -1,7 +1,6 @@ /* globals UploadFS */ import fs from 'fs'; -import path from 'path'; import stream from 'stream'; import mime from 'mime-type/with-db'; import Future from 'fibers/future'; @@ -251,10 +250,8 @@ Object.assign(FileUpload, { res.end(); }, - copy(file, targetFolder) { + copy(file, targetFile) { const store = this.getStoreByName(file.store); - const targetFile = path.join(targetFolder, `${ file._id }-${ file.name }`); - const out = fs.createWriteStream(targetFile); file = FileUpload.addExtensionTo(file); diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index 5e887fc701bf0..37e67a8e4a88c 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -712,6 +712,7 @@ "Example_s": "Example: %s", "Exclude_Botnames": "Exclude Bots", "Exclude_Botnames_Description": "Do not propagate messages from bots whose name matches the regular expression above. If left empty, all messages from bots will be propagated.", + "Export_My_Data" : "Export My Data", "External_Service": "External Service", "External_Queue_Service_URL": "External Queue Service URL", "Facebook_Page": "Facebook Page", diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js index 1f7e029bc1684..82bf0e29bb4ba 100644 --- a/packages/rocketchat-lib/server/models/ExportOperations.js +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -15,9 +15,10 @@ RocketChat.models.ExportOperations = new class ModelExportOperations extends Roc return this.find(query); } - findLastOperationByUser(userId, options = {}) { + findLastOperationByUser(userId, fullExport = false, options = {}) { const query = { - userId + userId, + fullExport }; options.sort = {'createdAt' : -1}; diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html index ef799599b3b3c..9efbe34ebc7f2 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.html +++ b/packages/rocketchat-ui-account/client/accountPreferences.html @@ -298,6 +298,7 @@

{{_ "My Data"}}

+
diff --git a/packages/rocketchat-ui-account/client/accountPreferences.js b/packages/rocketchat-ui-account/client/accountPreferences.js index f856c469be258..988e50a49d74b 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.js +++ b/packages/rocketchat-ui-account/client/accountPreferences.js @@ -199,8 +199,8 @@ Template.accountPreferences.onCreated(function() { }); }; - this.downloadMyData = function() { - Meteor.call('requestDataDownload', {}, function(error, results) { + this.downloadMyData = function(fullExport = false) { + Meteor.call('requestDataDownload', {fullExport}, function(error, results) { if (results) { if (results.requested) { modal.open({ @@ -243,6 +243,10 @@ Template.accountPreferences.onCreated(function() { } }); }; + + this.exportMyData = function() { + this.downloadMyData(true); + }; }); Template.accountPreferences.onRendered(function() { @@ -266,6 +270,10 @@ Template.accountPreferences.events({ e.preventDefault(); t.downloadMyData(); }, + 'click .export-my-data'(e, t) { + e.preventDefault(); + t.exportMyData(); + }, 'click .test-notifications'(e) { e.preventDefault(); KonchatNotification.notify({ diff --git a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js index eb4bc791bdaea..ddac705ca18ec 100644 --- a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -4,7 +4,7 @@ import fs from 'fs'; import path from 'path'; import archiver from 'archiver'; -let zipFolder = '~/zipFiles'; +let zipFolder = '/tmp/zipFiles'; if (RocketChat.settings.get('UserData_FileSystemZipPath') != null) { if (RocketChat.settings.get('UserData_FileSystemZipPath').trim() !== '') { zipFolder = RocketChat.settings.get('UserData_FileSystemZipPath'); @@ -33,24 +33,43 @@ const createDir = function(folderName) { const loadUserSubscriptions = function(exportOperation) { exportOperation.roomList = []; - const cursor = RocketChat.models.Subscriptions.findByUserId(exportOperation.userId); + const exportUserId = exportOperation.userId; + const cursor = RocketChat.models.Subscriptions.findByUserId(exportUserId); cursor.forEach((subscription) => { - const roomId = subscription._room._id; + const roomId = subscription.rid; const roomData = subscription._room; - const roomName = roomData.name ? roomData.name : roomId; + let roomName = roomData.name ? roomData.name : roomId; + let userId = null; - const fileName = `${ roomName }.json`; + if (subscription.t === 'd') { + userId = roomId.replace(exportUserId, ''); + const userData = RocketChat.models.Users.findOneById(userId); + + if (userData) { + roomName = userData.name; + } + } + + const fileName = exportOperation.fullExport ? roomId : roomName; + const fileType = exportOperation.fullExport ? 'json' : 'html'; + const targetFile = `${ fileName }.${ fileType }`; exportOperation.roomList.push({ roomId, roomName, + userId, exportedCount: 0, status: 'pending', - targetFile: fileName + targetFile, + type: subscription.t }); }); - exportOperation.status = 'exporting'; + if (exportOperation.fullExport) { + exportOperation.status = 'exporting-rooms'; + } else { + exportOperation.status = 'exporting'; + } }; const getAttachmentData = function(attachment) { @@ -67,21 +86,47 @@ const getAttachmentData = function(attachment) { video_size: attachment.video_size, video_type: attachment.video_type, audio_size: attachment.audio_size, - audio_type: attachment.audio_type + audio_type: attachment.audio_type, + url: null, + remote: false, + fileId: null, + fileName: null }; + const url = attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url || attachment.message_link; + if (url) { + attachmentData.url = url; + + const urlMatch = /\:\/\//.exec(url); + if (urlMatch && urlMatch.length > 0) { + attachmentData.remote = true; + } else { + const match = /^\/([^\/]+)\/([^\/]+)\/(.*)/.exec(url); + + if (match && match[2]) { + const file = RocketChat.models.Uploads.findOneById(match[2]); + + if (file) { + attachmentData.fileId = file._id; + attachmentData.fileName = file.name; + } + } + } + } + return attachmentData; }; const addToFileList = function(exportOperation, attachment) { - const url = attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url || attachment.message_link; - if (!url) { - return; - } + const targetFile = path.join(exportOperation.assetsPath, `${ attachment.fileId }-${ attachment.fileName }`); const attachmentData = { - url, - copied: false + url: attachment.url, + copied: false, + remote: attachment.remote, + fileId: attachment.fileId, + fileName: attachment.fileName, + targetFile }; exportOperation.fileList.push(attachmentData); @@ -95,7 +140,7 @@ const getMessageData = function(msg, exportOperation) { const attachmentData = getAttachmentData(attachment); attachments.push(attachmentData); - addToFileList(exportOperation, attachment); + addToFileList(exportOperation, attachmentData); }); } @@ -119,26 +164,16 @@ const getMessageData = function(msg, exportOperation) { }; const copyFile = function(exportOperation, attachmentData) { - if (attachmentData.copied) { - return; - } - - //If it is an URL, just mark as downloaded - const urlMatch = /\:\/\//.exec(attachmentData.url); - if (urlMatch && urlMatch.length > 0) { + if (attachmentData.copied || attachmentData.remote || !attachmentData.fileId) { attachmentData.copied = true; return; } - const match = /^\/([^\/]+)\/([^\/]+)\/(.*)/.exec(attachmentData.url); + const file = RocketChat.models.Uploads.findOneById(attachmentData.fileId); - if (match && match[2]) { - const file = RocketChat.models.Uploads.findOneById(match[2]); - - if (file) { - if (FileUpload.copy(file, exportOperation.assetsPath)) { - attachmentData.copied = true; - } + if (file) { + if (FileUpload.copy(file, attachmentData.targetFile)) { + attachmentData.copied = true; } } }; @@ -152,6 +187,9 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { if (exportOpRoomData.status === 'pending') { exportOpRoomData.status = 'exporting'; startFile(filePath, ''); + if (!exportOperation.fullExport) { + writeToFile(filePath, ''); + } } let limit = 100; @@ -166,9 +204,62 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { cursor.forEach((msg) => { const messageObject = getMessageData(msg, exportOperation); - const messageString = JSON.stringify(messageObject); - writeToFile(filePath, `${ messageString }\n`); + if (exportOperation.fullExport) { + const messageString = JSON.stringify(messageObject); + writeToFile(filePath, `${ messageString }\n`); + } else { + const messageType = msg.t; + const userName = msg.u.username || msg.u.name; + const timestamp = msg.ts ? new Date(msg.ts).toUTCString() : ''; + let message = msg.msg; + + switch (messageType) { + case 'uj': + message = TAPi18n.__('User_joined_channel'); + break; + case 'ul': + message = TAPi18n.__('User_left'); + break; + case 'au': + message = TAPi18n.__('User_added_by', {user_added : msg.msg, user_by : msg.u.username }); + break; + case 'r': + message = TAPi18n.__('Room_name_changed', { room_name: msg.msg, user_by: msg.u.username }); + break; + case 'ru': + message = TAPi18n.__('User_removed_by', {user_removed : msg.msg, user_by : msg.u.username }); + break; + case 'wm': + message = TAPi18n.__('Welcome', {user: msg.u.username }); + break; + case 'livechat-close': + message = TAPi18n.__('Conversation_finished'); + break; + } + + if (message !== msg.msg) { + message = `${ message }`; + } + + writeToFile(filePath, `

${ userName } (${ timestamp }):
`); + writeToFile(filePath, message); + + if (messageObject.attachments && messageObject.attachments.length > 0) { + messageObject.attachments.forEach((attachment) => { + if (attachment.type === 'file') { + const description = attachment.description || attachment.title || TAPi18n.__('Message_Attachments'); + + const assetUrl = `./assets/${ attachment.fileId }-${ attachment.fileName }`; + const link = `
${ description }`; + writeToFile(filePath, link); + } + }); + } + + writeToFile(filePath, '

'); + } + exportOpRoomData.exportedCount++; }); @@ -180,7 +271,7 @@ const continueExportingRoom = function(exportOperation, exportOpRoomData) { return false; }; -const isOperationFinished = function(exportOperation) { +const isExportComplete = function(exportOperation) { const incomplete = exportOperation.roomList.some((exportOpRoomData) => { return exportOpRoomData.status !== 'completed'; }); @@ -190,7 +281,7 @@ const isOperationFinished = function(exportOperation) { const isDownloadFinished = function(exportOperation) { const anyDownloadPending = exportOperation.fileList.some((fileData) => { - return !fileData.copied; + return !fileData.copied && !fileData.remote; }); return !anyDownloadPending; @@ -280,6 +371,26 @@ const uploadZipFile = function(exportOperation, callback) { }); }; +const generateChannelsFile = function(exportOperation) { + if (exportOperation.fullExport) { + const fileName = path.join(exportOperation.exportPath, 'channels.json'); + startFile(fileName, ''); + + exportOperation.roomList.forEach((roomData) => { + const newRoomData = { + roomId: roomData.roomId, + roomName: roomData.roomName, + type: roomData.type + }; + + const messageString = JSON.stringify(newRoomData); + writeToFile(fileName, `${ messageString }\n`); + }); + } + + exportOperation.status = 'exporting'; +}; + const continueExportOperation = function(exportOperation) { if (exportOperation.status === 'completed') { return; @@ -290,13 +401,18 @@ const continueExportOperation = function(exportOperation) { } try { + + if (exportOperation.status === 'exporting-rooms') { + generateChannelsFile(exportOperation); + } + //Run every room on every request, to avoid missing new messages on the rooms that finished first. if (exportOperation.status === 'exporting') { exportOperation.roomList.forEach((exportOpRoomData) => { continueExportingRoom(exportOperation, exportOpRoomData); }); - if (isOperationFinished(exportOperation)) { + if (isExportComplete(exportOperation)) { exportOperation.status = 'downloading'; return; } diff --git a/server/methods/requestDataDownload.js b/server/methods/requestDataDownload.js index 752450b734238..13f7aeaf224d7 100644 --- a/server/methods/requestDataDownload.js +++ b/server/methods/requestDataDownload.js @@ -1,6 +1,7 @@ +import fs from 'fs'; import path from 'path'; -let tempFolder = '~/userData'; +let tempFolder = '/tmp/userData'; if (RocketChat.settings.get('UserData_FileSystemPath') != null) { if (RocketChat.settings.get('UserData_FileSystemPath').trim() !== '') { tempFolder = RocketChat.settings.get('UserData_FileSystemPath'); @@ -8,11 +9,11 @@ if (RocketChat.settings.get('UserData_FileSystemPath') != null) { } Meteor.methods({ - requestDataDownload() { + requestDataDownload({fullExport = false}) { const currentUserData = Meteor.user(); const userId = currentUserData._id; - const lastOperation = RocketChat.models.ExportOperations.findLastOperationByUser(userId); + const lastOperation = RocketChat.models.ExportOperations.findLastOperationByUser(userId, fullExport); if (lastOperation) { const yesterday = new Date(); @@ -26,8 +27,20 @@ Meteor.methods({ } } - const folderName = path.join(tempFolder, userId); + const subFolderName = fullExport ? 'full' : 'partial'; + const baseFolder = path.join(tempFolder, userId); + if (!fs.existsSync(baseFolder)) { + fs.mkdirSync(baseFolder); + } + + const folderName = path.join(baseFolder, subFolderName); + if (!fs.existsSync(folderName)) { + fs.mkdirSync(folderName); + } const assetsFolder = path.join(folderName, 'assets'); + if (!fs.existsSync(assetsFolder)) { + fs.mkdirSync(assetsFolder); + } const exportOperation = { userId : currentUserData._id, @@ -36,7 +49,8 @@ Meteor.methods({ exportPath: folderName, assetsPath: assetsFolder, fileList: [], - generatedFile: null + generatedFile: null, + fullExport }; RocketChat.models.ExportOperations.create(exportOperation); From 79743a85c6a07339669ff518a3503f0c0894fe2c Mon Sep 17 00:00:00 2001 From: Hudell Date: Mon, 5 Mar 2018 17:21:48 -0300 Subject: [PATCH 14/14] Removed commented code --- .../rocketchat-file-upload/client/lib/fileUploadHandler.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index cbcf7c6df4f8f..8b6008b3fa5bd 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -17,11 +17,6 @@ new UploadFS.Store({ }) }); -// new UploadFS.Store({ -// collection: RocketChat.models.UserDataFiles.model, -// name: 'UserDataFiles' -// }); - fileUploadHandler = (directive, meta, file) => { const store = UploadFS.getStore(directive);