diff --git a/.meteor/packages b/.meteor/packages index 951eadb8980c3..3ba1459162262 100644 --- a/.meteor/packages +++ b/.meteor/packages @@ -146,6 +146,7 @@ rocketchat:ui-master rocketchat:ui-message rocketchat:ui-sidenav rocketchat:ui-vrecord +rocketchat:user-data-download rocketchat:version rocketchat:videobridge rocketchat:webrtc diff --git a/.meteor/versions b/.meteor/versions index 3fa982c52f2d7..4cd41c05e9db3 100644 --- a/.meteor/versions +++ b/.meteor/versions @@ -235,6 +235,7 @@ rocketchat:ui-master@0.1.0 rocketchat:ui-message@0.1.0 rocketchat:ui-sidenav@0.1.0 rocketchat:ui-vrecord@0.0.1 +rocketchat:user-data-download@1.0.0 rocketchat:version@1.0.0 rocketchat:version-check@0.0.1 rocketchat:videobridge@0.2.0 diff --git a/package.json b/package.json index e88d1abbd7eff..76870049621ab 100644 --- a/package.json +++ b/package.json @@ -113,6 +113,7 @@ "@google-cloud/storage": "^1.6.0", "@google-cloud/vision": "^0.15.2", "adm-zip": "^0.4.7", + "archiver": "^2.1.1", "atlassian-crowd": "^0.5.0", "autolinker": "^1.6.2", "aws-sdk": "^2.199.0", diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index 73474ab5181f3..8b6008b3fa5bd 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -17,7 +17,6 @@ new UploadFS.Store({ }) }); - fileUploadHandler = (directive, meta, file) => { const store = UploadFS.getStore(directive); diff --git a/packages/rocketchat-file-upload/lib/FileUploadBase.js b/packages/rocketchat-file-upload/lib/FileUploadBase.js index 3e2b166547f59..97fd9ced52731 100644 --- a/packages/rocketchat-file-upload/lib/FileUploadBase.js +++ b/packages/rocketchat-file-upload/lib/FileUploadBase.js @@ -4,7 +4,21 @@ import _ from 'underscore'; UploadFS.config.defaultStorePermissions = new UploadFS.StorePermissions({ insert(userId, doc) { - return userId || (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0); // allow inserts from slackbridge (message_id = slack-timestamp-milli) + if (userId) { + return true; + } + + // allow inserts from slackbridge (message_id = slack-timestamp-milli) + if (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0) { + return true; + } + + // allow inserts to the UserDataFiles store + if (doc && doc.store && doc.store.split(':').pop() === 'UserDataFiles') { + return true; + } + + return false; }, update(userId, doc) { return RocketChat.authz.hasPermission(Meteor.userId(), 'delete-message', doc.rid) || (RocketChat.settings.get('Message_AllowDeleting') && userId === doc.userId); diff --git a/packages/rocketchat-file-upload/server/config/AmazonS3.js b/packages/rocketchat-file-upload/server/config/AmazonS3.js index f633c386e21e5..571d592465100 100644 --- a/packages/rocketchat-file-upload/server/config/AmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/AmazonS3.js @@ -25,15 +25,35 @@ const get = function(file, req, res) { } }; +const copy = function(file, out) { + const fileUrl = this.store.getRedirectURL(file); + + if (fileUrl) { + const request = /^https:/.test(fileUrl) ? https : http; + request.get(fileUrl, fileRes => fileRes.pipe(out)); + } else { + out.end(); + } +}; + const AmazonS3Uploads = new FileUploadClass({ name: 'AmazonS3:Uploads', - get + get, + copy // store setted bellow }); const AmazonS3Avatars = new FileUploadClass({ name: 'AmazonS3:Avatars', - get + get, + copy + // store setted bellow +}); + +const AmazonS3UserDataFiles = new FileUploadClass({ + name: 'AmazonS3:UserDataFiles', + get, + copy // store setted bellow }); @@ -74,6 +94,7 @@ const configure = _.debounce(function() { AmazonS3Uploads.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Uploads.name, config); AmazonS3Avatars.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Avatars.name, config); + AmazonS3UserDataFiles.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3UserDataFiles.name, config); }, 500); RocketChat.settings.get(/^FileUpload_S3_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/FileSystem.js b/packages/rocketchat-file-upload/server/config/FileSystem.js index 096df87e4dee0..fab473b5ab6cf 100644 --- a/packages/rocketchat-file-upload/server/config/FileSystem.js +++ b/packages/rocketchat-file-upload/server/config/FileSystem.js @@ -28,6 +28,22 @@ const FileSystemUploads = new FileUploadClass({ res.end(); return; } + }, + + copy(file, out) { + const filePath = this.store.getFilePath(file._id, file); + try { + const stat = Meteor.wrapAsync(fs.stat)(filePath); + + if (stat && stat.isFile()) { + file = FileUpload.addExtensionTo(file); + + this.store.getReadStream(file._id, file).pipe(out); + } + } catch (e) { + out.end(); + return; + } } }); @@ -54,6 +70,31 @@ const FileSystemAvatars = new FileUploadClass({ } }); +const FileSystemUserDataFiles = new FileUploadClass({ + name: 'FileSystem:UserDataFiles', + + get(file, req, res) { + const filePath = this.store.getFilePath(file._id, file); + + try { + const stat = Meteor.wrapAsync(fs.stat)(filePath); + + if (stat && stat.isFile()) { + file = FileUpload.addExtensionTo(file); + res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`); + res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); + res.setHeader('Content-Type', file.type); + res.setHeader('Content-Length', file.size); + + this.store.getReadStream(file._id, file).pipe(res); + } + } catch (e) { + res.writeHead(404); + res.end(); + return; + } + } +}); const createFileSystemStore = _.debounce(function() { const options = { @@ -62,6 +103,7 @@ const createFileSystemStore = _.debounce(function() { FileSystemUploads.store = FileUpload.configureUploadsStore('Local', FileSystemUploads.name, options); FileSystemAvatars.store = FileUpload.configureUploadsStore('Local', FileSystemAvatars.name, options); + FileSystemUserDataFiles.store = FileUpload.configureUploadsStore('Local', FileSystemUserDataFiles.name, options); // DEPRECATED backwards compatibililty (remove) UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name]; diff --git a/packages/rocketchat-file-upload/server/config/GoogleStorage.js b/packages/rocketchat-file-upload/server/config/GoogleStorage.js index fdcebdc352b48..67ecb74b4f192 100644 --- a/packages/rocketchat-file-upload/server/config/GoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/GoogleStorage.js @@ -29,15 +29,39 @@ const get = function(file, req, res) { }); }; +const copy = function(file, out) { + this.store.getRedirectURL(file, (err, fileUrl) => { + if (err) { + console.error(err); + } + + if (fileUrl) { + const request = /^https:/.test(fileUrl) ? https : http; + request.get(fileUrl, fileRes => fileRes.pipe(out)); + } else { + out.end(); + } + }); +}; + const GoogleCloudStorageUploads = new FileUploadClass({ name: 'GoogleCloudStorage:Uploads', - get + get, + copy // store setted bellow }); const GoogleCloudStorageAvatars = new FileUploadClass({ name: 'GoogleCloudStorage:Avatars', - get + get, + copy + // store setted bellow +}); + +const GoogleCloudStorageUserDataFiles = new FileUploadClass({ + name: 'GoogleCloudStorage:UserDataFiles', + get, + copy // store setted bellow }); @@ -64,6 +88,7 @@ const configure = _.debounce(function() { GoogleCloudStorageUploads.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUploads.name, config); GoogleCloudStorageAvatars.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageAvatars.name, config); + GoogleCloudStorageUserDataFiles.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUserDataFiles.name, config); }, 500); RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/GridFS.js b/packages/rocketchat-file-upload/server/config/GridFS.js index 54bb40712cacd..0ad63b5e86264 100644 --- a/packages/rocketchat-file-upload/server/config/GridFS.js +++ b/packages/rocketchat-file-upload/server/config/GridFS.js @@ -62,7 +62,6 @@ const getByteRange = function(header) { return null; }; - // code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310 const readFromGridFS = function(storeName, fileId, file, req, res) { const store = UploadFS.getStore(storeName); @@ -123,10 +122,26 @@ const readFromGridFS = function(storeName, fileId, file, req, res) { } }; +const copyFromGridFS = function(storeName, fileId, file, out) { + const store = UploadFS.getStore(storeName); + const rs = store.getReadStream(fileId, file); + + [rs, out].forEach(stream => stream.on('error', function(err) { + store.onReadError.call(store, err, fileId, file); + out.end(); + })); + + rs.pipe(out); +}; + FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', { collectionName: 'rocketchat_uploads' }); +FileUpload.configureUploadsStore('GridFS', 'GridFS:UserDataFiles', { + collectionName: 'rocketchat_userDataFiles' +}); + // DEPRECATED: backwards compatibility (remove) UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; @@ -147,6 +162,29 @@ new FileUploadClass({ res.setHeader('Content-Length', file.size); return readFromGridFS(file.store, file._id, file, req, res); + }, + + copy(file, out) { + copyFromGridFS(file.store, file._id, file, out); + } +}); + +new FileUploadClass({ + name: 'GridFS:UserDataFiles', + + get(file, req, res) { + file = FileUpload.addExtensionTo(file); + + res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`); + res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); + res.setHeader('Content-Type', file.type); + res.setHeader('Content-Length', file.size); + + return readFromGridFS(file.store, file._id, file, req, res); + }, + + copy(file, out) { + copyFromGridFS(file.store, file._id, file, out); } }); diff --git a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js index d2ba5505f2d4d..8855eb76fca8a 100644 --- a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js +++ b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js @@ -14,6 +14,7 @@ const configStore = _.debounce(() => { console.log('Setting default file store to', store); UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`); UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`); + UploadFS.getStores().UserDataFiles = UploadFS.getStore(`${ store }:UserDataFiles`); } }, 1000); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index a056e991fba5e..3194dc8999a82 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -58,6 +58,25 @@ Object.assign(FileUpload, { }; }, + defaultUserDataFiles() { + return { + collection: RocketChat.models.UserDataFiles.model, + getPath(file) { + return `${ RocketChat.settings.get('uniqueID') }/uploads/userData/${ file.userId }`; + }, + onValidate: FileUpload.uploadsOnValidate, + onRead(fileId, file, req, res) { + if (!FileUpload.requestCanAccessFiles(req)) { + res.writeHead(403); + return false; + } + + res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`); + return true; + } + }; + }, + avatarsOnValidate(file) { if (RocketChat.settings.get('Accounts_AvatarResize') !== true) { return; @@ -229,16 +248,30 @@ Object.assign(FileUpload, { } res.writeHead(404); res.end(); + }, + + copy(file, targetFile) { + const store = this.getStoreByName(file.store); + const out = fs.createWriteStream(targetFile); + + file = FileUpload.addExtensionTo(file); + + if (store.copy) { + store.copy(file, out); + return true; + } + + return false; } }); - export class FileUploadClass { - constructor({ name, model, store, get, insert, getStore }) { + constructor({ name, model, store, get, insert, getStore, copy }) { this.name = name; this.model = model || this.getModelFromName(); this._store = store || UploadFS.getStore(name); this.get = get; + this.copy = copy; if (insert) { this.insert = insert; diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index c6ddc5191eec6..9c2c3ca6cb30f 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -567,6 +567,7 @@ "Domain_removed": "Domain Removed", "Domains": "Domains", "Domains_allowed_to_embed_the_livechat_widget": "Comma-separated list of domains allowed to embed the livechat widget. Leave blank to allow all domains.", + "Download_My_Data" : "Download My Data", "Download_Snippet": "Download", "Drop_to_upload_file": "Drop to upload file", "Dry_run": "Dry run", @@ -724,6 +725,7 @@ "Example_s": "Example: %s", "Exclude_Botnames": "Exclude Bots", "Exclude_Botnames_Description": "Do not propagate messages from bots whose name matches the regular expression above. If left empty, all messages from bots will be propagated.", + "Export_My_Data" : "Export My Data", "External_Service": "External Service", "External_Queue_Service_URL": "External Queue Service URL", "Facebook_Page": "Facebook Page", @@ -2106,6 +2108,18 @@ "User_uploaded_file": "Uploaded a file", "User_uploaded_image": "Uploaded an image", "User_Presence": "User Presence", + "UserDataDownload" : "User Data Download", + "UserData_EnableDownload" : "Enable User Data Download", + "UserData_FileSystemPath" : "System Path (Exported Files)", + "UserData_FileSystemZipPath" : "System Path (Compressed File)", + "UserData_ProcessingFrequency" : "Processing Frequency (Minutes)", + "UserData_MessageLimitPerRequest" : "Message Limit per Request", + "UserDataDownload_EmailSubject" : "Your Data File is Ready to Download", + "UserDataDownload_EmailBody" : "Your data file is now ready to download. Click here to download it.", + "UserDataDownload_Requested" : "Download File Requested", + "UserDataDownload_Requested_Text" : "Your data file will be generated. A link to download it will be sent to your email address when ready.", + "UserDataDownload_RequestExisted_Text" : "Your data file is already being generated. A link to download it will be sent to your email address when ready.", + "UserDataDownload_CompletedRequestExisted_Text" : "Your data file was already generated. Check your email account for the download link.", "Username": "Username", "Username_and_message_must_not_be_empty": "Username and message must not be empty.", "Username_cant_be_empty": "The username cannot be empty", diff --git a/packages/rocketchat-lib/client/models/UserDataFiles.js b/packages/rocketchat-lib/client/models/UserDataFiles.js new file mode 100644 index 0000000000000..80e7e22e162ed --- /dev/null +++ b/packages/rocketchat-lib/client/models/UserDataFiles.js @@ -0,0 +1,6 @@ +RocketChat.models.UserDataFiles = new class extends RocketChat.models._Base { + constructor() { + super(); + this._initModel('userDataFiles'); + } +}; diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js index eddbc615d0e5c..d85c5b0842af0 100644 --- a/packages/rocketchat-lib/package.js +++ b/packages/rocketchat-lib/package.js @@ -126,6 +126,8 @@ Package.onUse(function(api) { api.addFiles('server/models/Subscriptions.js', 'server'); api.addFiles('server/models/Uploads.js', 'server'); api.addFiles('server/models/Users.js', 'server'); + api.addFiles('server/models/ExportOperations.js', 'server'); + api.addFiles('server/models/UserDataFiles.js', 'server'); api.addFiles('server/oauth/oauth.js', 'server'); api.addFiles('server/oauth/facebook.js', 'server'); diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js new file mode 100644 index 0000000000000..d7beda9d94b92 --- /dev/null +++ b/packages/rocketchat-lib/server/models/ExportOperations.js @@ -0,0 +1,78 @@ +import _ from 'underscore'; + +RocketChat.models.ExportOperations = new class ModelExportOperations extends RocketChat.models._Base { + constructor() { + super('export_operations'); + + this.tryEnsureIndex({ 'userId': 1 }); + this.tryEnsureIndex({ 'status': 1 }); + } + + // FIND + findById(id) { + const query = {_id: id}; + + return this.find(query); + } + + findLastOperationByUser(userId, fullExport = false, options = {}) { + const query = { + userId, + fullExport + }; + + options.sort = {'createdAt' : -1}; + return this.findOne(query, options); + } + + findPendingByUser(userId, options) { + const query = { + userId, + status: { + $nin: ['completed'] + } + }; + + return this.find(query, options); + } + + findAllPending(options) { + const query = { + status: { $nin: ['completed'] } + }; + + return this.find(query, options); + } + + // UPDATE + updateOperation(data) { + const update = { + $set: { + roomList: data.roomList, + status: data.status, + fileList: data.fileList, + generatedFile: data.generatedFile + } + }; + + return this.update(data._id, update); + } + + + // INSERT + create(data) { + const exportOperation = { + createdAt: new Date + }; + + _.extend(exportOperation, data); + + return this.insert(exportOperation); + } + + + // REMOVE + removeById(_id) { + return this.remove(_id); + } +}; diff --git a/packages/rocketchat-lib/server/models/Messages.js b/packages/rocketchat-lib/server/models/Messages.js index 2bb631520ddb4..4fe8b43864628 100644 --- a/packages/rocketchat-lib/server/models/Messages.js +++ b/packages/rocketchat-lib/server/models/Messages.js @@ -284,6 +284,14 @@ RocketChat.models.Messages = new class extends RocketChat.models._Base { return this.findOne(query); } + findByRoomId(roomId, options) { + const query = { + rid: roomId + }; + + return this.find(query, options); + } + getLastVisibleMessageSentWithNoTypeByRoomId(rid, messageId) { const query = { rid, diff --git a/packages/rocketchat-lib/server/models/UserDataFiles.js b/packages/rocketchat-lib/server/models/UserDataFiles.js new file mode 100644 index 0000000000000..de9efbb5e883c --- /dev/null +++ b/packages/rocketchat-lib/server/models/UserDataFiles.js @@ -0,0 +1,40 @@ +import _ from 'underscore'; + +RocketChat.models.UserDataFiles = new class ModelUserDataFiles extends RocketChat.models._Base { + constructor() { + super('user_data_files'); + + this.tryEnsureIndex({ 'userId': 1 }); + } + + // FIND + findById(id) { + const query = {_id: id}; + return this.find(query); + } + + findLastFileByUser(userId, options = {}) { + const query = { + userId + }; + + options.sort = {'_updatedAt' : -1}; + return this.findOne(query, options); + } + + // INSERT + create(data) { + const userDataFile = { + createdAt: new Date + }; + + _.extend(userDataFile, data); + + return this.insert(userDataFile); + } + + // REMOVE + removeById(_id) { + return this.remove(_id); + } +}; diff --git a/packages/rocketchat-lib/server/models/Users.js b/packages/rocketchat-lib/server/models/Users.js index 9217dad7dc8f7..db9982b25ca0a 100644 --- a/packages/rocketchat-lib/server/models/Users.js +++ b/packages/rocketchat-lib/server/models/Users.js @@ -51,6 +51,11 @@ class ModelUsers extends RocketChat.models._Base { return this.findOne(query, options); } + findOneById(userId) { + const query = {_id: userId}; + + return this.findOne(query); + } // FIND findById(userId) { diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html index d7d50eedf8d2d..501b28ccd0278 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.html +++ b/packages/rocketchat-ui-account/client/accountPreferences.html @@ -296,6 +296,19 @@

{{_ "Sound"}}

+ + {{#if userDataDownloadEnabled}} +
+

{{_ "My Data"}}

+
+
+ + +
+ +
+
+ {{/if}} diff --git a/packages/rocketchat-ui-account/client/accountPreferences.js b/packages/rocketchat-ui-account/client/accountPreferences.js index 6d05e9470f963..9dcbb99ff2b69 100644 --- a/packages/rocketchat-ui-account/client/accountPreferences.js +++ b/packages/rocketchat-ui-account/client/accountPreferences.js @@ -87,6 +87,9 @@ Template.accountPreferences.helpers({ showRoles() { return RocketChat.settings.get('UI_DisplayRoles'); }, + userDataDownloadEnabled() { + return RocketChat.settings.get('UserData_EnableDownload') !== false; + }, notificationsSoundVolume() { return RocketChat.getUserPreference(Meteor.user(), 'notificationsSoundVolume'); } @@ -202,6 +205,55 @@ Template.accountPreferences.onCreated(function() { } }); }; + + this.downloadMyData = function(fullExport = false) { + Meteor.call('requestDataDownload', {fullExport}, function(error, results) { + if (results) { + if (results.requested) { + modal.open({ + title: t('UserDataDownload_Requested'), + text: t('UserDataDownload_Requested_Text'), + type: 'success' + }); + + return true; + } + + if (results.exportOperation) { + if (results.exportOperation.status === 'completed') { + modal.open({ + title: t('UserDataDownload_Requested'), + text: t('UserDataDownload_CompletedRequestExisted_Text'), + type: 'success' + }); + + return true; + } + + modal.open({ + title: t('UserDataDownload_Requested'), + text: t('UserDataDownload_RequestExisted_Text'), + type: 'success' + }); + return true; + } + + modal.open({ + title: t('UserDataDownload_Requested'), + type: 'success' + }); + return true; + } + + if (error) { + return handleError(error); + } + }); + }; + + this.exportMyData = function() { + this.downloadMyData(true); + }; }); Template.accountPreferences.onRendered(function() { @@ -221,6 +273,14 @@ Template.accountPreferences.events({ 'click .enable-notifications'() { KonchatNotification.getDesktopPermission(); }, + 'click .download-my-data'(e, t) { + e.preventDefault(); + t.downloadMyData(); + }, + 'click .export-my-data'(e, t) { + e.preventDefault(); + t.exportMyData(); + }, 'click .test-notifications'(e) { e.preventDefault(); KonchatNotification.notify({ diff --git a/packages/rocketchat-user-data-download/package.js b/packages/rocketchat-user-data-download/package.js new file mode 100644 index 0000000000000..d3eacedee5023 --- /dev/null +++ b/packages/rocketchat-user-data-download/package.js @@ -0,0 +1,18 @@ +Package.describe({ + name: 'rocketchat:user-data-download', + version: '1.0.0', + summary: 'Adds setting to allow the user to download all their data stored in the servers.', + git: '' +}); + +Package.onUse(function(api) { + api.use([ + 'ecmascript', + 'rocketchat:file', + 'rocketchat:lib', + 'webapp' + ]); + + api.addFiles('server/startup/settings.js', 'server'); + api.addFiles('server/cronProcessDownloads.js', 'server'); +}); diff --git a/packages/rocketchat-user-data-download/server/cronProcessDownloads.js b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js new file mode 100644 index 0000000000000..ddac705ca18ec --- /dev/null +++ b/packages/rocketchat-user-data-download/server/cronProcessDownloads.js @@ -0,0 +1,476 @@ +/* globals SyncedCron */ + +import fs from 'fs'; +import path from 'path'; +import archiver from 'archiver'; + +let zipFolder = '/tmp/zipFiles'; +if (RocketChat.settings.get('UserData_FileSystemZipPath') != null) { + if (RocketChat.settings.get('UserData_FileSystemZipPath').trim() !== '') { + zipFolder = RocketChat.settings.get('UserData_FileSystemZipPath'); + } +} + +let processingFrequency = 15; +if (RocketChat.settings.get('UserData_ProcessingFrequency') > 0) { + processingFrequency = RocketChat.settings.get('UserData_ProcessingFrequency'); +} + +const startFile = function(fileName, content) { + fs.writeFileSync(fileName, content); +}; + +const writeToFile = function(fileName, content) { + fs.appendFileSync(fileName, content); +}; + +const createDir = function(folderName) { + if (!fs.existsSync(folderName)) { + fs.mkdirSync(folderName); + } +}; + +const loadUserSubscriptions = function(exportOperation) { + exportOperation.roomList = []; + + const exportUserId = exportOperation.userId; + const cursor = RocketChat.models.Subscriptions.findByUserId(exportUserId); + cursor.forEach((subscription) => { + const roomId = subscription.rid; + const roomData = subscription._room; + let roomName = roomData.name ? roomData.name : roomId; + let userId = null; + + if (subscription.t === 'd') { + userId = roomId.replace(exportUserId, ''); + const userData = RocketChat.models.Users.findOneById(userId); + + if (userData) { + roomName = userData.name; + } + } + + const fileName = exportOperation.fullExport ? roomId : roomName; + const fileType = exportOperation.fullExport ? 'json' : 'html'; + const targetFile = `${ fileName }.${ fileType }`; + + exportOperation.roomList.push({ + roomId, + roomName, + userId, + exportedCount: 0, + status: 'pending', + targetFile, + type: subscription.t + }); + }); + + if (exportOperation.fullExport) { + exportOperation.status = 'exporting-rooms'; + } else { + exportOperation.status = 'exporting'; + } +}; + +const getAttachmentData = function(attachment) { + const attachmentData = { + type : attachment.type, + title: attachment.title, + title_link: attachment.title_link, + image_url: attachment.image_url, + audio_url: attachment.audio_url, + video_url: attachment.video_url, + message_link: attachment.message_link, + image_type: attachment.image_type, + image_size: attachment.image_size, + video_size: attachment.video_size, + video_type: attachment.video_type, + audio_size: attachment.audio_size, + audio_type: attachment.audio_type, + url: null, + remote: false, + fileId: null, + fileName: null + }; + + const url = attachment.title_link || attachment.image_url || attachment.audio_url || attachment.video_url || attachment.message_link; + if (url) { + attachmentData.url = url; + + const urlMatch = /\:\/\//.exec(url); + if (urlMatch && urlMatch.length > 0) { + attachmentData.remote = true; + } else { + const match = /^\/([^\/]+)\/([^\/]+)\/(.*)/.exec(url); + + if (match && match[2]) { + const file = RocketChat.models.Uploads.findOneById(match[2]); + + if (file) { + attachmentData.fileId = file._id; + attachmentData.fileName = file.name; + } + } + } + } + + return attachmentData; +}; + +const addToFileList = function(exportOperation, attachment) { + const targetFile = path.join(exportOperation.assetsPath, `${ attachment.fileId }-${ attachment.fileName }`); + + const attachmentData = { + url: attachment.url, + copied: false, + remote: attachment.remote, + fileId: attachment.fileId, + fileName: attachment.fileName, + targetFile + }; + + exportOperation.fileList.push(attachmentData); +}; + +const getMessageData = function(msg, exportOperation) { + const attachments = []; + + if (msg.attachments) { + msg.attachments.forEach((attachment) => { + const attachmentData = getAttachmentData(attachment); + + attachments.push(attachmentData); + addToFileList(exportOperation, attachmentData); + }); + } + + const messageObject = { + msg: msg.msg, + username: msg.u.username, + ts: msg.ts + }; + + if (attachments && attachments.length > 0) { + messageObject.attachments = attachments; + } + if (msg.t) { + messageObject.type = msg.t; + } + if (msg.u.name) { + messageObject.name = msg.u.name; + } + + return messageObject; +}; + +const copyFile = function(exportOperation, attachmentData) { + if (attachmentData.copied || attachmentData.remote || !attachmentData.fileId) { + attachmentData.copied = true; + return; + } + + const file = RocketChat.models.Uploads.findOneById(attachmentData.fileId); + + if (file) { + if (FileUpload.copy(file, attachmentData.targetFile)) { + attachmentData.copied = true; + } + } +}; + +const continueExportingRoom = function(exportOperation, exportOpRoomData) { + createDir(exportOperation.exportPath); + createDir(exportOperation.assetsPath); + + const filePath = path.join(exportOperation.exportPath, exportOpRoomData.targetFile); + + if (exportOpRoomData.status === 'pending') { + exportOpRoomData.status = 'exporting'; + startFile(filePath, ''); + if (!exportOperation.fullExport) { + writeToFile(filePath, ''); + } + } + + let limit = 100; + if (RocketChat.settings.get('UserData_MessageLimitPerRequest') > 0) { + limit = RocketChat.settings.get('UserData_MessageLimitPerRequest'); + } + + const skip = exportOpRoomData.exportedCount; + + const cursor = RocketChat.models.Messages.findByRoomId(exportOpRoomData.roomId, { limit, skip }); + const count = cursor.count(); + + cursor.forEach((msg) => { + const messageObject = getMessageData(msg, exportOperation); + + if (exportOperation.fullExport) { + const messageString = JSON.stringify(messageObject); + writeToFile(filePath, `${ messageString }\n`); + } else { + const messageType = msg.t; + const userName = msg.u.username || msg.u.name; + const timestamp = msg.ts ? new Date(msg.ts).toUTCString() : ''; + let message = msg.msg; + + switch (messageType) { + case 'uj': + message = TAPi18n.__('User_joined_channel'); + break; + case 'ul': + message = TAPi18n.__('User_left'); + break; + case 'au': + message = TAPi18n.__('User_added_by', {user_added : msg.msg, user_by : msg.u.username }); + break; + case 'r': + message = TAPi18n.__('Room_name_changed', { room_name: msg.msg, user_by: msg.u.username }); + break; + case 'ru': + message = TAPi18n.__('User_removed_by', {user_removed : msg.msg, user_by : msg.u.username }); + break; + case 'wm': + message = TAPi18n.__('Welcome', {user: msg.u.username }); + break; + case 'livechat-close': + message = TAPi18n.__('Conversation_finished'); + break; + } + + if (message !== msg.msg) { + message = `${ message }`; + } + + writeToFile(filePath, `

${ userName } (${ timestamp }):
`); + writeToFile(filePath, message); + + if (messageObject.attachments && messageObject.attachments.length > 0) { + messageObject.attachments.forEach((attachment) => { + if (attachment.type === 'file') { + const description = attachment.description || attachment.title || TAPi18n.__('Message_Attachments'); + + const assetUrl = `./assets/${ attachment.fileId }-${ attachment.fileName }`; + const link = `
${ description }`; + writeToFile(filePath, link); + } + }); + } + + writeToFile(filePath, '

'); + } + + exportOpRoomData.exportedCount++; + }); + + if (count <= exportOpRoomData.exportedCount) { + exportOpRoomData.status = 'completed'; + return true; + } + + return false; +}; + +const isExportComplete = function(exportOperation) { + const incomplete = exportOperation.roomList.some((exportOpRoomData) => { + return exportOpRoomData.status !== 'completed'; + }); + + return !incomplete; +}; + +const isDownloadFinished = function(exportOperation) { + const anyDownloadPending = exportOperation.fileList.some((fileData) => { + return !fileData.copied && !fileData.remote; + }); + + return !anyDownloadPending; +}; + +const sendEmail = function(userId) { + const lastFile = RocketChat.models.UserDataFiles.findLastFileByUser(userId); + if (lastFile) { + const userData = RocketChat.models.Users.findOneById(userId); + + if (userData && userData.emails && userData.emails[0] && userData.emails[0].address) { + const emailAddress = `${ userData.name } <${ userData.emails[0].address }>`; + const fromAddress = RocketChat.settings.get('From_Email'); + const subject = TAPi18n.__('UserDataDownload_EmailSubject'); + + const download_link = lastFile.url; + const body = TAPi18n.__('UserDataDownload_EmailBody', { download_link }); + + const rfcMailPatternWithName = /^(?:.*<)?([a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*)(?:>?)$/; + + if (rfcMailPatternWithName.test(emailAddress)) { + Meteor.defer(function() { + return Email.send({ + to: emailAddress, + from: fromAddress, + subject, + html: body + }); + }); + + return console.log(`Sending email to ${ emailAddress }`); + } + } + } +}; + +const makeZipFile = function(exportOperation) { + const targetFile = path.join(zipFolder, `${ exportOperation.userId }.zip`); + const output = fs.createWriteStream(targetFile); + + exportOperation.generatedFile = targetFile; + + const archive = archiver('zip'); + + output.on('close', () => { + }); + + archive.on('error', (err) => { + throw err; + }); + + archive.pipe(output); + archive.directory(exportOperation.exportPath, false); + archive.finalize(); +}; + +const uploadZipFile = function(exportOperation, callback) { + const userDataStore = FileUpload.getStore('UserDataFiles'); + const filePath = exportOperation.generatedFile; + + const stat = Meteor.wrapAsync(fs.stat)(filePath); + const stream = fs.createReadStream(filePath); + + const contentType = 'application/zip'; + const size = stat.size; + + const userId = exportOperation.userId; + const user = RocketChat.models.Users.findOneById(userId); + const userDisplayName = user ? user.name : userId; + const utcDate = new Date().toISOString().split('T')[0]; + + const newFileName = encodeURIComponent(`${ utcDate }-${ userDisplayName }.zip`); + + const details = { + userId, + type: contentType, + size, + name: newFileName + }; + + userDataStore.insert(details, stream, (err) => { + if (err) { + throw new Meteor.Error('invalid-file', 'Invalid Zip File', { method: 'cronProcessDownloads.uploadZipFile' }); + } else { + callback(); + } + }); +}; + +const generateChannelsFile = function(exportOperation) { + if (exportOperation.fullExport) { + const fileName = path.join(exportOperation.exportPath, 'channels.json'); + startFile(fileName, ''); + + exportOperation.roomList.forEach((roomData) => { + const newRoomData = { + roomId: roomData.roomId, + roomName: roomData.roomName, + type: roomData.type + }; + + const messageString = JSON.stringify(newRoomData); + writeToFile(fileName, `${ messageString }\n`); + }); + } + + exportOperation.status = 'exporting'; +}; + +const continueExportOperation = function(exportOperation) { + if (exportOperation.status === 'completed') { + return; + } + + if (!exportOperation.roomList) { + loadUserSubscriptions(exportOperation); + } + + try { + + if (exportOperation.status === 'exporting-rooms') { + generateChannelsFile(exportOperation); + } + + //Run every room on every request, to avoid missing new messages on the rooms that finished first. + if (exportOperation.status === 'exporting') { + exportOperation.roomList.forEach((exportOpRoomData) => { + continueExportingRoom(exportOperation, exportOpRoomData); + }); + + if (isExportComplete(exportOperation)) { + exportOperation.status = 'downloading'; + return; + } + } + + if (exportOperation.status === 'downloading') { + exportOperation.fileList.forEach((attachmentData) => { + copyFile(exportOperation, attachmentData); + }); + + if (isDownloadFinished(exportOperation)) { + exportOperation.status = 'compressing'; + return; + } + } + + if (exportOperation.status === 'compressing') { + makeZipFile(exportOperation); + exportOperation.status = 'uploading'; + return; + } + + if (exportOperation.status === 'uploading') { + uploadZipFile(exportOperation, () => { + exportOperation.status = 'completed'; + RocketChat.models.ExportOperations.updateOperation(exportOperation); + }); + return; + } + } catch (e) { + console.error(e); + } +}; + +function processDataDownloads() { + const cursor = RocketChat.models.ExportOperations.findAllPending({limit: 1}); + cursor.forEach((exportOperation) => { + if (exportOperation.status === 'completed') { + return; + } + + continueExportOperation(exportOperation); + RocketChat.models.ExportOperations.updateOperation(exportOperation); + + if (exportOperation.status === 'completed') { + sendEmail(exportOperation.userId); + } + }); +} + +Meteor.startup(function() { + Meteor.defer(function() { + processDataDownloads(); + + SyncedCron.add({ + name: 'Generate download files for user data', + schedule: (parser) => parser.cron(`*/${ processingFrequency } * * * *`), + job: processDataDownloads + }); + }); +}); diff --git a/packages/rocketchat-user-data-download/server/startup/settings.js b/packages/rocketchat-user-data-download/server/startup/settings.js new file mode 100644 index 0000000000000..3f4cc24ba157c --- /dev/null +++ b/packages/rocketchat-user-data-download/server/startup/settings.js @@ -0,0 +1,34 @@ +RocketChat.settings.addGroup('UserDataDownload', function() { + + this.add('UserData_EnableDownload', true, { + type: 'boolean', + public: true, + i18nLabel: 'UserData_EnableDownload' + }); + + this.add('UserData_FileSystemPath', '', { + type: 'string', + public: true, + i18nLabel: 'UserData_FileSystemPath' + }); + + this.add('UserData_FileSystemZipPath', '', { + type: 'string', + public: true, + i18nLabel: 'UserData_FileSystemZipPath' + }); + + this.add('UserData_ProcessingFrequency', 15, { + type: 'int', + public: true, + i18nLabel: 'UserData_ProcessingFrequency' + }); + + this.add('UserData_MessageLimitPerRequest', 100, { + type: 'int', + public: true, + i18nLabel: 'UserData_MessageLimitPerRequest' + }); + + +}); diff --git a/server/methods/requestDataDownload.js b/server/methods/requestDataDownload.js new file mode 100644 index 0000000000000..13f7aeaf224d7 --- /dev/null +++ b/server/methods/requestDataDownload.js @@ -0,0 +1,63 @@ +import fs from 'fs'; +import path from 'path'; + +let tempFolder = '/tmp/userData'; +if (RocketChat.settings.get('UserData_FileSystemPath') != null) { + if (RocketChat.settings.get('UserData_FileSystemPath').trim() !== '') { + tempFolder = RocketChat.settings.get('UserData_FileSystemPath'); + } +} + +Meteor.methods({ + requestDataDownload({fullExport = false}) { + const currentUserData = Meteor.user(); + const userId = currentUserData._id; + + const lastOperation = RocketChat.models.ExportOperations.findLastOperationByUser(userId, fullExport); + + if (lastOperation) { + const yesterday = new Date(); + yesterday.setUTCDate(yesterday.getUTCDate() - 1); + + if (lastOperation.createdAt > yesterday) { + return { + requested: false, + exportOperation: lastOperation + }; + } + } + + const subFolderName = fullExport ? 'full' : 'partial'; + const baseFolder = path.join(tempFolder, userId); + if (!fs.existsSync(baseFolder)) { + fs.mkdirSync(baseFolder); + } + + const folderName = path.join(baseFolder, subFolderName); + if (!fs.existsSync(folderName)) { + fs.mkdirSync(folderName); + } + const assetsFolder = path.join(folderName, 'assets'); + if (!fs.existsSync(assetsFolder)) { + fs.mkdirSync(assetsFolder); + } + + const exportOperation = { + userId : currentUserData._id, + roomList: null, + status: 'pending', + exportPath: folderName, + assetsPath: assetsFolder, + fileList: [], + generatedFile: null, + fullExport + }; + + RocketChat.models.ExportOperations.create(exportOperation); + + return { + requested: true, + exportOperation + }; + } +}); diff --git a/server/startup/migrations/v110.js b/server/startup/migrations/v110.js index 5b64b680d4ce0..bf01bdc8667b8 100644 --- a/server/startup/migrations/v110.js +++ b/server/startup/migrations/v110.js @@ -18,12 +18,10 @@ RocketChat.Migrations.add({ } if (RocketChat.models.Users) { - RocketChat.models.Users.find({ 'settings.preferences.viewMode': { $exists: 1 } }).forEach(function(user) { - RocketChat.models.Users.update( - { _id: user._id }, - { $rename: { 'settings.preferences.viewMode': 'user.settings.preferences.messageViewMode' } }, - ); - }); + RocketChat.models.Users.update( + { 'settings.preferences.viewMode': { $exists: 1 } }, + { $rename: { 'settings.preferences.viewMode': 'user.settings.preferences.messageViewMode' } }, + ); } } }, @@ -45,12 +43,10 @@ RocketChat.Migrations.add({ } if (RocketChat.models.Users) { - RocketChat.models.Users.find({ 'settings.preferences.messageViewMode': { $exists: 1 } }).forEach(function(user) { - RocketChat.models.Users.update( - { _id: user._id }, - { $rename: { 'settings.preferences.messageViewMode': 'user.settings.preferences.viewMode' } }, - ); - }); + RocketChat.models.Users.update( + { 'settings.preferences.messageViewMode': { $exists: 1 } }, + { $rename: { 'settings.preferences.messageViewMode': 'user.settings.preferences.viewMode' } }, + ); } } }