diff --git a/.meteor/packages b/.meteor/packages
index 951eadb8980c3..3ba1459162262 100644
--- a/.meteor/packages
+++ b/.meteor/packages
@@ -146,6 +146,7 @@ rocketchat:ui-master
rocketchat:ui-message
rocketchat:ui-sidenav
rocketchat:ui-vrecord
+rocketchat:user-data-download
rocketchat:version
rocketchat:videobridge
rocketchat:webrtc
diff --git a/.meteor/versions b/.meteor/versions
index 3fa982c52f2d7..4cd41c05e9db3 100644
--- a/.meteor/versions
+++ b/.meteor/versions
@@ -235,6 +235,7 @@ rocketchat:ui-master@0.1.0
rocketchat:ui-message@0.1.0
rocketchat:ui-sidenav@0.1.0
rocketchat:ui-vrecord@0.0.1
+rocketchat:user-data-download@1.0.0
rocketchat:version@1.0.0
rocketchat:version-check@0.0.1
rocketchat:videobridge@0.2.0
diff --git a/package.json b/package.json
index e88d1abbd7eff..76870049621ab 100644
--- a/package.json
+++ b/package.json
@@ -113,6 +113,7 @@
"@google-cloud/storage": "^1.6.0",
"@google-cloud/vision": "^0.15.2",
"adm-zip": "^0.4.7",
+ "archiver": "^2.1.1",
"atlassian-crowd": "^0.5.0",
"autolinker": "^1.6.2",
"aws-sdk": "^2.199.0",
diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js
index 73474ab5181f3..8b6008b3fa5bd 100644
--- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js
+++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js
@@ -17,7 +17,6 @@ new UploadFS.Store({
})
});
-
fileUploadHandler = (directive, meta, file) => {
const store = UploadFS.getStore(directive);
diff --git a/packages/rocketchat-file-upload/lib/FileUploadBase.js b/packages/rocketchat-file-upload/lib/FileUploadBase.js
index 3e2b166547f59..97fd9ced52731 100644
--- a/packages/rocketchat-file-upload/lib/FileUploadBase.js
+++ b/packages/rocketchat-file-upload/lib/FileUploadBase.js
@@ -4,7 +4,21 @@ import _ from 'underscore';
UploadFS.config.defaultStorePermissions = new UploadFS.StorePermissions({
insert(userId, doc) {
- return userId || (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0); // allow inserts from slackbridge (message_id = slack-timestamp-milli)
+ if (userId) {
+ return true;
+ }
+
+ // allow inserts from slackbridge (message_id = slack-timestamp-milli)
+ if (doc && doc.message_id && doc.message_id.indexOf('slack-') === 0) {
+ return true;
+ }
+
+ // allow inserts to the UserDataFiles store
+ if (doc && doc.store && doc.store.split(':').pop() === 'UserDataFiles') {
+ return true;
+ }
+
+ return false;
},
update(userId, doc) {
return RocketChat.authz.hasPermission(Meteor.userId(), 'delete-message', doc.rid) || (RocketChat.settings.get('Message_AllowDeleting') && userId === doc.userId);
diff --git a/packages/rocketchat-file-upload/server/config/AmazonS3.js b/packages/rocketchat-file-upload/server/config/AmazonS3.js
index f633c386e21e5..571d592465100 100644
--- a/packages/rocketchat-file-upload/server/config/AmazonS3.js
+++ b/packages/rocketchat-file-upload/server/config/AmazonS3.js
@@ -25,15 +25,35 @@ const get = function(file, req, res) {
}
};
+const copy = function(file, out) {
+ const fileUrl = this.store.getRedirectURL(file);
+
+ if (fileUrl) {
+ const request = /^https:/.test(fileUrl) ? https : http;
+ request.get(fileUrl, fileRes => fileRes.pipe(out));
+ } else {
+ out.end();
+ }
+};
+
const AmazonS3Uploads = new FileUploadClass({
name: 'AmazonS3:Uploads',
- get
+ get,
+ copy
// store setted bellow
});
const AmazonS3Avatars = new FileUploadClass({
name: 'AmazonS3:Avatars',
- get
+ get,
+ copy
+ // store setted bellow
+});
+
+const AmazonS3UserDataFiles = new FileUploadClass({
+ name: 'AmazonS3:UserDataFiles',
+ get,
+ copy
// store setted bellow
});
@@ -74,6 +94,7 @@ const configure = _.debounce(function() {
AmazonS3Uploads.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Uploads.name, config);
AmazonS3Avatars.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Avatars.name, config);
+ AmazonS3UserDataFiles.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3UserDataFiles.name, config);
}, 500);
RocketChat.settings.get(/^FileUpload_S3_/, configure);
diff --git a/packages/rocketchat-file-upload/server/config/FileSystem.js b/packages/rocketchat-file-upload/server/config/FileSystem.js
index 096df87e4dee0..fab473b5ab6cf 100644
--- a/packages/rocketchat-file-upload/server/config/FileSystem.js
+++ b/packages/rocketchat-file-upload/server/config/FileSystem.js
@@ -28,6 +28,22 @@ const FileSystemUploads = new FileUploadClass({
res.end();
return;
}
+ },
+
+ copy(file, out) {
+ const filePath = this.store.getFilePath(file._id, file);
+ try {
+ const stat = Meteor.wrapAsync(fs.stat)(filePath);
+
+ if (stat && stat.isFile()) {
+ file = FileUpload.addExtensionTo(file);
+
+ this.store.getReadStream(file._id, file).pipe(out);
+ }
+ } catch (e) {
+ out.end();
+ return;
+ }
}
});
@@ -54,6 +70,31 @@ const FileSystemAvatars = new FileUploadClass({
}
});
+const FileSystemUserDataFiles = new FileUploadClass({
+ name: 'FileSystem:UserDataFiles',
+
+ get(file, req, res) {
+ const filePath = this.store.getFilePath(file._id, file);
+
+ try {
+ const stat = Meteor.wrapAsync(fs.stat)(filePath);
+
+ if (stat && stat.isFile()) {
+ file = FileUpload.addExtensionTo(file);
+ res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`);
+ res.setHeader('Last-Modified', file.uploadedAt.toUTCString());
+ res.setHeader('Content-Type', file.type);
+ res.setHeader('Content-Length', file.size);
+
+ this.store.getReadStream(file._id, file).pipe(res);
+ }
+ } catch (e) {
+ res.writeHead(404);
+ res.end();
+ return;
+ }
+ }
+});
const createFileSystemStore = _.debounce(function() {
const options = {
@@ -62,6 +103,7 @@ const createFileSystemStore = _.debounce(function() {
FileSystemUploads.store = FileUpload.configureUploadsStore('Local', FileSystemUploads.name, options);
FileSystemAvatars.store = FileUpload.configureUploadsStore('Local', FileSystemAvatars.name, options);
+ FileSystemUserDataFiles.store = FileUpload.configureUploadsStore('Local', FileSystemUserDataFiles.name, options);
// DEPRECATED backwards compatibililty (remove)
UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name];
diff --git a/packages/rocketchat-file-upload/server/config/GoogleStorage.js b/packages/rocketchat-file-upload/server/config/GoogleStorage.js
index fdcebdc352b48..67ecb74b4f192 100644
--- a/packages/rocketchat-file-upload/server/config/GoogleStorage.js
+++ b/packages/rocketchat-file-upload/server/config/GoogleStorage.js
@@ -29,15 +29,39 @@ const get = function(file, req, res) {
});
};
+const copy = function(file, out) {
+ this.store.getRedirectURL(file, (err, fileUrl) => {
+ if (err) {
+ console.error(err);
+ }
+
+ if (fileUrl) {
+ const request = /^https:/.test(fileUrl) ? https : http;
+ request.get(fileUrl, fileRes => fileRes.pipe(out));
+ } else {
+ out.end();
+ }
+ });
+};
+
const GoogleCloudStorageUploads = new FileUploadClass({
name: 'GoogleCloudStorage:Uploads',
- get
+ get,
+ copy
// store setted bellow
});
const GoogleCloudStorageAvatars = new FileUploadClass({
name: 'GoogleCloudStorage:Avatars',
- get
+ get,
+ copy
+ // store setted bellow
+});
+
+const GoogleCloudStorageUserDataFiles = new FileUploadClass({
+ name: 'GoogleCloudStorage:UserDataFiles',
+ get,
+ copy
// store setted bellow
});
@@ -64,6 +88,7 @@ const configure = _.debounce(function() {
GoogleCloudStorageUploads.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUploads.name, config);
GoogleCloudStorageAvatars.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageAvatars.name, config);
+ GoogleCloudStorageUserDataFiles.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUserDataFiles.name, config);
}, 500);
RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure);
diff --git a/packages/rocketchat-file-upload/server/config/GridFS.js b/packages/rocketchat-file-upload/server/config/GridFS.js
index 54bb40712cacd..0ad63b5e86264 100644
--- a/packages/rocketchat-file-upload/server/config/GridFS.js
+++ b/packages/rocketchat-file-upload/server/config/GridFS.js
@@ -62,7 +62,6 @@ const getByteRange = function(header) {
return null;
};
-
// code from: https://github.com/jalik/jalik-ufs/blob/master/ufs-server.js#L310
const readFromGridFS = function(storeName, fileId, file, req, res) {
const store = UploadFS.getStore(storeName);
@@ -123,10 +122,26 @@ const readFromGridFS = function(storeName, fileId, file, req, res) {
}
};
+const copyFromGridFS = function(storeName, fileId, file, out) {
+ const store = UploadFS.getStore(storeName);
+ const rs = store.getReadStream(fileId, file);
+
+ [rs, out].forEach(stream => stream.on('error', function(err) {
+ store.onReadError.call(store, err, fileId, file);
+ out.end();
+ }));
+
+ rs.pipe(out);
+};
+
FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', {
collectionName: 'rocketchat_uploads'
});
+FileUpload.configureUploadsStore('GridFS', 'GridFS:UserDataFiles', {
+ collectionName: 'rocketchat_userDataFiles'
+});
+
// DEPRECATED: backwards compatibility (remove)
UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads'];
@@ -147,6 +162,29 @@ new FileUploadClass({
res.setHeader('Content-Length', file.size);
return readFromGridFS(file.store, file._id, file, req, res);
+ },
+
+ copy(file, out) {
+ copyFromGridFS(file.store, file._id, file, out);
+ }
+});
+
+new FileUploadClass({
+ name: 'GridFS:UserDataFiles',
+
+ get(file, req, res) {
+ file = FileUpload.addExtensionTo(file);
+
+ res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${ encodeURIComponent(file.name) }`);
+ res.setHeader('Last-Modified', file.uploadedAt.toUTCString());
+ res.setHeader('Content-Type', file.type);
+ res.setHeader('Content-Length', file.size);
+
+ return readFromGridFS(file.store, file._id, file, req, res);
+ },
+
+ copy(file, out) {
+ copyFromGridFS(file.store, file._id, file, out);
}
});
diff --git a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js
index d2ba5505f2d4d..8855eb76fca8a 100644
--- a/packages/rocketchat-file-upload/server/config/_configUploadStorage.js
+++ b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js
@@ -14,6 +14,7 @@ const configStore = _.debounce(() => {
console.log('Setting default file store to', store);
UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`);
UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`);
+ UploadFS.getStores().UserDataFiles = UploadFS.getStore(`${ store }:UserDataFiles`);
}
}, 1000);
diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js
index a056e991fba5e..3194dc8999a82 100644
--- a/packages/rocketchat-file-upload/server/lib/FileUpload.js
+++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js
@@ -58,6 +58,25 @@ Object.assign(FileUpload, {
};
},
+ defaultUserDataFiles() {
+ return {
+ collection: RocketChat.models.UserDataFiles.model,
+ getPath(file) {
+ return `${ RocketChat.settings.get('uniqueID') }/uploads/userData/${ file.userId }`;
+ },
+ onValidate: FileUpload.uploadsOnValidate,
+ onRead(fileId, file, req, res) {
+ if (!FileUpload.requestCanAccessFiles(req)) {
+ res.writeHead(403);
+ return false;
+ }
+
+ res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`);
+ return true;
+ }
+ };
+ },
+
avatarsOnValidate(file) {
if (RocketChat.settings.get('Accounts_AvatarResize') !== true) {
return;
@@ -229,16 +248,30 @@ Object.assign(FileUpload, {
}
res.writeHead(404);
res.end();
+ },
+
+ copy(file, targetFile) {
+ const store = this.getStoreByName(file.store);
+ const out = fs.createWriteStream(targetFile);
+
+ file = FileUpload.addExtensionTo(file);
+
+ if (store.copy) {
+ store.copy(file, out);
+ return true;
+ }
+
+ return false;
}
});
-
export class FileUploadClass {
- constructor({ name, model, store, get, insert, getStore }) {
+ constructor({ name, model, store, get, insert, getStore, copy }) {
this.name = name;
this.model = model || this.getModelFromName();
this._store = store || UploadFS.getStore(name);
this.get = get;
+ this.copy = copy;
if (insert) {
this.insert = insert;
diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json
index c6ddc5191eec6..9c2c3ca6cb30f 100644
--- a/packages/rocketchat-i18n/i18n/en.i18n.json
+++ b/packages/rocketchat-i18n/i18n/en.i18n.json
@@ -567,6 +567,7 @@
"Domain_removed": "Domain Removed",
"Domains": "Domains",
"Domains_allowed_to_embed_the_livechat_widget": "Comma-separated list of domains allowed to embed the livechat widget. Leave blank to allow all domains.",
+ "Download_My_Data" : "Download My Data",
"Download_Snippet": "Download",
"Drop_to_upload_file": "Drop to upload file",
"Dry_run": "Dry run",
@@ -724,6 +725,7 @@
"Example_s": "Example: %s",
"Exclude_Botnames": "Exclude Bots",
"Exclude_Botnames_Description": "Do not propagate messages from bots whose name matches the regular expression above. If left empty, all messages from bots will be propagated.",
+ "Export_My_Data" : "Export My Data",
"External_Service": "External Service",
"External_Queue_Service_URL": "External Queue Service URL",
"Facebook_Page": "Facebook Page",
@@ -2106,6 +2108,18 @@
"User_uploaded_file": "Uploaded a file",
"User_uploaded_image": "Uploaded an image",
"User_Presence": "User Presence",
+ "UserDataDownload" : "User Data Download",
+ "UserData_EnableDownload" : "Enable User Data Download",
+ "UserData_FileSystemPath" : "System Path (Exported Files)",
+ "UserData_FileSystemZipPath" : "System Path (Compressed File)",
+ "UserData_ProcessingFrequency" : "Processing Frequency (Minutes)",
+ "UserData_MessageLimitPerRequest" : "Message Limit per Request",
+ "UserDataDownload_EmailSubject" : "Your Data File is Ready to Download",
+ "UserDataDownload_EmailBody" : "Your data file is now ready to download. Click here to download it.",
+ "UserDataDownload_Requested" : "Download File Requested",
+ "UserDataDownload_Requested_Text" : "Your data file will be generated. A link to download it will be sent to your email address when ready.",
+ "UserDataDownload_RequestExisted_Text" : "Your data file is already being generated. A link to download it will be sent to your email address when ready.",
+ "UserDataDownload_CompletedRequestExisted_Text" : "Your data file was already generated. Check your email account for the download link.",
"Username": "Username",
"Username_and_message_must_not_be_empty": "Username and message must not be empty.",
"Username_cant_be_empty": "The username cannot be empty",
diff --git a/packages/rocketchat-lib/client/models/UserDataFiles.js b/packages/rocketchat-lib/client/models/UserDataFiles.js
new file mode 100644
index 0000000000000..80e7e22e162ed
--- /dev/null
+++ b/packages/rocketchat-lib/client/models/UserDataFiles.js
@@ -0,0 +1,6 @@
+RocketChat.models.UserDataFiles = new class extends RocketChat.models._Base {
+ constructor() {
+ super();
+ this._initModel('userDataFiles');
+ }
+};
diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js
index eddbc615d0e5c..d85c5b0842af0 100644
--- a/packages/rocketchat-lib/package.js
+++ b/packages/rocketchat-lib/package.js
@@ -126,6 +126,8 @@ Package.onUse(function(api) {
api.addFiles('server/models/Subscriptions.js', 'server');
api.addFiles('server/models/Uploads.js', 'server');
api.addFiles('server/models/Users.js', 'server');
+ api.addFiles('server/models/ExportOperations.js', 'server');
+ api.addFiles('server/models/UserDataFiles.js', 'server');
api.addFiles('server/oauth/oauth.js', 'server');
api.addFiles('server/oauth/facebook.js', 'server');
diff --git a/packages/rocketchat-lib/server/models/ExportOperations.js b/packages/rocketchat-lib/server/models/ExportOperations.js
new file mode 100644
index 0000000000000..d7beda9d94b92
--- /dev/null
+++ b/packages/rocketchat-lib/server/models/ExportOperations.js
@@ -0,0 +1,78 @@
+import _ from 'underscore';
+
+RocketChat.models.ExportOperations = new class ModelExportOperations extends RocketChat.models._Base {
+ constructor() {
+ super('export_operations');
+
+ this.tryEnsureIndex({ 'userId': 1 });
+ this.tryEnsureIndex({ 'status': 1 });
+ }
+
+ // FIND
+ findById(id) {
+ const query = {_id: id};
+
+ return this.find(query);
+ }
+
+ findLastOperationByUser(userId, fullExport = false, options = {}) {
+ const query = {
+ userId,
+ fullExport
+ };
+
+ options.sort = {'createdAt' : -1};
+ return this.findOne(query, options);
+ }
+
+ findPendingByUser(userId, options) {
+ const query = {
+ userId,
+ status: {
+ $nin: ['completed']
+ }
+ };
+
+ return this.find(query, options);
+ }
+
+ findAllPending(options) {
+ const query = {
+ status: { $nin: ['completed'] }
+ };
+
+ return this.find(query, options);
+ }
+
+ // UPDATE
+ updateOperation(data) {
+ const update = {
+ $set: {
+ roomList: data.roomList,
+ status: data.status,
+ fileList: data.fileList,
+ generatedFile: data.generatedFile
+ }
+ };
+
+ return this.update(data._id, update);
+ }
+
+
+ // INSERT
+ create(data) {
+ const exportOperation = {
+ createdAt: new Date
+ };
+
+ _.extend(exportOperation, data);
+
+ return this.insert(exportOperation);
+ }
+
+
+ // REMOVE
+ removeById(_id) {
+ return this.remove(_id);
+ }
+};
diff --git a/packages/rocketchat-lib/server/models/Messages.js b/packages/rocketchat-lib/server/models/Messages.js
index 2bb631520ddb4..4fe8b43864628 100644
--- a/packages/rocketchat-lib/server/models/Messages.js
+++ b/packages/rocketchat-lib/server/models/Messages.js
@@ -284,6 +284,14 @@ RocketChat.models.Messages = new class extends RocketChat.models._Base {
return this.findOne(query);
}
+ findByRoomId(roomId, options) {
+ const query = {
+ rid: roomId
+ };
+
+ return this.find(query, options);
+ }
+
getLastVisibleMessageSentWithNoTypeByRoomId(rid, messageId) {
const query = {
rid,
diff --git a/packages/rocketchat-lib/server/models/UserDataFiles.js b/packages/rocketchat-lib/server/models/UserDataFiles.js
new file mode 100644
index 0000000000000..de9efbb5e883c
--- /dev/null
+++ b/packages/rocketchat-lib/server/models/UserDataFiles.js
@@ -0,0 +1,40 @@
+import _ from 'underscore';
+
+RocketChat.models.UserDataFiles = new class ModelUserDataFiles extends RocketChat.models._Base {
+ constructor() {
+ super('user_data_files');
+
+ this.tryEnsureIndex({ 'userId': 1 });
+ }
+
+ // FIND
+ findById(id) {
+ const query = {_id: id};
+ return this.find(query);
+ }
+
+ findLastFileByUser(userId, options = {}) {
+ const query = {
+ userId
+ };
+
+ options.sort = {'_updatedAt' : -1};
+ return this.findOne(query, options);
+ }
+
+ // INSERT
+ create(data) {
+ const userDataFile = {
+ createdAt: new Date
+ };
+
+ _.extend(userDataFile, data);
+
+ return this.insert(userDataFile);
+ }
+
+ // REMOVE
+ removeById(_id) {
+ return this.remove(_id);
+ }
+};
diff --git a/packages/rocketchat-lib/server/models/Users.js b/packages/rocketchat-lib/server/models/Users.js
index 9217dad7dc8f7..db9982b25ca0a 100644
--- a/packages/rocketchat-lib/server/models/Users.js
+++ b/packages/rocketchat-lib/server/models/Users.js
@@ -51,6 +51,11 @@ class ModelUsers extends RocketChat.models._Base {
return this.findOne(query, options);
}
+ findOneById(userId) {
+ const query = {_id: userId};
+
+ return this.findOne(query);
+ }
// FIND
findById(userId) {
diff --git a/packages/rocketchat-ui-account/client/accountPreferences.html b/packages/rocketchat-ui-account/client/accountPreferences.html
index d7d50eedf8d2d..501b28ccd0278 100644
--- a/packages/rocketchat-ui-account/client/accountPreferences.html
+++ b/packages/rocketchat-ui-account/client/accountPreferences.html
@@ -296,6 +296,19 @@
${ userName } (${ timestamp }):
`);
+ writeToFile(filePath, message);
+
+ if (messageObject.attachments && messageObject.attachments.length > 0) {
+ messageObject.attachments.forEach((attachment) => {
+ if (attachment.type === 'file') {
+ const description = attachment.description || attachment.title || TAPi18n.__('Message_Attachments');
+
+ const assetUrl = `./assets/${ attachment.fileId }-${ attachment.fileName }`;
+ const link = `
${ description }`;
+ writeToFile(filePath, link);
+ }
+ });
+ }
+
+ writeToFile(filePath, '