From 306eb95605f888548afd1e8623a7bf0ca76c5549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=91=ED=9B=88?= Date: Mon, 16 Nov 2015 19:31:33 +0900 Subject: [PATCH 1/8] refactring auth-manager.js - introduced redis in package.json - refactoring in auth-manager.js - removed deprecated authMgr.verifyToken - replaced with ensureLogin - renamed verifySesssion to getUserInfoByToken --- src/server/app/lib/app-manager.js | 22 +-- src/server/app/test/test-app.js | 2 +- src/server/build/lib/build-manager.js | 12 +- src/server/buildjm/lib/emul.js | 2 +- src/server/common/auth-manager.js | 170 +++++++----------- src/server/conf/default-conf.js | 13 ++ .../incubator-ripple/lib/server/emulate.js | 2 +- src/server/fs/lib/console-manager.js | 2 +- src/server/fs/lib/fs-manager.js | 59 +++--- src/server/fs/test/test-fs.js | 2 +- src/server/mon/lib/pf-manager.js | 16 +- src/server/notify/lib/conn-msg.js | 2 +- src/server/package.json | 4 +- 13 files changed, 140 insertions(+), 168 deletions(-) diff --git a/src/server/app/lib/app-manager.js b/src/server/app/lib/app-manager.js index a774238..6bfe447 100644 --- a/src/server/app/lib/app-manager.js +++ b/src/server/app/lib/app-manager.js @@ -1355,7 +1355,7 @@ exports.deployFromWebidaFS = deployFromWebidaFS; // App APIs router.get('/webida/api/app/appinfo', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:getAppInfo', rsc:'app:*'}, res, next); }, @@ -1386,7 +1386,7 @@ router.get('/webida/api/app/appinfo', ); router.get('/webida/api/app/allapps', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res) { getAllAppInfos(APPINFO_PROJECTIONS, function (err, appInfos) { logger.debug('allapps', arguments); @@ -1401,7 +1401,7 @@ router.get('/webida/api/app/allapps', ); router.get('/webida/api/app/isValidDomain', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:isValidDomain', rsc:'app:*'}, res, next); }, @@ -1425,7 +1425,7 @@ router.get('/webida/api/app/isValidDomain', ); router.get('/webida/api/app/create', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:createApp', rsc:'app:*'}, res, next); }, @@ -1447,7 +1447,7 @@ router.get('/webida/api/app/create', ); router.get('/webida/api/app/delete', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:deleteApp', rsc:'app:*'}, res, next); }, @@ -1465,7 +1465,7 @@ router.get('/webida/api/app/delete', ); router.get('/webida/api/app/changeappinfo', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:setAppInfo', rsc:'app:*'}, res, next); }, @@ -1488,7 +1488,7 @@ router.get('/webida/api/app/changeappinfo', ); router.get('/webida/api/app/myapps', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:getMyAppInfo', rsc:'app:*'}, res, next); }, @@ -1505,7 +1505,7 @@ router.get('/webida/api/app/myapps', ); router.get('/webida/api/app/start', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:startApp', rsc:'app:*'}, res, next); }, @@ -1522,7 +1522,7 @@ router.get('/webida/api/app/start', ); router.get('/webida/api/app/stop', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:stopApp', rsc:'app:*'}, res, next); }, @@ -1541,7 +1541,7 @@ router.get('/webida/api/app/stop', ); router.get('/webida/api/app/deploy', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:deployApp', rsc:'app:*'}, res, next); }, @@ -1572,7 +1572,7 @@ router.get('/webida/api/app/deploy', //deploy package file router.post('/webida/api/app/deploy', - authMgr.verifyToken, + authMgr.ensureLogin, multipartMiddleware, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'app:deployPkg', rsc:'app:*'}, res, next); diff --git a/src/server/app/test/test-app.js b/src/server/app/test/test-app.js index 8013afa..29aca2e 100644 --- a/src/server/app/test/test-app.js +++ b/src/server/app/test/test-app.js @@ -33,7 +33,7 @@ var app; var appMgr; var isAdminValue = true; -authMgr.verifyToken = function(req, res, next) { +authMgr.ensureLogin = function(req, res, next) { req.user = {}; req.user.uid = account.uid; req.user.isAdmin = isAdminValue; diff --git a/src/server/build/lib/build-manager.js b/src/server/build/lib/build-manager.js index d605be9..baa6979 100644 --- a/src/server/build/lib/build-manager.js +++ b/src/server/build/lib/build-manager.js @@ -123,7 +123,7 @@ function invokeBuild(profileInfo, platformInfo, user, taskFunc, cb) { * build specific profile */ -router.post('/webida/api/build/build', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/build/build', authMgr.ensureLogin, function (req, res) { var profileInfo = JSON.parse(req.body.profileInfo); var platformInfo = JSON.parse(req.body.platformInfo); @@ -163,7 +163,7 @@ router.post('/webida/api/build/build', authMgr.verifyToken, function (req, res) }); -router.post('/webida/api/build/clean', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/build/clean', authMgr.ensureLogin, function (req, res) { logger.info('clean'); // have to check whether proj belongs to requester @@ -184,7 +184,7 @@ router.post('/webida/api/build/clean', authMgr.verifyToken, function (req, res) }); -router.post('/webida/api/build/rebuild', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/build/rebuild', authMgr.ensureLogin, function (req, res) { // have to check whether proj belongs to requester var profileInfo = JSON.parse(req.body.profileInfo); @@ -226,7 +226,7 @@ router.post('/webida/api/build/rebuild', authMgr.verifyToken, function (req, res }); -router.post('/webida/api/build/gcm/:regid', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/build/gcm/:regid', authMgr.ensureLogin, function (req, res) { var uid = req.user.uid; var regid = req.params.regid; var info = req.body.info; @@ -251,7 +251,7 @@ router.post('/webida/api/build/gcm/:regid', authMgr.verifyToken, function (req, }); -router.delete('/webida/api/build/gcm/:regid', authMgr.verifyToken, function(req, res) { +router.delete('/webida/api/build/gcm/:regid', authMgr.ensureLogin, function(req, res) { var uid = req.user.uid; var regid = req.params.regid; @@ -264,7 +264,7 @@ router.delete('/webida/api/build/gcm/:regid', authMgr.verifyToken, function(req, }); }); -router.get('/webida/api/build/gcm', authMgr.verifyToken, function(req, res) { +router.get('/webida/api/build/gcm', authMgr.ensureLogin, function(req, res) { var uid = req.user.uid; buildDb.getGcmInfo(uid, function (err, rs) { diff --git a/src/server/buildjm/lib/emul.js b/src/server/buildjm/lib/emul.js index 899b77d..9a278bf 100644 --- a/src/server/buildjm/lib/emul.js +++ b/src/server/buildjm/lib/emul.js @@ -51,5 +51,5 @@ var parseParam = function (req, res, next) { } module.exports.parseParam = parseParam; -module.exports.verifyToken = authMgr.verifyToken; +module.exports.ensureLogin = authMgr.ensureLogin; diff --git a/src/server/common/auth-manager.js b/src/server/common/auth-manager.js index 6f1ded9..26f4058 100644 --- a/src/server/common/auth-manager.js +++ b/src/server/common/auth-manager.js @@ -17,6 +17,7 @@ 'use strict'; var http = require('http'); +var util = require('util'); var utils = require('./utils'); var _ = require('lodash'); @@ -34,7 +35,7 @@ exports.init = function (db) { logger.info('auth-manager initialize. (' + db + ')'); }; -function requestTokenInfo(token, callback) { +function _requestTokenInfo(token, callback) { var template = _.template('/webida/api/oauth/verify?token=<%= token %>'); var options = { hostname: internalAccessInfo.auth.host, @@ -43,7 +44,7 @@ function requestTokenInfo(token, callback) { }; var req; - logger.info('req', options); + logger.debug('request token info', options); function handleResponse(err, res, body) { var tokenInfo; if (err) { return callback(err); } @@ -51,7 +52,7 @@ function requestTokenInfo(token, callback) { try { tokenInfo = JSON.parse(body).data; } catch (e) { - logger.error('Invalid verifyToken response:', arguments); + logger.error('Invalid oauth/verify response: ' + body, e); return callback(500); } return callback(0, tokenInfo); @@ -66,10 +67,10 @@ function requestTokenInfo(token, callback) { var data = ''; logger.info('res', res.statusCode); res.setEncoding('utf8'); - res.on('data', function (chunk) { - logger.info('data chunk', chunk); - data += chunk; - }); + //res.on('data', function (chunk) { + // logger.info('data chunk', chunk); + // data += chunk; + //}); res.on('end', function () { logger.info('end', data); handleResponse(null, res, data); @@ -81,7 +82,7 @@ function requestTokenInfo(token, callback) { req.end(); } -function checkExpired(info, callback) { +function _checkExpired(info, callback) { var current; var expire; if (!info) { @@ -94,7 +95,7 @@ function checkExpired(info, callback) { current = new Date().getTime(); expire = new Date(info.expireTime).getTime(); - logger.info('checkExpired', current, info, expire); + logger.info('_checkExpired', current, info, expire); if (expire - current < 0) { return callback(419); } else { @@ -107,108 +108,79 @@ function _verifyToken(token, callback) { logger.debug('token is null'); return callback(400); } - requestTokenInfo(token, function (err, info) { + _requestTokenInfo(token, function (err, info) { if (err) { - logger.debug('requestTokenInfo failed', err); + logger.debug('_requestTokenInfo failed', err); return callback(err); } - return checkExpired(info, callback); + return _checkExpired(info, callback); }); } -exports._verifyToken = _verifyToken; - -function getTokenVerifier(errHandler) { - var verifyToken = function (req, res, next) { - /* jshint camelcase: false */ - var token = req.headers.authorization || req.access_token || - req.query.access_token || req.parsedUrl.query.access_token; - if (!token) { - return errHandler(utils.err('TokenNotSpecified'), req, res, next); - } - logger.info('verifyToken', token); - _verifyToken(token, function (err, info) { - if (err) { - logger.info('_verifyToken failed', err); - console.error('_verifyToken failed', err); - errHandler(err, req, res, next); - } else { - req.user = info; - next(); - } - }); - }; - return verifyToken; +function _getTokenFromRequest(req) { + /* jshint camelcase: false */ + return req.headers.authorization || req.access_token || + req.query.access_token || req.parsedUrl.query.access_token; } -exports.getTokenVerifier = getTokenVerifier; + +// this function allows 'anonymous user' by default, but will not set req.user +// for the anonymous user function getUserInfo(req, res, next) { - getTokenVerifier(function errorHandler(err, req, res, next) { + var token = _getTokenFromRequest(req); + var allowAnonymous = req.disallowAnonymous ? false : true; + + logger.debug('checking token : ' + token); + _verifyToken(token, function (err, info) { + var errMsg = 'Internal server error'; if (err) { - if (err.name === 'TokenNotSpecified') { - return next(); - } else if (err === 419) { - return res.status(err).send(utils.fail('Access token is invalid or expired')); - } else { - return res.status(err).send(utils.fail('Internal server error')); + logger.debug('_verifyToken failed with ' + err); + if (err === 400 || err === 419) { + if (allowAnonymous) { + return next(); + } + errMsg = (err === 400) ? 'requires access token' : 'invalid access token'; } - } else { - return next(); + return res.status(err).send(utils.fail(errMsg)); } - })(req, res, next); + req.user = info; + return next(); + }); } -exports.getUserInfo = getUserInfo; +exports.getUserINfo = getUserInfo; + function ensureLogin(req, res, next) { - /* jshint unused:false */ - getTokenVerifier(function errorHandler(err, req, res/*, next*/) { - if (err.name === 'TokenNotSpecified') { - return res.status(400).send(utils.fail('Access token is required')); - } else if (err === 419) { - return res.status(err).send(utils.fail('Access token is invalid or expired')); - } else { - return res.status(err).send(utils.fail('Internal server error')); - } - //next(); - })(req, res, next); + req.disallowAnonymous = true; + return getUserInfo(req, res, next); } exports.ensureLogin = ensureLogin; -exports.verifyToken = ensureLogin; // deprecated -function ensureAdmin(req, res, next) { +exports.ensureAdmin = function ensureAdmin (req, res, next) { ensureLogin(req, res, function () { if (!req.user.isAdmin) { return res.status(400).send(utils.fail('Unauthorized Access')); } next(); }); -} -exports.ensureAdmin = ensureAdmin; +}; -function verifySession(token, callback) { +function _verifyToken(token, callback) { if (!token) { logger.debug('token is null'); return callback(400); } - - requestTokenInfo(token, function (err, info) { + _requestTokenInfo(token, function (err, info) { if (err) { - logger.debug('requestTokenInfo failed', err); + logger.debug('_requestTokenInfo failed', err); return callback(err); } - return checkExpired(info, callback); + return _checkExpired(info, callback); }); } -exports.verifySession = verifySession; +exports.getUserInfoByToken = _verifyToken; -function checkAuthorize(aclInfo, res, next) { - var template = _.template('/checkauthorize?uid=<%= uid %>&action=<%= action %>&rsc=<%= rsc %>'); - var options = { - hostname: internalAccessInfo.auth.host, - port: internalAccessInfo.auth.port, - path: encodeURI(template(aclInfo)) - }; - var cb = next; +function sendCheckAuthorizeRequest(aclInfo, options, res, next) { var req; logger.info('checkAuthorize', aclInfo); req = http.request(options, function (response) { @@ -220,7 +192,7 @@ function checkAuthorize(aclInfo, res, next) { }); response.on('end', function () { if (response.statusCode === 200) { - return cb(); + return next(); } else if (response.statusCode === 401) { return res.status(401).send(utils.fail('Not authorized.')); } else { @@ -228,18 +200,23 @@ function checkAuthorize(aclInfo, res, next) { } }); }); - req.on('error', function (e) { return res.send(500, utils.fail(e)); }); - req.end(); } +function checkAuthorize(aclInfo, res, next) { + var template = _.template('/checkauthorize?uid=<%= uid %>&action=<%= action %>&rsc=<%= rsc %>'); + var options = { + hostname: internalAccessInfo.auth.host, + port: internalAccessInfo.auth.port, + path: encodeURI(template(aclInfo)) + }; + sendCheckAuthorizeRequest(aclInfo, options, res, next); +} exports.checkAuthorize = checkAuthorize; function checkAuthorizeMulti(aclInfo, res, next) { - var req; - var cb = next; var template = _.template('/checkauthorizemulti' + '?uid=<%= uid %>' + '&action=<%= action %>' + @@ -250,31 +227,7 @@ function checkAuthorizeMulti(aclInfo, res, next) { port: internalAccessInfo.auth.port, path: encodeURI(template(aclInfo)) }; - logger.info('checkAuthorizeMulti', aclInfo); - - req = http.request(options, function (response) { - var data = ''; - response.setEncoding('utf8'); - response.on('data', function (chunk) { - logger.info('data chunk', chunk); - data += chunk; - }); - response.on('end', function () { - if (response.statusCode === 200) { - return cb(); - } else if (response.statusCode === 401) { - return res.send(401, utils.fail('Not authorized.')); - } else { - return res.send(500, utils.fail('Internal server error(checking authorization)')); - } - }); - }); - - req.on('error', function (e) { - return res.send(500, utils.fail(e)); - }); - - req.end(); + sendCheckAuthorizeRequest(aclInfo, options, res, next); } exports.checkAuthorizeMulti = checkAuthorizeMulti; @@ -524,6 +477,9 @@ function updatePolicyResource(oldPath, newPath, token, callback) { } exports.updatePolicyResource = updatePolicyResource; + +// TODO : move this function into userdb.js, or somewhere else +// why not use fs-manager, instead? function getFSInfo(fsid, token, callback) { var req; var path = '/webida/api/fs/' + fsid + '?access_token=' + token; @@ -538,7 +494,7 @@ function getFSInfo(fsid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('getFSInfo data chunk', chunk); + logger.debug('getFSInfo data chunk', chunk); data += chunk; }); response.on('end', function () { diff --git a/src/server/conf/default-conf.js b/src/server/conf/default-conf.js index 662bde6..497401e 100755 --- a/src/server/conf/default-conf.js +++ b/src/server/conf/default-conf.js @@ -185,6 +185,19 @@ var conf = { connHostUrl: serviceInstances.conn[0].url, monHostUrl: serviceInstances.mon[0].url, + cache: { + // conf.cache.redis object will be passed to redis client library directly. don't check with jshint. + /* jshint ignore:start */ + redis : { + host: '127.0.0.1', + port: 6370, + retry_max_delay: 3000, + connect_timeout : 10000, // 10 secs for normal connection timeout. we need no such a long timeout + }, + /* jshint ignore:end*/ + tokenExpireTime : conf.services.auth.tokenExpireTime / 2 + }, + dataMapperConf: { connectors: { mysql: { diff --git a/src/server/emul/incubator-ripple/lib/server/emulate.js b/src/server/emul/incubator-ripple/lib/server/emulate.js index 3ad2c10..e1ff5b7 100644 --- a/src/server/emul/incubator-ripple/lib/server/emulate.js +++ b/src/server/emul/incubator-ripple/lib/server/emulate.js @@ -89,7 +89,7 @@ module.exports = { next(); }); - router.get(qPath, options.verifyToken); + router.get(qPath, options.ensureLogin); router.get(qPath, cordovaProject.inject(options)); router.get(qPath, hosted.inject(options)); router.get(qPath, static.inject(options)); diff --git a/src/server/fs/lib/console-manager.js b/src/server/fs/lib/console-manager.js index 03a7c46..f08132e 100644 --- a/src/server/fs/lib/console-manager.js +++ b/src/server/fs/lib/console-manager.js @@ -353,7 +353,7 @@ exports.registerTerminalService = registerTerminalService; exports.router = new express.Router(); exports.router.post('/webida/api/fs/exec/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var rsc = 'fs:' + req.params.fsid + '/*'; authMgr.checkAuthorize({uid:req.user.uid, action:'fs:exec', rsc:rsc}, res, next); diff --git a/src/server/fs/lib/fs-manager.js b/src/server/fs/lib/fs-manager.js index 8962c83..735cef2 100644 --- a/src/server/fs/lib/fs-manager.js +++ b/src/server/fs/lib/fs-manager.js @@ -1106,7 +1106,7 @@ function serveFile(req, res, srcUrl, serveErrorPage) { * @returns {Object} fsinfo - {owner: , fsid: } */ router.get('/webida/api/fs/:fsid', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var fs = new WebidaFS(fsid); @@ -1137,7 +1137,7 @@ router.get('/webida/api/fs/:fsid', * @returns {Object} fsinfos - [{owner: , fsid: }] */ router.get('/webida/api/fs', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'fssvc:getMyFSInfos', rsc:'fssvc:*'}, res, next); }, @@ -1163,7 +1163,7 @@ router.get('/webida/api/fs', * @returns {Object} result - {result:'ok'|'fail', fsinfo: {owner: , fsid: }} */ router.post('/webida/api/fs', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { authMgr.checkAuthorize({uid:req.user.uid, action:'fssvc:addMyFS', rsc:'fssvc:*'}, res, next); }, @@ -1179,6 +1179,7 @@ router.post('/webida/api/fs', var fsid; // rollback function + /* jshint -W086 : we use some 'falling through' trick here*/ var rollback = function (err, result) { var msg; switch (state) { @@ -1256,7 +1257,7 @@ router.post('/webida/api/fs', * @param {String} fsid */ router.delete('/webida/api/fs/:fsid', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var user = req.user; if (!user.isAdmin) { @@ -1551,7 +1552,7 @@ exports.writeFile = writeFile; */ router.post('/webida/api/fs/file/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var fsid = req.params.fsid; var dest = decodeURI(req.params[0]); @@ -1689,7 +1690,7 @@ router.post('/webida/api/fs/file/:fsid/*', * @param {String} recursive - true / false [default=false] */ router['delete']('/webida/api/fs/file/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = decodeURI(req.params[0]); if (path[0] !== '/') { @@ -1873,7 +1874,7 @@ function findFirstExist(wfs, path, callback) { * @param {String} recursive - true / false [default=false] */ router.post('/webida/api/fs/directory/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, multipartMiddleware, function (req, res, next) { var rsc; @@ -1943,7 +1944,7 @@ router.post('/webida/api/fs/directory/:fsid/*', * @method RESTful API copy - /webida/api/fs/copy//?src=&dest=&recursive=[false|true] */ router.post('/webida/api/fs/copy/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { // check src read permission var path = req.body.src; if (path[0] !== '/') { @@ -2027,7 +2028,7 @@ router.post('/webida/api/fs/copy/:fsid/*', * @method RESTful API rename - /webida/api/fs/rename/{fsid}/?oldpath={oldpath}&newpath={newpath} */ router.post('/webida/api/fs/rename/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { // check src write permission var path = req.body.oldpath; if (path[0] !== '/') { @@ -2267,7 +2268,7 @@ exports.search = search; * @param {String} includefile - include file/dir regex pattern. If not specified, include all files and dirs. */ router.get('/webida/api/fs/search/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = req.query.where; if (path[0] !== '/') { @@ -2365,7 +2366,7 @@ function replace(rootPath, targetPaths, wholePattern, callback) { * @param {String} wholeword - 'true' / 'false' [default='false'] */ router.post('/webida/api/fs/replace/:fsid', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res) { // TODO ACL var fsid = req.params.fsid; @@ -2433,7 +2434,7 @@ router.post('/webida/api/fs/replace/:fsid', * @param {String} mode - create / extract / export */ router.get('/webida/api/fs/archive/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { // check srclist read permission var aclInfo = {uid: req.user.uid, action: 'fs:archive', rsc: req.query.source, fsid: req.params.fsid}; authMgr.checkAuthorizeMulti(aclInfo, res, next); @@ -2588,7 +2589,7 @@ router.get('/webida/api/fs/exists/:fsid/*', * @param {String} fsid - fsid * @param {String} path - file path */ -router.get('/webida/api/fs/acl/:fsid/*', authMgr.verifyToken, function (req, res) { +router.get('/webida/api/fs/acl/:fsid/*', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var pathUrl = 'wfs://' + fsid + Path.join('/', decodeURI(req.params[0])); canReadAcl(req.user.uid, pathUrl, function (err, canRead) { @@ -2617,7 +2618,7 @@ router.get('/webida/api/fs/acl/:fsid/*', authMgr.verifyToken, function (req, res * @param {String} path - file path * @param {String} acl - stringified acl object. eg. {"usrename1":"r","username2":"w","usrename3":"rw"} */ -router.post('/webida/api/fs/acl/:fsid/*', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/fs/acl/:fsid/*', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var pathUrl = 'wfs://' + fsid + Path.join('/', decodeURI(req.params[0])); var newAcl = JSON.parse(req.body.acl); @@ -2690,7 +2691,7 @@ router.get('/webida/api/fs/meta/:fsid/*', * @param {String} data - stringified metadata object */ router.post('/webida/api/fs/meta/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = decodeURI(req.params[0]); if (path[0] !== '/') { @@ -2772,7 +2773,7 @@ router.get(config.services.fs.fsAliasUrlPrefix + '/*', function (req, res) { } */ router.post('/webida/api/fs/alias/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = decodeURI(req.params[0]); if (path[0] !== '/') { @@ -2822,7 +2823,7 @@ router.post('/webida/api/fs/alias/:fsid/*', * @param {aliasKey} - aliasKey */ router['delete']('/webida/api/fs/alias/:aliasKey', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var aliasKey = decodeURI(req.params.aliasKey); if (!aliasKey) { @@ -2878,7 +2879,7 @@ router['delete']('/webida/api/fs/alias/:aliasKey', } */ router.get('/webida/api/fs/alias/:aliasKey', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var aliasKey = decodeURI(req.params.aliasKey); if (!aliasKey) { @@ -2925,7 +2926,7 @@ router.get('/webida/api/fs/alias/:aliasKey', * @return {string} - usage in bytes */ router.get('/webida/api/fs/usage/:fsid', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var rsc = 'fs:' + req.params.fsid + '/*'; authMgr.checkAuthorize({uid:req.user.uid, action:'fs:getQuotaUsage', rsc:rsc}, res, next); @@ -2964,7 +2965,7 @@ router.get('/webida/api/fs/usage/:fsid', * @return {string} - quota limit in bytes */ router.get('/webida/api/fs/limit/:fsid', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var rsc = 'fs:' + req.params.fsid + '/*'; authMgr.checkAuthorize({uid:req.user.uid, action:'fs:getQuotaLimit', rsc:rsc}, res, next); @@ -3223,7 +3224,7 @@ function writeKsFile(req, res, cb) { * @return {string} - succss or failure */ -router.post('/webida/api/fs/mobile/ks/:fsid/*', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/fs/mobile/ks/:fsid/*', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; writeKsFile(req, res, function (err, reason, file, fields) { @@ -3291,7 +3292,7 @@ function deleteKsFile(uid, fsid, filename, cb) { * @return {string} - succss or failure */ -router.delete('/webida/api/fs/mobile/ks/:fsid', authMgr.verifyToken, function (req, res) { +router.delete('/webida/api/fs/mobile/ks/:fsid', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var uid = req.user && req.user.uid; var alias = req.body.alias; @@ -3328,7 +3329,7 @@ router.delete('/webida/api/fs/mobile/ks/:fsid', authMgr.verifyToken, function (r * @return {string} - keystore info */ -router.get('/webida/api/fs/mobile/ks/:fsid', authMgr.verifyToken, function (req, res) { +router.get('/webida/api/fs/mobile/ks/:fsid', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; getKsList(req.user.userId, fsid, function (err, ksList) { @@ -3340,7 +3341,7 @@ router.get('/webida/api/fs/mobile/ks/:fsid', authMgr.verifyToken, function (req, }); router.get('/webida/api/fs/lockfile/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = decodeURI(req.params[0]); if (path[0] !== '/') { @@ -3391,7 +3392,7 @@ router.get('/webida/api/fs/lockfile/:fsid/*', router.get('/webida/api/fs/unlockfile/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = decodeURI(req.params[0]); if (path[0] !== '/') { @@ -3437,7 +3438,7 @@ router.get('/webida/api/fs/unlockfile/:fsid/*', ); router.get('/webida/api/fs/getlockedfiles/:fsid/*', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { var path = decodeURI(req.params[0]); if (path[0] !== '/') { @@ -3476,7 +3477,7 @@ router.get('/webida/api/fs/getlockedfiles/:fsid/*', * @return {string} - succss or failure */ -router.post('/webida/api/fs/flink/:fsid/*', authMgr.verifyToken, function (req, res) { +router.post('/webida/api/fs/flink/:fsid/*', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var filePath = decodeURI(req.params[0]); if (filePath[0] !== '/') { @@ -3505,7 +3506,7 @@ router.post('/webida/api/fs/flink/:fsid/*', authMgr.verifyToken, function (req, * @return {string} - succss or failure */ -router.get('/webida/api/fs/flink/:fsid/:fileid', authMgr.verifyToken, function (req, res) { +router.get('/webida/api/fs/flink/:fsid/:fileid', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var fileId = req.params.fileid; @@ -3542,7 +3543,7 @@ router.get('/webida/api/fs/flink/:fsid/:fileid', authMgr.verifyToken, function ( * @return {string} - succss or failure */ -router.get('/webida/api/fs/flinkbypath/:fsid/*', authMgr.verifyToken, function (req, res) { +router.get('/webida/api/fs/flinkbypath/:fsid/*', authMgr.ensureLogin, function (req, res) { var fsid = req.params.fsid; var wfsUrl = 'wfs://' + fsid + Path.join('/', decodeURI(req.params[0])); diff --git a/src/server/fs/test/test-fs.js b/src/server/fs/test/test-fs.js index 2e4d7de..b2ccf24 100644 --- a/src/server/fs/test/test-fs.js +++ b/src/server/fs/test/test-fs.js @@ -30,7 +30,7 @@ var account = {email: 'test1@webida.org', uid: 200000, isAdmin: false}; var app; -authMgr.verifyToken = function(req, res, next) { +authMgr.ensureLogin = function(req, res, next) { req.user = {}; req.user.uid = account.uid; req.user.isAdmin = account.isAdmin; diff --git a/src/server/mon/lib/pf-manager.js b/src/server/mon/lib/pf-manager.js index 21642de..cd82d7b 100644 --- a/src/server/mon/lib/pf-manager.js +++ b/src/server/mon/lib/pf-manager.js @@ -26,7 +26,7 @@ module.exports.router = router; router.get('/webida/api/mon/pf/getSvcTypeList', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { pfdb.profile_inst.getSvcTypeList(function (err, result) { if (err) { @@ -39,7 +39,7 @@ router.get('/webida/api/mon/pf/getSvcTypeList', ); router.get('/webida/api/mon/pf/getInstNameList', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { pfdb.profile_inst.getInstNameList(function (err, result) { if (err) { @@ -52,7 +52,7 @@ router.get('/webida/api/mon/pf/getInstNameList', ); router.get('/webida/api/mon/pf/getInstList', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { pfdb.profile_inst.getInstList(function (err, result) { if (err) { @@ -66,7 +66,7 @@ router.get('/webida/api/mon/pf/getInstList', ); router.get('/webida/api/mon/pf/getInstListByInstName', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { pfdb.profile_inst.getInstListByInstName(req.query.instname, function (err, result) { if (err) { @@ -80,7 +80,7 @@ router.get('/webida/api/mon/pf/getInstListByInstName', ); router.get('/webida/api/mon/pf/getUrlList', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { pfdb.profile_inst_req.getUrlList(function (err, result) { if (err) { @@ -93,7 +93,7 @@ router.get('/webida/api/mon/pf/getUrlList', ); router.get('/webida/api/mon/pf/getCurrentReqs', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { logger.debug('query : ', req.query); logger.debug('options : ', req.query.options); @@ -111,7 +111,7 @@ router.get('/webida/api/mon/pf/getCurrentReqs', ); router.get('/webida/api/mon/pf/getCurrentReqsStat', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { logger.debug('query : ', req.query); logger.debug('options : ', req.query.options); @@ -130,7 +130,7 @@ router.get('/webida/api/mon/pf/getCurrentReqsStat', router.get('/webida/api/mon/pf/getStatisticsHistory', - authMgr.verifyToken, + authMgr.ensureLogin, function (req, res, next) { logger.debug('query : ', req.query); logger.debug('options : ', req.query.options); diff --git a/src/server/notify/lib/conn-msg.js b/src/server/notify/lib/conn-msg.js index 8260fb6..e2ff3fd 100644 --- a/src/server/notify/lib/conn-msg.js +++ b/src/server/notify/lib/conn-msg.js @@ -34,7 +34,7 @@ module.exports.setNtfSvr = function (ntf) { var onAuth = function (cli, msg) { logger.info('cli = ' + cli.id); logger.info('auth = ' + JSON.stringify(msg)); - authMgr.verifySession(msg.token, function(err, user) { + authMgr.getUserInfoByToken(msg.token, function(err, user) { if (err) { cli.sendMsg('authAns', _MSG(Err.authInvalidToken, null)); } else { diff --git a/src/server/package.json b/src/server/package.json index ba43ad5..6d1e3f6 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -73,7 +73,9 @@ "data-mapper": "*", "cron": "^1.0.9", "jsonfile": "~2.2.1", - "node-watch": "~0.3.4" + "node-watch": "~0.3.4", + "redis" : "^2.3.0", + "hiredis" : "^0.4.1" }, "devDependencies": { "q-io": "1.11.0", From a2c0365c297473d9f0e3e9e29d181f8e6de42e19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=91=ED=9B=88?= Date: Tue, 17 Nov 2015 21:00:59 +0900 Subject: [PATCH 2/8] added cache.js, bulit on redis client - changed node-redis to ioredis (faster & more powerful) - sorted dependency list & pull '*' versioned packages - added cache-related configuration in default-conf.js - fixed some bugs in auth-manager.js - added simple test for cache.js --- src/server/common/auth-manager.js | 47 ++++---- src/server/common/cache.js | 188 ++++++++++++++++++++++++++++++ src/server/conf/default-conf.js | 28 +++-- src/server/package.json | 100 ++++++++-------- src/server/test/cache-test.js | 25 ++++ 5 files changed, 309 insertions(+), 79 deletions(-) create mode 100644 src/server/common/cache.js create mode 100644 src/server/test/cache-test.js diff --git a/src/server/common/auth-manager.js b/src/server/common/auth-manager.js index 26f4058..867f6a6 100644 --- a/src/server/common/auth-manager.js +++ b/src/server/common/auth-manager.js @@ -47,7 +47,10 @@ function _requestTokenInfo(token, callback) { logger.debug('request token info', options); function handleResponse(err, res, body) { var tokenInfo; - if (err) { return callback(err); } + if (err) { + logger.info('oauth/verify - result : error ', err); + return callback(err); + } if (res.statusCode === 200) { try { tokenInfo = JSON.parse(body).data; @@ -67,12 +70,12 @@ function _requestTokenInfo(token, callback) { var data = ''; logger.info('res', res.statusCode); res.setEncoding('utf8'); - //res.on('data', function (chunk) { - // logger.info('data chunk', chunk); - // data += chunk; - //}); + res.on('data', function (chunk) { + logger.debug('data chunk', chunk); + data += chunk; + }); res.on('end', function () { - logger.info('end', data); + logger.debug('end', data); handleResponse(null, res, data); }); }); @@ -95,11 +98,12 @@ function _checkExpired(info, callback) { current = new Date().getTime(); expire = new Date(info.expireTime).getTime(); - logger.info('_checkExpired', current, info, expire); + logger.debug('_checkExpired', current, info, expire); if (expire - current < 0) { + logger.debug('token expired - callback with 419'); return callback(419); } else { - return callback(0, info); + return callback(null, info); } } @@ -134,7 +138,7 @@ function getUserInfo(req, res, next) { _verifyToken(token, function (err, info) { var errMsg = 'Internal server error'; if (err) { - logger.debug('_verifyToken failed with ' + err); + logger.debug('_verifyToken failed ', err); if (err === 400 || err === 419) { if (allowAnonymous) { return next(); @@ -147,7 +151,7 @@ function getUserInfo(req, res, next) { return next(); }); } -exports.getUserINfo = getUserInfo; +exports.getUserInfo = getUserInfo; function ensureLogin(req, res, next) { req.disallowAnonymous = true; @@ -180,14 +184,13 @@ function _verifyToken(token, callback) { exports.getUserInfoByToken = _verifyToken; -function sendCheckAuthorizeRequest(aclInfo, options, res, next) { +function _sendCheckAuthorizeRequest(options, res, next) { var req; - logger.info('checkAuthorize', aclInfo); req = http.request(options, function (response) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('checkAuthorize data chunk', chunk); + logger.debug('checkAuthorize data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -212,7 +215,8 @@ function checkAuthorize(aclInfo, res, next) { port: internalAccessInfo.auth.port, path: encodeURI(template(aclInfo)) }; - sendCheckAuthorizeRequest(aclInfo, options, res, next); + logger.debug('checkAuthorize', aclInfo); + _sendCheckAuthorizeRequest(options, res, next); } exports.checkAuthorize = checkAuthorize; @@ -227,7 +231,8 @@ function checkAuthorizeMulti(aclInfo, res, next) { port: internalAccessInfo.auth.port, path: encodeURI(template(aclInfo)) }; - sendCheckAuthorizeRequest(aclInfo, options, res, next); + logger.debug('checkAuthorizeMulti', aclInfo); + _sendCheckAuthorizeRequest(options, res, next); } exports.checkAuthorizeMulti = checkAuthorizeMulti; @@ -256,7 +261,7 @@ function createPolicy(policy, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('createPolicy data chunk', chunk); + logger.debug('createPolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -299,7 +304,7 @@ function deletePolicy(pid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('deletePolicy data chunk', chunk); + logger.debug('deletePolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -334,7 +339,7 @@ function assignPolicy(id, pid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('assignPolicy data chunk', chunk); + logger.debug('assignPolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -369,7 +374,7 @@ function removePolicy(pid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('removePolicy data chunk', chunk); + logger.debug('removePolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -407,7 +412,7 @@ function getPolicy(policyRule, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('getPolicy data chunk', chunk); + logger.debug('getPolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -456,7 +461,7 @@ function updatePolicyResource(oldPath, newPath, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.info('updatePolicyResource data chunk', chunk); + logger.debug('updatePolicyResource data chunk', chunk); data += chunk; }); response.on('end', function () { diff --git a/src/server/common/cache.js b/src/server/common/cache.js new file mode 100644 index 0000000..fdf1ce9 --- /dev/null +++ b/src/server/common/cache.js @@ -0,0 +1,188 @@ +/* + * Copyright (c) 2012-2015 S-Core Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +var clone = require('clone'); +var Redis = require('ioredis'); +var Promise = Promise || Redis.Promise; // ioredis exports bluebird promise +var conf = require('./conf-manager'); +var logger = require('./log-manager'); + +//Redis.Promise.onPossiblyUnhandledRejection(function(err) { +// logger.warn('unhandled redis error : ', err); +//}); + +function Cache(typeName) { + var cacheConf = conf.cache.types[typeName]; + var redisConf; + if (!cacheConf) { + throw new Error ('Invalid cache type : ' + typeName); + } + redisConf = clone(cacheConf.cache.redis); + redisConf.keyPrefix = cacheConf.prefix + ':'; + + this.name = 'Cache( ' + typeName + ')'; + this._ttl = cacheConf.ttl; + this._ttlGenerator = cacheConf.ttlGenerator; + this._autoExtendTtl = cacheConf.autoExtendTtl; + + this.redis = new Redis(redisConf); + this.redis.on('connect', this.createLoggingFunction('connect', logger.debug) ); + this.redis.on('reconnecting', this.createLoggingFunction('reconnecting', logger.info)); + this.redis.on('error', this.createLoggingFunction('error', logger.error)); + this.redis.on('end', this.createLoggingFunction('end', logger.error)); +} + +Cache.prototype = { + createLoggingFunction : function(eventName, logFunc) { + var that = this; + return function(event) { + var msg = that.name + ' event : ' + eventName; + var args = [msg]; + if (event) { + args.push(event); + } + logFunc.apply(null, args); + }; + }, + + // when callback is not specified + // then redis.set will return a promise + write : function (key, value, callback) { + var self = this; + var ret = new Promise(function(resolve, reject) { + var serialized = JSON.stringify(value); + var ttl = self._getTtl(value); + var detail = { + key:key, value : serialized, ttl : ttl + }; + logger.debug('saving to ' + self.name, detail); + var promise = null; + if ( typeof(ttl) === 'number') { + if (ttl <= 0 ) { + return reject(new Error(self.name + ' cannot write value with negative TTL ' + ttl)); + } + promise = self.redis.set(key, serialized, 'EX', ttl); + } else { + promise = self.redis.set(key, serialized); + } + promise.then( function(value) { + resolve(value); + }).catch( function(err) { + reject(err); + }); + }); + if (typeof(callback) === 'function') { + ret.then( function(value) { + callback(null, value); + }).catch( function(err) { + callback(err); + }); + ret = undefined; + } + return ret; + }, + + read : function (key, callback) { + var self = this; + var ret = new Promise(function(resolve, reject) { + self.redis.get(key).then( function(value) { + var deserialized = value; + if(typeof(value) === 'object') { + deserialized = JSON.parse(value); + } + logger.debug(self.name + ' read', { key : key, value : value}); + if (deserialized, self._autoExtendTtl) { + var ttl = self._getTtl(deserialized); + self.redis.expire(key, ttl, function(err) { + if (err) { + logger(self.name + ' could not extend ttl of ' + key); + } else { + logger.debug(self.name + ' has extended ttl ' + key + ' , ' + ttl); + } + }); + } + resolve(deserialized); + }).catch(function(err) { + logger.error(self.name + ' read fail - ' + key, err); + reject(err); + }); + }); + if (typeof(callback) === 'function') { + ret.then( function(value) { + callback(null, value); + }).catch( function(err) { + callback(err); + }); + ret = undefined; + } + return ret; + }, + + remove : function(key, callback) { + var self = this; + var ret = new Promise( function(resolve, reject) { + self.redis.del(key).then(function(value) { + logger.debug(self.name + ' deleted', { key : key, value : value }); + resolve(value); + }).catch(function(err) { + logger.error(self.name + ' remove fail - ' + key, err); + reject(err); + }); + }); + if (typeof(callback) === 'function') { + ret.then( function(value) { + callback(null, value); + }).catch( function(err) { + callback(err); + }); + ret = undefined; + } + return ret; + }, + + _getTtl: function(value) { + if (this._ttlGenerator) { + return this._ttlGenerator(value); + } + if (this._ttl) { + return this._ttl; + } + return -1; + } +}; + +// we may enable monitor connection to redis server +// But it's silly to activate monitoring in every process. +// so, monitoring function should be manually caleld by some dedicated unit service +module.exports = { + createCache: function(type) { + return new Cache(type); + }, + enableMonitor : function() { + var redis = new Redis(); + redis.monitor(function(err, monitor) { + if (err) { + logger.error('cannot create monitor'); + } else { + monitor.on('monitor', function (time, args) { + logger.debug("cache monitor : " + time, args ); + }); + } + }); + } +}; \ No newline at end of file diff --git a/src/server/conf/default-conf.js b/src/server/conf/default-conf.js index 497401e..a172a04 100755 --- a/src/server/conf/default-conf.js +++ b/src/server/conf/default-conf.js @@ -186,16 +186,30 @@ var conf = { monHostUrl: serviceInstances.mon[0].url, cache: { - // conf.cache.redis object will be passed to redis client library directly. don't check with jshint. - /* jshint ignore:start */ + // conf.cache.redis object will be passed to ioredis client library directly + // see ioredis API document for detalied options. (We don't support redis-cluster yet); redis : { host: '127.0.0.1', - port: 6370, - retry_max_delay: 3000, - connect_timeout : 10000, // 10 secs for normal connection timeout. we need no such a long timeout + port: 6379, + showFriendlyErrorStack:true }, - /* jshint ignore:end*/ - tokenExpireTime : conf.services.auth.tokenExpireTime / 2 + // cache items are categorized + types : { + token : { + prefix:'tk', + ttlGenerator: function(cacheValueObject) { + var expireDate = cacheValueObject.expireTime.getTime(); + var currentDate = new Date().getTime(); + // getTime() returns in msec unit. + return Math.floor((expireDate - currentDate) / 1000); + } + }, + policy : { + prefix:'acl', + ttl: 10*60, + autoExtendTtl:true + } + } }, dataMapperConf: { diff --git a/src/server/package.json b/src/server/package.json index 6d1e3f6..f9ef30e 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -7,75 +7,73 @@ "url": "git://github.com/webida/webida-server.git" }, "dependencies": { - "dateformat": "1.0.11", + "async": "*", + "data-mapper": "*", + "jquery": "*", + "optimist": "*", + "url": "*", + "underscore": "*", "connect-domain": "*", - "connect-multiparty": "1.2.5", - "express-session": "1.9.1", - "http-proxy": "0.10.3", - "http-master": "~1.0.18", - "connect-sqlite3": "0.9.5", - "socket.io": "1.2.0", - "hashmap": "1.0.1", - "cuid": "1.2.1", - "jsdom": "0.10.6", - "unzip": "0.1.9", - "fstream": "0.1.25", - "corser": "2.0.0", - "optimist": "0.6.0", - "underscore": "1.5.2", - "request": "*", - "winston": "~1.1.2", "querystring": "*", - "passport": "~0.1.12", - "express": "4.10.2", - "compression": "1.2.0", + "request": "*", "body-parser": "1.9.2", + "clone" : "^1.0.2", + "compression": "1.2.0", + "connect-ensure-login": "0.1.x", + "connect-multiparty": "1.2.5", + "connect-sqlite3": "0.9.5", "cookie-parser": "1.3.3", - "on-finished": "2.3.0", - "morgan": "1.5.0", + "corser": "2.0.0", + "cron": "^1.0.9", + "cuid": "1.2.1", + "dateformat": "1.0.11", "dnode": "1.2.0", + "ejs": "0.7.x", + "express": "4.10.2", + "express-session": "1.9.1", + "ffi": "1.2.6", + "filequeue": "0.5.0", + "formidable": "1.0.17", + "fs-extra": "0.8.1", + "fstream": "0.1.25", "glob": "3.2.9", + "graceful-fs": "3.0.2", + "guid": "0.0.12", + "hashmap": "1.0.1", + "hiredis" : "^0.4.1", + "http-master": "~1.0.18", + "http-proxy": "0.10.3", + "ioredis" : "^1.10.0", + "jsdom": "0.10.6", + "jsonfile": "~2.2.1", + "line-reader": "0.2.3", + "lodash": "3.9.3", + "mkdirp": "0.5.0", + "morgan": "1.5.0", + "mysql": "2.5.3", "nexpect": "0.4.2", - "connect-ensure-login": "0.1.x", + "nodemailer": "0.7.x", + "node-watch": "~0.3.4", "oauth2orize": "0.x.x", - "url": "*", - "passport-local": "0.1.x", + "on-finished": "2.3.0", + "passport": "~0.1.12", + "passport-github": "*", + "passport-google-oauth": "*", "passport-http": "0.2.x", "passport-http-bearer": "0.2.x", + "passport-local": "0.1.x", "passport-oauth2-client-password": "0.1.x", - "ejs": "0.7.x", - "optimist": "*", - "underscore": "*", - "async": "*", - "jquery": "*", - "passport-github": "*", - "passport-google-oauth": "*", - "nodemailer": "0.7.x", - "mysql": "2.5.3", - "line-reader": "0.2.3", - "filequeue": "0.5.0", - "graceful-fs": "3.0.2", - "ffi": "1.2.6", - "formidable": "1.0.17", - "fs-extra": "0.8.1", "pty.js": "0.2.7-1", "q": "1.0.1", + "replace": "0.2.9", "send": "0.2.0", "shortid": "2.2.2", + "socket.io": "1.2.0", "tmp": "0.0.23", - "underscore": "1.6.0", + "unzip": "0.1.9", "URIjs": "1.12.1", "walkdir": "0.0.7", - "mkdirp": "0.5.0", - "guid": "0.0.12", - "replace": "0.2.9", - "lodash": "3.9.3", - "data-mapper": "*", - "cron": "^1.0.9", - "jsonfile": "~2.2.1", - "node-watch": "~0.3.4", - "redis" : "^2.3.0", - "hiredis" : "^0.4.1" + "winston": "~1.1.2" }, "devDependencies": { "q-io": "1.11.0", diff --git a/src/server/test/cache-test.js b/src/server/test/cache-test.js new file mode 100644 index 0000000..8b12d2e --- /dev/null +++ b/src/server/test/cache-test.js @@ -0,0 +1,25 @@ +'use strict'; + +var cache = require('../common/cache'); + +var test = cache.createCache('token'); + +cache.enableMonitor(); + +test.set('123', {aa:'bb'}, function(err) { + if (err) { + console.error(err); + } + else { + console.log('saved...'); + } +}); + +test.get('123', function(err, value) { + if (err) { + console.error(err); + } + else { + console.log('loaded...', value); + } +}); \ No newline at end of file From 9b2acc7115a054111151e36ae8c365246b42069a Mon Sep 17 00:00:00 2001 From: gotchazipc Date: Tue, 17 Nov 2015 12:09:38 +0000 Subject: [PATCH 3/8] bugfix of cache-test - removed test/conf, replaced with symbolic link - fixed bugs of Cache constructor - fixed bugs in cache test --- src/server/common/cache.js | 6 +++--- src/server/test/cache-test.js | 8 +++++--- src/server/test/conf | 1 + src/server/test/conf/data-mapper-conf.json | 16 ---------------- src/server/test/conf/mapper/user-mapper.json5 | 11 ----------- 5 files changed, 9 insertions(+), 33 deletions(-) create mode 120000 src/server/test/conf delete mode 100644 src/server/test/conf/data-mapper-conf.json delete mode 100644 src/server/test/conf/mapper/user-mapper.json5 diff --git a/src/server/common/cache.js b/src/server/common/cache.js index fdf1ce9..84162cb 100644 --- a/src/server/common/cache.js +++ b/src/server/common/cache.js @@ -19,7 +19,7 @@ var clone = require('clone'); var Redis = require('ioredis'); var Promise = Promise || Redis.Promise; // ioredis exports bluebird promise -var conf = require('./conf-manager'); +var conf = require('./conf-manager').conf; var logger = require('./log-manager'); //Redis.Promise.onPossiblyUnhandledRejection(function(err) { @@ -32,7 +32,7 @@ function Cache(typeName) { if (!cacheConf) { throw new Error ('Invalid cache type : ' + typeName); } - redisConf = clone(cacheConf.cache.redis); + redisConf = clone(conf.cache.redis); redisConf.keyPrefix = cacheConf.prefix + ':'; this.name = 'Cache( ' + typeName + ')'; @@ -185,4 +185,4 @@ module.exports = { } }); } -}; \ No newline at end of file +}; diff --git a/src/server/test/cache-test.js b/src/server/test/cache-test.js index 8b12d2e..3327aaa 100644 --- a/src/server/test/cache-test.js +++ b/src/server/test/cache-test.js @@ -6,7 +6,9 @@ var test = cache.createCache('token'); cache.enableMonitor(); -test.set('123', {aa:'bb'}, function(err) { +var ext = new Date(); +ext.setDate(ext.getDate() + 1); +test.write('123', {aa:'bb', expireTime : ext}, function(err) { if (err) { console.error(err); } @@ -15,11 +17,11 @@ test.set('123', {aa:'bb'}, function(err) { } }); -test.get('123', function(err, value) { +test.read('123', function(err, value) { if (err) { console.error(err); } else { console.log('loaded...', value); } -}); \ No newline at end of file +}); diff --git a/src/server/test/conf b/src/server/test/conf new file mode 120000 index 0000000..59f0502 --- /dev/null +++ b/src/server/test/conf @@ -0,0 +1 @@ +../conf \ No newline at end of file diff --git a/src/server/test/conf/data-mapper-conf.json b/src/server/test/conf/data-mapper-conf.json deleted file mode 100644 index 8841bf6..0000000 --- a/src/server/test/conf/data-mapper-conf.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "connectors": { - "mysql": { - "type": "mysql", - "connectionLimit": 5, - "host": "172.21.101.124", - "user": "test", - "password": "test", - "database": "test", - "default": true - } - }, - "mappers": { - "user": "conf/mapper/user-mapper.json5" - } -} \ No newline at end of file diff --git a/src/server/test/conf/mapper/user-mapper.json5 b/src/server/test/conf/mapper/user-mapper.json5 deleted file mode 100644 index 85b7552..0000000 --- a/src/server/test/conf/mapper/user-mapper.json5 +++ /dev/null @@ -1,11 +0,0 @@ -{ - "getUserById": "SELECT * FROM t_user WHERE id = {{id}}", - "getUsers": "SELECT * FROM t_user", - "modifyUser": "UPDATE t_user SET \ - {{#if name}} name = '{{name}}' {{/if}} \ - {{#if phone}}, phone = '{{phone}}' {{/if}} \ - WHERE id = {{id}}", - "deleteUserById": "DELETE FROM t_user WHERE id = {{id}}", - "deleteAllUser": "DELETE FROM t_user", - "addUser": "INSERT INTO t_user VALUES ({{id}}, '{{name}}', '{{phone}}')" -} \ No newline at end of file From dd0ae29de161cbbccc489443343cc29b1e1d9af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=91=ED=9B=88?= Date: Wed, 18 Nov 2015 18:04:14 +0900 Subject: [PATCH 4/8] implemented caching in auth-manager.js - implemented token read/write logics - merged checkAuthorization & checkAuthorize - fixed some typos --- src/server/auth/lib/acl-manager.js | 38 ++---- src/server/common/auth-manager.js | 205 +++++++++++++++++++---------- src/server/common/cache.js | 6 +- src/server/conf/default-conf.js | 2 +- src/server/fs/lib/fs-manager.js | 6 +- src/server/test/cache-test.js | 69 +++++++--- 6 files changed, 208 insertions(+), 118 deletions(-) diff --git a/src/server/auth/lib/acl-manager.js b/src/server/auth/lib/acl-manager.js index ac0a9bd..c144fa4 100644 --- a/src/server/auth/lib/acl-manager.js +++ b/src/server/auth/lib/acl-manager.js @@ -689,37 +689,19 @@ router.get('/webida/api/acl/getownedpolicy', } ); -// aclInfo : {uid:int, action:string, rsc:string} +// req.query : {uid:int, action:string, rsc:[string]} router.get('/checkauthorize', - function (req, res, next) { - var aclInfo = req.query; - - userdb.checkAuthorize(aclInfo, function(err) { - if (!err) { - return res.send(utils.ok()); - } else { - return res.sendfail(new ClientError(401, 'Not authorized.')); - } - }); - } -); - -// req.query : {uid:int, action:string, rsc:[string], fsid:string} -router.get('/checkauthorizemulti', function (req, res, next) { var query = req.query; - var source = []; + var resources = []; if (query.rsc.length > 0) { - source = query.rsc.split(';'); + resources = query.rsc.split(';'); } - - async.each(source, function(value, callback) { - if (value[0] !== '/') { - value = Path.join('/', value); - } - - var rsc = 'fs:' + query.fsid + value; - var aclInfo = {uid:query.uid, action:query.action, rsc:rsc}; + if (resources.length > 0 ) { + logger.debug('check authorize with multple resources : ', resources); + } + async.each(resources, function(resource, callback) { + var aclInfo = {uid:query.uid, action:query.action, rsc:resource}; userdb.checkAuthorize(aclInfo, function(err) { if (!err) { return callback(); @@ -729,10 +711,10 @@ router.get('/checkauthorizemulti', }); }, function (err) { if (err) { - errLog('Not authorized.', err); + errLog('checkAuthroze error - will return not authorized.', err); return res.send(401, utils.fail('Not authorized.')); } else { - return res.send(utils.ok()); + return next(); } }); } diff --git a/src/server/common/auth-manager.js b/src/server/common/auth-manager.js index 867f6a6..8c440bd 100644 --- a/src/server/common/auth-manager.js +++ b/src/server/common/auth-manager.js @@ -18,13 +18,14 @@ var http = require('http'); var util = require('util'); -var utils = require('./utils'); -var _ = require('lodash'); +var _ = require('lodash'); var querystring = require('querystring'); -var logger = require('./log-manager'); +var cache = require('./cache'); var config = require('./conf-manager').conf; +var logger = require('./log-manager'); +var utils = require('./utils'); var internalAccessInfo = config.internalAccessInfo; @@ -35,6 +36,36 @@ exports.init = function (db) { logger.info('auth-manager initialize. (' + db + ')'); }; +var tokenCache = cache.createCache('token'); +var authroizationCache = cache.createCache('authorization'); + +function _checkExpired(token, info, callback) { + var current; + var expire; + if (!info) { + return callback(500); + } + + if (info.validityPeriod <= 0) { // INFINITE + return callback(0, info); + } + + current = new Date().getTime(); + expire = new Date(info.expireTime).getTime(); + logger.debug('_checkExpired', current, info, expire); + if (expire - current < 0) { + logger.debug('token expired - callback with 419'); + return callback(419); + } else { + logger.debug ('saving token to cache ' + token, info); + tokenCache.write(token).then( () => { + return callback(null, info); + }).catch( err => { + logger.warn('cound not save token in cache ' + token, err); + }); + } +} + function _requestTokenInfo(token, callback) { var template = _.template('/webida/api/oauth/verify?token=<%= token %>'); var options = { @@ -58,7 +89,7 @@ function _requestTokenInfo(token, callback) { logger.error('Invalid oauth/verify response: ' + body, e); return callback(500); } - return callback(0, tokenInfo); + return _checkExpired(token, tokenInfo, callback); } else if (res.statusCode === 419) { return callback(419); } else { @@ -85,39 +116,20 @@ function _requestTokenInfo(token, callback) { req.end(); } -function _checkExpired(info, callback) { - var current; - var expire; - if (!info) { - return callback(500); - } - - if (info.validityPeriod <= 0) { // INFINITE - return callback(0, info); - } - - current = new Date().getTime(); - expire = new Date(info.expireTime).getTime(); - logger.debug('_checkExpired', current, info, expire); - if (expire - current < 0) { - logger.debug('token expired - callback with 419'); - return callback(419); - } else { - return callback(null, info); - } -} - function _verifyToken(token, callback) { if (!token) { logger.debug('token is null'); return callback(400); } - _requestTokenInfo(token, function (err, info) { - if (err) { - logger.debug('_requestTokenInfo failed', err); - return callback(err); + authroizationCache.read(token).then(value => { + if (value) { + return callback(value); + } else { + _requestTokenInfo(token, callback); } - return _checkExpired(info, callback); + }).catch(err => { + logger.warn('cannot read token cache ' + token, err); + _requestTokenInfo(token, callback); }); } @@ -168,23 +180,9 @@ exports.ensureAdmin = function ensureAdmin (req, res, next) { }); }; -function _verifyToken(token, callback) { - if (!token) { - logger.debug('token is null'); - return callback(400); - } - _requestTokenInfo(token, function (err, info) { - if (err) { - logger.debug('_requestTokenInfo failed', err); - return callback(err); - } - return _checkExpired(info, callback); - }); -} - exports.getUserInfoByToken = _verifyToken; -function _sendCheckAuthorizeRequest(options, res, next) { +function _sendCheckAuthorizeRequest(options, cacheKey, res, next) { var req; req = http.request(options, function (response) { var data = ''; @@ -195,8 +193,18 @@ function _sendCheckAuthorizeRequest(options, res, next) { }); response.on('end', function () { if (response.statusCode === 200) { + authroizationCache.write(cacheKey, true, function(err) { + if (err) { + logger.warn('could not wrote authorization result into cache for ', cacheKey); + } else { + logger.debug('cached authorization result ' + cacheKey); + } + next(); + }); return next(); } else if (response.statusCode === 401) { + // TODO : auth server should return 400 / 404 for invalid request + // then We can use cache to save 'forbidden' access for 401 response return res.status(401).send(utils.fail('Not authorized.')); } else { return res.status(500).send(utils.fail('Internal server error(checking authorization)')); @@ -208,33 +216,97 @@ function _sendCheckAuthorizeRequest(options, res, next) { }); req.end(); } -function checkAuthorize(aclInfo, res, next) { - var template = _.template('/checkauthorize?uid=<%= uid %>&action=<%= action %>&rsc=<%= rsc %>'); - var options = { - hostname: internalAccessInfo.auth.host, - port: internalAccessInfo.auth.port, - path: encodeURI(template(aclInfo)) + +function _createAuthorizationCacheKey(aclInfo) { + var ret = util.format('%s/%s/%s', aclInfo.uid, aclInfo.action, aclInfo.rsc); + if (aclInfo.fsid) { + ret += '/' + aclInfo.fsid; + } + return ret; +} + +function _buildAuthorizeRequest(rawRequest) { + function _normalizeFileSystemResources(rawResource) { + var rawResourceArray = rawResource.split(';'); + var resources = []; + rawResourceArray.forEach(function (resource) { + // fs:/abc/def/gef --> /abc/def/gef + // defs --> /defs + // fs:opqr/stuv --> /opqr/stuv + var path = '', arr; + if (resource.indexOf('fs:') === 0) { + if (resource[3] === '/') { + path = resource.slice(3); + } else { + path = '/' + resource.slice(3); + } + } else { + if (resource[0] === '/') { + path = resource; + } else { + path = '/' + resource; + } + } + // should find first non-empty element + if (path === '/') { + path = 'fs:/'; + } else { + arr = path.split('/'); + path = undefined; + for (var i = 1; i < arr.length; i++) { + path = arr[i]; + if (path) { + path = 'fs:/' + path; + break; + } + } + } + if (path && resources.indexOf(path) < 0) { + resources.push(path); + } + }); + return resources; + } + var ret = { + uid: rawRequest.uid, + action: rawRequest.action, + rsc: rawRequest.rsc }; - logger.debug('checkAuthorize', aclInfo); - _sendCheckAuthorizeRequest(options, res, next); + // when fsid is set, rsc should be normalized + if (rawRequest.fsid || rawRequest.action.indexOf('fs') === 0) { + ret.rsc = _normalizeFileSystemResources(rawRequest.rsc); + } + return ret; } -exports.checkAuthorize = checkAuthorize; -function checkAuthorizeMulti(aclInfo, res, next) { - var template = _.template('/checkauthorizemulti' + - '?uid=<%= uid %>' + - '&action=<%= action %>' + - '&rsc=<%= rsc %>' + - '&fsid=<%= fsid %>'); + +function checkAuthorize(aclInfo, res, next) { + var authorizeRequest = _buildAuthorizeRequest(aclInfo); + var cacheKey = _createAuthorizationCacheKey(aclInfo); + + var template = _.template('/checkauthorize?uid=<%= uid %>&action=<%= action %>&rsc=<%= rsc %>'); var options = { hostname: internalAccessInfo.auth.host, port: internalAccessInfo.auth.port, - path: encodeURI(template(aclInfo)) + path: encodeURI(template(authorizeRequest)) }; - logger.debug('checkAuthorizeMulti', aclInfo); - _sendCheckAuthorizeRequest(options, res, next); + + logger.debug('checkAuthorize : ', authorizeRequest); + // TODO : split cache items for multiple file system resources + // - auth server should response which resource is accessible and whichi is not. + // - then, we can cache each resource into differnt cache item. + authroizationCache.read(cacheKey).then(value => { + if(value === true) { + next(); + } else { + _sendCheckAuthorizeRequest(options, cacheKey, res, next); + } + }). catch( err => { + logger.warn('cound not read cache for ' + cacheKey, err); + _sendCheckAuthorizeRequest(options, cacheKey, res, next); + }); } -exports.checkAuthorizeMulti = checkAuthorizeMulti; +exports.checkAuthorize = checkAuthorize; function createPolicy(policy, token, callback) { var data = querystring.stringify({ @@ -311,8 +383,7 @@ function deletePolicy(pid, token, callback) { if (response.statusCode === 200) { return callback(null, pid); } else { - return callback(new ServerError('deletePolicy failed'), - pid); + return callback(new ServerError('deletePolicy failed'), pid); } }); }); diff --git a/src/server/common/cache.js b/src/server/common/cache.js index 84162cb..0f090e6 100644 --- a/src/server/common/cache.js +++ b/src/server/common/cache.js @@ -168,7 +168,7 @@ Cache.prototype = { // we may enable monitor connection to redis server // But it's silly to activate monitoring in every process. -// so, monitoring function should be manually caleld by some dedicated unit service +// So, monitoring should be enabled by some dedicated unit service module.exports = { createCache: function(type) { return new Cache(type); @@ -180,9 +180,9 @@ module.exports = { logger.error('cannot create monitor'); } else { monitor.on('monitor', function (time, args) { - logger.debug("cache monitor : " + time, args ); + logger.debug('cache monitor : ' + time, args ); }); } }); } -}; +}; \ No newline at end of file diff --git a/src/server/conf/default-conf.js b/src/server/conf/default-conf.js index a172a04..4d073e4 100755 --- a/src/server/conf/default-conf.js +++ b/src/server/conf/default-conf.js @@ -204,7 +204,7 @@ var conf = { return Math.floor((expireDate - currentDate) / 1000); } }, - policy : { + authorization: { prefix:'acl', ttl: 10*60, autoExtendTtl:true diff --git a/src/server/fs/lib/fs-manager.js b/src/server/fs/lib/fs-manager.js index 735cef2..45e9a21 100644 --- a/src/server/fs/lib/fs-manager.js +++ b/src/server/fs/lib/fs-manager.js @@ -1410,7 +1410,7 @@ router.get('/webida/api/fs/stat/:fsid/*', action:'fs:stat', rsc:req.query.source, fsid:req.params.fsid}; - authMgr.checkAuthorizeMulti(aclInfo, res, next); + authMgr.checkAuthorize(aclInfo, res, next); }, function (req, res) { var fsid = req.params.fsid; @@ -2427,7 +2427,7 @@ router.post('/webida/api/fs/replace/:fsid', * TODO acl * * @method RESTful API search - - * /webida/api/fs/archive/{fsid}/?source='list1,list2'&target='archive.zip'&mode=[create|extract|export] + * /webida/api/fs/archive/{fsid}/?source='list1;list2;list3'&target='archive.zip'&mode=[create|extract|export] * @param {String} fsid - fsid * @param {Array} Create and Export mode: the list of source(multiple), Extract mode: the source file(single) * @param {String} target @@ -2437,7 +2437,7 @@ router.get('/webida/api/fs/archive/:fsid/*', authMgr.ensureLogin, function (req, res, next) { // check srclist read permission var aclInfo = {uid: req.user.uid, action: 'fs:archive', rsc: req.query.source, fsid: req.params.fsid}; - authMgr.checkAuthorizeMulti(aclInfo, res, next); + authMgr.checkAuthorize(aclInfo, res, next); }, function (req, res, next) { // check dest write permission if (req.query.mode === 'export') { diff --git a/src/server/test/cache-test.js b/src/server/test/cache-test.js index 3327aaa..54d4c34 100644 --- a/src/server/test/cache-test.js +++ b/src/server/test/cache-test.js @@ -1,27 +1,64 @@ 'use strict'; +var assert = require('assert'); var cache = require('../common/cache'); - +var logger = require('../common/log-manager'); var test = cache.createCache('token'); +var ext = new Date(); +var value123; + +ext.setDate(ext.getDate() + 1); +value123 = {aa:'bb', expireTime : ext}; + cache.enableMonitor(); -var ext = new Date(); -ext.setDate(ext.getDate() + 1); -test.write('123', {aa:'bb', expireTime : ext}, function(err) { - if (err) { - console.error(err); - } - else { - console.log('saved...'); - } +test.write('123',value123, function(err, value) { + assert.ifError(err); + console.log('wrote token 123', value); }); test.read('123', function(err, value) { - if (err) { - console.error(err); - } - else { - console.log('loaded...', value); - } + assert.ifError(err); + assert.deepEqual(value, value123, 'read object has been changed'); + console.log('read token 123', value); +}); + +cache.read('1234', function(err, value) { + assert.ifError(err); + assert.equal(value, null, 'reading ghost key should return null'); + console.log('read token 1234 - was null, as expected'); +}); + + +cache.remove('123', function(err, value) { + assert.ifError(err); + assert.equal(value, 1, 'removing a key should return 1'); +}); + +cache.remove('1234', function(err, value) { + assert.ifError(err); + assert.equal(value, 0, 'removing a key should return 0'); +}); + +/* +test = cache.createCache('policy'); +var p = test.redis.Promise.all([ + test.write('100', { aa:100}), + test.write('101', { aa:101}), + test.write('102', { aa:102}), + test.write('103', { aa:103}) +]); +p.then ( function() { + return test.remove(['100', '101']); +}).then( function() { + return test.read('100'); +}).then( function(value) { + logger.info ("acl read 100", value); + return test.read('103'); +}).then( function(value) { + logger.info ("acl read 103", value); +}).catch( function(err) { + logger.test("acl test fail", err); }); +*/ \ No newline at end of file From 5abfab56974bf91f44a9b2f2a1076933dbf2f2ab Mon Sep 17 00:00:00 2001 From: gotchazipc Date: Wed, 18 Nov 2015 14:38:10 +0000 Subject: [PATCH 5/8] Bug Fix & package version up for Node 4.2 - many bugs are fixed - added some logs in userdb, acl-manager, fs-manager - fixed log timestamp format bug - hh --> HH - fixed default-conf.js sanity checker for right propery paths. --- src/server/auth/auth.js | 6 +-- src/server/auth/lib/acl-manager.js | 6 +-- src/server/auth/lib/userdb.js | 11 ++-- src/server/common/auth-manager.js | 82 ++++++++++++++---------------- src/server/common/cache.js | 4 +- src/server/common/log-manager.js | 2 +- src/server/conf/default-conf.js | 11 ++-- src/server/fs/lib/fs-manager.js | 7 ++- src/server/package.json | 36 ++++++------- 9 files changed, 81 insertions(+), 84 deletions(-) diff --git a/src/server/auth/auth.js b/src/server/auth/auth.js index da40fb3..df9bc84 100644 --- a/src/server/auth/auth.js +++ b/src/server/auth/auth.js @@ -104,10 +104,10 @@ var register = function (auth, conf, unitName, svcType) { auth.use(user.router); auth.use(acl.router); auth.use(group.router); - auth.use(function(err, req, res) { - logger.debug('errorHandler middleware', err); + auth.use(function(err, req, res, next) { + logger.error('errorHandler middleware', err); res.status(500).send('Internal server error'); - }); + }); auth.disable('x-powered-by'); }; diff --git a/src/server/auth/lib/acl-manager.js b/src/server/auth/lib/acl-manager.js index c144fa4..8de30d4 100644 --- a/src/server/auth/lib/acl-manager.js +++ b/src/server/auth/lib/acl-manager.js @@ -689,7 +689,7 @@ router.get('/webida/api/acl/getownedpolicy', } ); -// req.query : {uid:int, action:string, rsc:[string]} +// req.query : {uid:int, action:string, rsc:string;string;string} router.get('/checkauthorize', function (req, res, next) { var query = req.query; @@ -698,7 +698,7 @@ router.get('/checkauthorize', resources = query.rsc.split(';'); } if (resources.length > 0 ) { - logger.debug('check authorize with multple resources : ', resources); + logger.debug('check authorize for : ', resources); } async.each(resources, function(resource, callback) { var aclInfo = {uid:query.uid, action:query.action, rsc:resource}; @@ -714,7 +714,7 @@ router.get('/checkauthorize', errLog('checkAuthroze error - will return not authorized.', err); return res.send(401, utils.fail('Not authorized.')); } else { - return next(); + return res.sendok(); } }); } diff --git a/src/server/auth/lib/userdb.js b/src/server/auth/lib/userdb.js index 7d81239..eaaf112 100644 --- a/src/server/auth/lib/userdb.js +++ b/src/server/auth/lib/userdb.js @@ -1432,14 +1432,19 @@ exports.checkAuthorize = function (aclInfo, callback) { }, function (next) { var policy; var allowed = false; - dao.policy.getPolicyBySubjectIdsAndResources({subjectIds: idArr, resources: rscArr}, + var daoRequest = { + subjectIds : idArr, + resources : rscArr, + } + logger.debug('getPolicyBySubjectIdAndResources - policy dao request', daoRequest) + dao.policy.getPolicyBySubjectIdsAndResources(daoRequest, function (err, context) { var i; var result = context.result(); if (err) { next(new ServerError(500, 'Server error while check authorization.')); } else { - console.log('getPolicyBySubjectIdAndResources: ', idArr, rscArr, result); + logger.debug('getPolicyBySubjectIdAndResources - result = ', result) for (i = 0; i < result.length; i++) { policy = result[i]; if ((policy.action.indexOf(aclInfo.action) > -1) || (policy.action.indexOf('*') > -1)) { @@ -1467,7 +1472,7 @@ exports.checkAuthorize = function (aclInfo, callback) { return callback(err); } else { logger.info('[acl] checkAuthorize allowed for ', aclInfo); - return callback(); + return callback(null); } }); }; diff --git a/src/server/common/auth-manager.js b/src/server/common/auth-manager.js index 8c440bd..6cdaeb9 100644 --- a/src/server/common/auth-manager.js +++ b/src/server/common/auth-manager.js @@ -52,13 +52,13 @@ function _checkExpired(token, info, callback) { current = new Date().getTime(); expire = new Date(info.expireTime).getTime(); - logger.debug('_checkExpired', current, info, expire); + logger.debug('_checkExpired', current, expire); if (expire - current < 0) { logger.debug('token expired - callback with 419'); return callback(419); } else { logger.debug ('saving token to cache ' + token, info); - tokenCache.write(token).then( () => { + tokenCache.write(token,info).then( () => { return callback(null, info); }).catch( err => { logger.warn('cound not save token in cache ' + token, err); @@ -85,11 +85,11 @@ function _requestTokenInfo(token, callback) { if (res.statusCode === 200) { try { tokenInfo = JSON.parse(body).data; + return _checkExpired(token, tokenInfo, callback); } catch (e) { logger.error('Invalid oauth/verify response: ' + body, e); return callback(500); } - return _checkExpired(token, tokenInfo, callback); } else if (res.statusCode === 419) { return callback(419); } else { @@ -102,11 +102,9 @@ function _requestTokenInfo(token, callback) { logger.info('res', res.statusCode); res.setEncoding('utf8'); res.on('data', function (chunk) { - logger.debug('data chunk', chunk); data += chunk; }); res.on('end', function () { - logger.debug('end', data); handleResponse(null, res, data); }); }); @@ -121,10 +119,12 @@ function _verifyToken(token, callback) { logger.debug('token is null'); return callback(400); } - authroizationCache.read(token).then(value => { + tokenCache.read(token).then(value => { if (value) { - return callback(value); + logger.debug('token check passed by cache for ' + token); + return callback(null,value); } else { + logger.debug('token cache seems to be lost for ' + token); _requestTokenInfo(token, callback); } }).catch(err => { @@ -150,9 +150,11 @@ function getUserInfo(req, res, next) { _verifyToken(token, function (err, info) { var errMsg = 'Internal server error'; if (err) { - logger.debug('_verifyToken failed ', err); + logger.error('_verifyToken failed ', err); if (err === 400 || err === 419) { if (allowAnonymous) { + logger.debug('_verifyToken added empty user in req'); + req.user = {} return next(); } errMsg = (err === 400) ? 'requires access token' : 'invalid access token'; @@ -188,7 +190,6 @@ function _sendCheckAuthorizeRequest(options, cacheKey, res, next) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('checkAuthorize data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -201,7 +202,6 @@ function _sendCheckAuthorizeRequest(options, cacheKey, res, next) { } next(); }); - return next(); } else if (response.statusCode === 401) { // TODO : auth server should return 400 / 404 for invalid request // then We can use cache to save 'forbidden' access for 401 response @@ -225,47 +225,45 @@ function _createAuthorizationCacheKey(aclInfo) { return ret; } -function _buildAuthorizeRequest(rawRequest) { +function _buildAuthorizeRequest(rawRequest, fsid) { function _normalizeFileSystemResources(rawResource) { var rawResourceArray = rawResource.split(';'); var resources = []; rawResourceArray.forEach(function (resource) { - // fs:/abc/def/gef --> /abc/def/gef - // defs --> /defs - // fs:opqr/stuv --> /opqr/stuv - var path = '', arr; + // fs:AAXXCCDD/abc/def/gef --> fs:AAXXCCDD/abc + // fs:/AAXXCCDD/abc/def/gef --> fs:AAXXCCDD/abc + // AAXXCCDD/defs --> fs:AAXXCCDD/defs + // /AAXXCCDD/wert --> fs:AAXXCCDD/wert + // XXYYZZ/ -> fs:XXYYZZ/ + // / -> fs:/ + let path = ''; if (resource.indexOf('fs:') === 0) { if (resource[3] === '/') { - path = resource.slice(3); + path = resource.slice(4); } else { - path = '/' + resource.slice(3); + path = resource.slice(3); } } else { if (resource[0] === '/') { - path = resource; + path = resource.slice(1); } else { - path = '/' + resource; + path = resource; } } // should find first non-empty element - if (path === '/') { - path = 'fs:/'; + let normalized = 'fs:' + (fsid ? fsid : ''); + let arr = path.split('/'); + if(arr.length >= 2) { + arr = arr.slice(0,2); + normalized += arr.join('/'); } else { - arr = path.split('/'); - path = undefined; - for (var i = 1; i < arr.length; i++) { - path = arr[i]; - if (path) { - path = 'fs:/' + path; - break; - } - } + normalized += arr[0]; } - if (path && resources.indexOf(path) < 0) { - resources.push(path); + if (resources.indexOf(normalized) < 0) { + resources.push(normalized); } }); - return resources; + return resources.join(';'); } var ret = { uid: rawRequest.uid, @@ -273,7 +271,7 @@ function _buildAuthorizeRequest(rawRequest) { rsc: rawRequest.rsc }; // when fsid is set, rsc should be normalized - if (rawRequest.fsid || rawRequest.action.indexOf('fs') === 0) { + if (rawRequest.fsid || rawRequest.rsc.indexOf('fs:') === 0) { ret.rsc = _normalizeFileSystemResources(rawRequest.rsc); } return ret; @@ -282,8 +280,7 @@ function _buildAuthorizeRequest(rawRequest) { function checkAuthorize(aclInfo, res, next) { var authorizeRequest = _buildAuthorizeRequest(aclInfo); - var cacheKey = _createAuthorizationCacheKey(aclInfo); - + var cacheKey = _createAuthorizationCacheKey(authorizeRequest); var template = _.template('/checkauthorize?uid=<%= uid %>&action=<%= action %>&rsc=<%= rsc %>'); var options = { hostname: internalAccessInfo.auth.host, @@ -291,12 +288,14 @@ function checkAuthorize(aclInfo, res, next) { path: encodeURI(template(authorizeRequest)) }; - logger.debug('checkAuthorize : ', authorizeRequest); + logger.debug('checkAuthorize request: ', { raw: aclInfo, actual: authorizeRequest } ); // TODO : split cache items for multiple file system resources // - auth server should response which resource is accessible and whichi is not. // - then, we can cache each resource into differnt cache item. authroizationCache.read(cacheKey).then(value => { - if(value === true) { + logger.debug("checkAuthorization : cache check " + cacheKey, value); + if (value) { + logger.debug("checkAuthorization : passed by cache"); next(); } else { _sendCheckAuthorizeRequest(options, cacheKey, res, next); @@ -333,7 +332,6 @@ function createPolicy(policy, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('createPolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -376,7 +374,6 @@ function deletePolicy(pid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('deletePolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -410,7 +407,6 @@ function assignPolicy(id, pid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('assignPolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -445,7 +441,6 @@ function removePolicy(pid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('removePolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -483,7 +478,6 @@ function getPolicy(policyRule, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('getPolicy data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -532,7 +526,6 @@ function updatePolicyResource(oldPath, newPath, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('updatePolicyResource data chunk', chunk); data += chunk; }); response.on('end', function () { @@ -570,7 +563,6 @@ function getFSInfo(fsid, token, callback) { var data = ''; response.setEncoding('utf8'); response.on('data', function (chunk) { - logger.debug('getFSInfo data chunk', chunk); data += chunk; }); response.on('end', function () { diff --git a/src/server/common/cache.js b/src/server/common/cache.js index 0f090e6..c45661e 100644 --- a/src/server/common/cache.js +++ b/src/server/common/cache.js @@ -102,7 +102,7 @@ Cache.prototype = { var ret = new Promise(function(resolve, reject) { self.redis.get(key).then( function(value) { var deserialized = value; - if(typeof(value) === 'object') { + if (value) { deserialized = JSON.parse(value); } logger.debug(self.name + ' read', { key : key, value : value}); @@ -185,4 +185,4 @@ module.exports = { } }); } -}; \ No newline at end of file +}; diff --git a/src/server/common/log-manager.js b/src/server/common/log-manager.js index 8e7ddb0..76c6444 100644 --- a/src/server/common/log-manager.js +++ b/src/server/common/log-manager.js @@ -39,7 +39,7 @@ moduleFileName = path.basename(moduleFileName, '.js'); var logFileName = config.logPath + '/' + moduleFileName + '.log'; function curTime() { - return dateFormat(new Date(), 'yyyy-mm-dd hh:MM:ss-l'); + return dateFormat(new Date(), 'yyyy-mm-dd HH:MM:ss-l'); } var logger = null; diff --git a/src/server/conf/default-conf.js b/src/server/conf/default-conf.js index 4d073e4..7644d36 100755 --- a/src/server/conf/default-conf.js +++ b/src/server/conf/default-conf.js @@ -632,18 +632,15 @@ function checkConfiguration(conf) { checkDirExists(conf.logPath, 'conf.logPath'); - if (conf.signup.allowSignup) { - if(conf.signup.emailHost === 'your.smtp.server') { - console.warn('conf.signup.emailHost is not configured. New users will not receive an activation mail.'); + if (conf.services.auth.signup.allowSignup) { + if(conf.services.auth.signup.emailHost === 'your.smtp.server') { + console.warn('conf.services.auth.signup.emailHost is not configured. New users will not receive an activation mail.'); } } // TODO : add more configuration properties if (conf.services.fs.container.type === 'lxc') { - checkFileExists(conf.logPath, 'conf.services.fs.container.lxc.confPath'); - if (conf.services.fs.container.lxc.rootfsPath) { - checkFileExists(conf.logPath, 'conf.services.fs.container.lxc.rootfsPath'); - } + checkFileExists(conf.services.fs.container.lxc.confPath, 'conf.services.fs.container.lxc.confPath'); } } diff --git a/src/server/fs/lib/fs-manager.js b/src/server/fs/lib/fs-manager.js index 45e9a21..f4bc703 100644 --- a/src/server/fs/lib/fs-manager.js +++ b/src/server/fs/lib/fs-manager.js @@ -1578,13 +1578,16 @@ router.post('/webida/api/fs/file/:fsid/*', } db.lock.$findOne({wfsId:fsid, path:path}, function(err, context) { var lock = context.result(); - logger.info('writeFile check lock', err, lock); + logger.debug('writeFile check lock - read lock object from db = ', lock); if (err) { + logger.error('writeFile locking error ', err) return res.sendfail(err, 'Failed to write file.(failed to get lock info)'); } else if (lock && req.user.userId !== lock.userId && !req.user.isAdmin) { return res.sendfail(new ClientError(409, 'Specified file is locked by'+lock.email)); + } else { + logger.debug('writeFile check lock - not locked'); + return next(); } - return next(); }); }, function (req, res) { diff --git a/src/server/package.json b/src/server/package.json index f9ef30e..38644e3 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -7,18 +7,10 @@ "url": "git://github.com/webida/webida-server.git" }, "dependencies": { - "async": "*", - "data-mapper": "*", - "jquery": "*", - "optimist": "*", - "url": "*", - "underscore": "*", - "connect-domain": "*", - "querystring": "*", - "request": "*", + "async": "^1.5.0", "body-parser": "1.9.2", - "clone" : "^1.0.2", "compression": "1.2.0", + "connect-domain": "*", "connect-ensure-login": "0.1.x", "connect-multiparty": "1.2.5", "connect-sqlite3": "0.9.5", @@ -26,12 +18,13 @@ "corser": "2.0.0", "cron": "^1.0.9", "cuid": "1.2.1", + "data-mapper": "*", "dateformat": "1.0.11", - "dnode": "1.2.0", + "dnode": "^1.2.2", "ejs": "0.7.x", "express": "4.10.2", "express-session": "1.9.1", - "ffi": "1.2.6", + "ffi": "^2.0.0", "filequeue": "0.5.0", "formidable": "1.0.17", "fs-extra": "0.8.1", @@ -40,11 +33,11 @@ "graceful-fs": "3.0.2", "guid": "0.0.12", "hashmap": "1.0.1", - "hiredis" : "^0.4.1", + "hiredis" : "^0.4.1" "http-master": "~1.0.18", "http-proxy": "0.10.3", - "ioredis" : "^1.10.0", - "jsdom": "0.10.6", + "jquery": "*", + "jsdom": "^7.0.2", "jsonfile": "~2.2.1", "line-reader": "0.2.3", "lodash": "3.9.3", @@ -56,6 +49,8 @@ "node-watch": "~0.3.4", "oauth2orize": "0.x.x", "on-finished": "2.3.0", + "optimist": "*", + "optimist": "0.6.0", "passport": "~0.1.12", "passport-github": "*", "passport-google-oauth": "*", @@ -63,17 +58,22 @@ "passport-http-bearer": "0.2.x", "passport-local": "0.1.x", "passport-oauth2-client-password": "0.1.x", - "pty.js": "0.2.7-1", + "pty.js": "^0.3.0", "q": "1.0.1", + "querystring": "*", + "redis" : "^2.3.0", "replace": "0.2.9", + "request": "*", "send": "0.2.0", "shortid": "2.2.2", - "socket.io": "1.2.0", + "socket.io": "^1.3.7", "tmp": "0.0.23", + "underscore": "^1.5.2", "unzip": "0.1.9", "URIjs": "1.12.1", + "url": "*", "walkdir": "0.0.7", - "winston": "~1.1.2" + "winston": "~1.1.2", }, "devDependencies": { "q-io": "1.11.0", From 3ecffbcffbe54747f2d5f00b60d5d9758fff861a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=91=ED=9B=88?= Date: Thu, 19 Nov 2015 00:00:22 +0900 Subject: [PATCH 6/8] changed .jshintrc files for ES6 grammar - changed auth & common - we may need to change other modules, later --- src/server/auth/.jshintrc | 4 +--- src/server/common/.jshintrc | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/server/auth/.jshintrc b/src/server/auth/.jshintrc index 3f3110e..9877224 100644 --- a/src/server/auth/.jshintrc +++ b/src/server/auth/.jshintrc @@ -17,7 +17,7 @@ "define", "require" ], - + "exnext" : true, "bitwise": true, "curly": true, "eqeqeq": true, @@ -33,10 +33,8 @@ "unused": true, "strict": true, "trailing": true, - "camelcase": true, "indent": 4, "maxlen": 120, - "quotmark": "single", "white": true } diff --git a/src/server/common/.jshintrc b/src/server/common/.jshintrc index 3f3110e..2574f7a 100644 --- a/src/server/common/.jshintrc +++ b/src/server/common/.jshintrc @@ -13,11 +13,10 @@ "waits", "waitsFor", "runs", - "define", "require" ], - + "exnext" : true, "bitwise": true, "curly": true, "eqeqeq": true, @@ -33,10 +32,8 @@ "unused": true, "strict": true, "trailing": true, - "camelcase": true, "indent": 4, "maxlen": 120, - "quotmark": "single", "white": true } From cc9097ff6a7080bb22e5f7f43d6c861fc493ce7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=91=ED=9B=88?= Date: Thu, 19 Nov 2015 00:10:07 +0900 Subject: [PATCH 7/8] fixed package.json syntax error - missed ',' after sorting dependencies. sorry. --- src/server/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/package.json b/src/server/package.json index 38644e3..526441c 100644 --- a/src/server/package.json +++ b/src/server/package.json @@ -33,7 +33,7 @@ "graceful-fs": "3.0.2", "guid": "0.0.12", "hashmap": "1.0.1", - "hiredis" : "^0.4.1" + "hiredis" : "^0.4.1", "http-master": "~1.0.18", "http-proxy": "0.10.3", "jquery": "*", @@ -73,7 +73,7 @@ "URIjs": "1.12.1", "url": "*", "walkdir": "0.0.7", - "winston": "~1.1.2", + "winston": "~1.1.2" }, "devDependencies": { "q-io": "1.11.0", From d8ae6603a54fc3b71229fd36ca04be5750408348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EC=A4=91=ED=9B=88?= Date: Thu, 19 Nov 2015 00:52:47 +0900 Subject: [PATCH 8/8] fixed typo on .jshintrc files --- src/server/auth/.jshintrc | 2 +- src/server/common/.jshintrc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/auth/.jshintrc b/src/server/auth/.jshintrc index 9877224..562dfb3 100644 --- a/src/server/auth/.jshintrc +++ b/src/server/auth/.jshintrc @@ -17,7 +17,7 @@ "define", "require" ], - "exnext" : true, + "esnext" : true, "bitwise": true, "curly": true, "eqeqeq": true, diff --git a/src/server/common/.jshintrc b/src/server/common/.jshintrc index 2574f7a..0744a33 100644 --- a/src/server/common/.jshintrc +++ b/src/server/common/.jshintrc @@ -16,7 +16,7 @@ "define", "require" ], - "exnext" : true, + "esnext" : true, "bitwise": true, "curly": true, "eqeqeq": true,