From c7a405c28a9f880a1c8cb49e9c0cbf22c2529472 Mon Sep 17 00:00:00 2001 From: Adam Burgess Date: Fri, 15 Jul 2016 00:36:17 +1000 Subject: [PATCH 01/23] replace exec-git.js with native node implementation of git. --- exec-git.js | 66 ----------------------------------------- github.js | 84 +++++++++++++++++++--------------------------------- ls-remote.js | 65 ++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- 4 files changed, 96 insertions(+), 122 deletions(-) delete mode 100644 exec-git.js create mode 100644 ls-remote.js diff --git a/exec-git.js b/exec-git.js deleted file mode 100644 index 1bd6edf..0000000 --- a/exec-git.js +++ /dev/null @@ -1,66 +0,0 @@ -var Promise = require('bluebird').Promise; -var exec = require('child_process').exec; -var os = require('os'); - -function Pool(count) { - this.count = count; - this.queue = []; - this.promises = new Array(count); -} - -/* Run the function immediately. */ -function run(pool, idx, executionFunction) { - var p = Promise.resolve() - .then(executionFunction) - .then(function() { - delete pool.promises[idx]; - var next = pool.queue.pop(); - if (next) - pool.execute(next); - }); - pool.promises[idx] = p; - return p; -} - -/* Defer function to run once all running and queued functions have run. */ -function enqueue(pool, executeFunction) { - return new Promise(function(resolve) { - pool.queue.push(function() { - return Promise.resolve().then(executeFunction).then(resolve); - }); - }); -} - -/* Take a function to execute within pool, and return promise delivering the functions - * result immediately once it is run. */ -Pool.prototype.execute = function(executionFunction) { - var idx = -1; - - for (var i=0; i Date: Sun, 24 Jul 2016 16:59:44 +1000 Subject: [PATCH 02/23] remove execOpt it isn't needed anymore --- github.js | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/github.js b/github.js index 7f036a0..a942a05 100644 --- a/github.js +++ b/github.js @@ -101,22 +101,10 @@ var GithubLocation = function(options, ui) { this.ui = ui; - this.execOpt = { - cwd: options.tmpDir, - timeout: options.timeout * 1000, - killSignal: 'SIGKILL', - maxBuffer: this.max_repo_size || 2 * 1024 * 1024, - env: extend({}, process.env) - }; - this.defaultRequestOptions = { strictSSL: 'strictSSL' in options ? options.strictSSL : true }; - if (!this.defaultRequestOptions.strictSSL) { - this.execOpt.env.GIT_SSL_NO_VERIFY = '1' - } - var self = this, envMap = { ca: 'GIT_SSL_CAINFO', cert: 'GIT_SSL_CERT', @@ -126,7 +114,6 @@ var GithubLocation = function(options, ui) { ['ca', 'cert', 'key'].forEach(function(key) { if (key in options) { var path = expandTilde(options[key]); - self.execOpt.env[envMap[key]] = path; self.defaultRequestOptions[key] = fs.readFileSync(path, 'ascii'); } }); @@ -510,7 +497,6 @@ GithubLocation.prototype = { if (meta.vPrefix) version = 'v' + version; - var execOpt = this.execOpt; var max_repo_size = this.max_repo_size; var remoteString = this.remoteString; var authSuffix = this.authSuffix; From f521dd2d389d6868e4f436e445bd86c2ed286041 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Sun, 24 Jul 2016 17:34:34 +1000 Subject: [PATCH 03/23] make extend work with objects recursively --- github.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/github.js b/github.js index a942a05..362d2d5 100644 --- a/github.js +++ b/github.js @@ -15,7 +15,8 @@ var semver = require('semver'); function extend(dest, src) { for (var key in src) { - dest[key] = src[key] + if(key in dest && typeof dest[key] === 'object') extend(dest[key], src[key]); + else dest[key] = src[key] } return dest; From bb4818568806aed62e3984bf726eaceb7fe63718 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Sun, 24 Jul 2016 18:29:36 +1000 Subject: [PATCH 04/23] make sure all requests set jspm as user-agent and respect strictSSL --- github.js | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/github.js b/github.js index 362d2d5..c4c8c97 100644 --- a/github.js +++ b/github.js @@ -103,6 +103,9 @@ var GithubLocation = function(options, ui) { this.ui = ui; this.defaultRequestOptions = { + headers: { + 'User-Agent': 'jspm' + }, strictSSL: 'strictSSL' in options ? options.strictSSL : true }; @@ -279,12 +282,8 @@ GithubLocation.prototype = { return new Promise(function(resolve, reject) { request(extend({ uri: remoteString + repo + authSuffix, - headers: { - 'User-Agent': 'jspm' - }, followRedirect: false - }, self.defaultRequestOptions - )) + }, self.defaultRequestOptions)) .on('response', function(res) { // redirect if (res.statusCode == 301) @@ -315,8 +314,11 @@ GithubLocation.prototype = { // { versions: { versionhash } } // { notfound: true } lookup: function(repo) { + var self = this; var remoteString = this.remoteString; - return lsRemote(remoteString + repo + '.git') + return lsRemote(extend({ + url: remoteString + repo + '.git' + }, self.defaultRequestOptions)) .then(function(refs) { var versions = {}; refs.forEach(function(ref) { @@ -348,7 +350,10 @@ GithubLocation.prototype = { }, function(error) { // 401 does not mean authentication failure in this instance if (error.statusCode && (error.statusCode == 401 || error.statusCode == 404)) return { notfound: true }; - if (error.statusCode) error = new Error('github returned ' + error.statusCode); + if (error.statusCode) error = new Error('Invalid status code when fetching versions: ' + error.statusCode); + else if(typeof error == 'string') { + error = new Error(error); + } error.retriable = true; error.hideStack = true; throw error; @@ -364,17 +369,16 @@ GithubLocation.prototype = { var self = this; var ui = this.ui; - return asp(request)({ + return asp(request)(extend({ uri: this.apiRemoteString + 'repos/' + repo + '/contents/package.json' + this.authSuffix, headers: { - 'User-Agent': 'jspm', 'Accept': 'application/vnd.github.v3.raw' }, qs: { ref: version - }, - strictSSL: this.defaultRequestOptions.strictSSL - }).then(function(res) { + } + }, self.defaultRequestOptions)) + .then(function(res) { // API auth failure warnings function apiFailWarn(reason, showAuthCommand) { if (apiWarned) @@ -506,11 +510,10 @@ GithubLocation.prototype = { // Download from the git archive return new Promise(function(resolve, reject) { - request({ + request(extend({ uri: remoteString + repo + '/archive/' + version + '.tar.gz' + authSuffix, - headers: { 'accept': 'application/octet-stream' }, - strictSSL: self.defaultRequestOptions.strictSSL - }) + headers: { 'accept': 'application/octet-stream' } + }, self.defaultRequestOptions)) .on('response', function(pkgRes) { if (pkgRes.statusCode != 200) return reject('Bad response code ' + pkgRes.statusCode); From ad1fa95faba815430e3a94778138c51acbb642e0 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Sun, 24 Jul 2016 19:47:26 +1000 Subject: [PATCH 05/23] change ls-remote.js's argument to use a request object --- ls-remote.js | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ls-remote.js b/ls-remote.js index ab1d8c6..402f0f1 100644 --- a/ls-remote.js +++ b/ls-remote.js @@ -1,35 +1,35 @@ var request = require('request'); -module.exports = function(repo, cb) { - var opts = { - url: repo + '/info/refs?service=git-upload-pack', - encoding: null, - headers: { 'user-agent': 'jspm' } - }; + +module.exports = function(opts, cb) { + opts.encoding = null; + opts.url += '/info/refs?service=git-upload-pack'; + request(opts, function(err, response, body) { // network errors if (err) return cb(err); - if (response.statusCode != 200) return cb({ statusCode: response.statusCode }); - if (response.headers['content-type'] != 'application/x-git-upload-pack-advertisement') return cb({ statusCode: 500 }); + if (response.statusCode != 200) return cb({ statusCode: response.statusCode, headers: response.headers }); + // per the specification, content-type must be this + if (response.headers['content-type'] != 'application/x-git-upload-pack-advertisement') return cb({ statusCode: 500, headers: response.statusCode }); var lines = []; var pos = 0; while (pos < body.length) { // read 4 bytes and parse line length in hex - var lineLength = parseInt(body.toString('ascii', pos, pos + 4), 16); + var lineLength = parseInt(body.toString('utf8', pos, pos + 4), 16); // if line isn't blank, read it, otherwise read the next line if (lineLength != 0) { - var line = body.toString('ascii', pos + 4, pos += lineLength); + var line = body.toString('utf8', pos + 4, pos += lineLength); lines.push(line.trim()); } else pos += 4; } // verify the first line is git-upload-pack - if( lines.shift() != '# service=git-upload-pack') return cb({ statusCode: 500 }); + if( lines.shift() != '# service=git-upload-pack') return cb({ statusCode: 500, headers: response.headers }); // verify the second line is the HEAD // e.g. 21d0a9aa00806bb7f67ef5cd98c876aa20e4d803 HEAD\u0000multi_ack thin-pack [...] - if (lines.shift().substr(41, 4) != 'HEAD') return cb({ statusCode: 500 }); + if (lines.shift().substr(41, 4) != 'HEAD') return cb({ statusCode: 500, headers: response.headers }); // parse the remaining lines var refs = lines.map(function(line) { From 128ced4e345367e68a5f1fb051147e9d5ae6d5db Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Wed, 27 Jul 2016 12:19:32 +1000 Subject: [PATCH 06/23] add github api to locate --- github.js | 67 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/github.js b/github.js index c4c8c97..0a6ee8d 100644 --- a/github.js +++ b/github.js @@ -316,9 +316,41 @@ GithubLocation.prototype = { lookup: function(repo) { var self = this; var remoteString = this.remoteString; - return lsRemote(extend({ - url: remoteString + repo + '.git' - }, self.defaultRequestOptions)) + + return Promise.resolve() + .then(function() { + if(this.auth && this.auth.token) { + // use API to get branches/tags + return Promise.all(['tags', 'heads'].map(function(type) { + return asp(request)(extend({ + uri: this.apiRemoteString + 'repos/' + repo + '/git/refs/' + type + this.authSuffix, + headers: { + 'Accept': 'application/vnd.github.v3.raw' + }, + qs: { + ref: version + } + }, self.defaultRequestOptions)); + })).then(function(tagRes, headRes) { + if (tagRes.statusCode != 200) + throw { statusCode: tagRes.statusCode, headers: tagRes.headers, api: true }; + else if (headRes.statusCode != 200) + throw { statusCode: headRes.statusCode, headers: tagRes.headers, api: true }; + + var tags = JSON.parse(tagRes.body); + var heads = JSON.parse(headRes.body); + + return tags.concat(heads).map(function(obj) { + return { hash: obj.object.sha, name: obj.ref }; + }); + }); + } else { + // fallback to git-based approach + return lsRemote(extend({ + url: remoteString + repo + '.git' + }, self.defaultRequestOptions)); + } + }) .then(function(refs) { var versions = {}; refs.forEach(function(ref) { @@ -347,13 +379,32 @@ GithubLocation.prototype = { }); return { versions: versions }; - }, function(error) { - // 401 does not mean authentication failure in this instance - if (error.statusCode && (error.statusCode == 401 || error.statusCode == 404)) return { notfound: true }; - if (error.statusCode) error = new Error('Invalid status code when fetching versions: ' + error.statusCode); - else if(typeof error == 'string') { + } + .catch(function(error) { + if (error.statusCode) { + var headerSuffix = '\n' + JSON.stringify(error.headers, null, 2)); + + if (error.api && (error.statusCode == 406 || error.statusCode == 401)) { + if (error.api) { + hasApiFailure + // TODO: replace this with the api failure response + error = new Error('api says invalid auth: ' error.statusCode + headerSuffix); + } + else + error = new Error('Invalid authentication details.\n' + + 'Run %jspm registry config ' + self.name + '% to reconfigure the credentials, or update them in your ~/.netrc file.'; + } + } + else if (error.statusCode == 404) + return { notfound: true }; + else + error = new Error('invalid status code: ' + error.statusCode + headerSuffix); + } + + if(typeof error == 'string') { error = new Error(error); } + error.retriable = true; error.hideStack = true; throw error; From 3c055914ec9b4479ffded82d9faa86c25f085cb6 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 10:43:05 +1000 Subject: [PATCH 07/23] update semver ranges to caret --- package.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8493c7e..a6ad3bb 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,7 @@ "description": "jspm GitHub endpoint", "main": "github.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "test": "mocha" }, "repository": { "type": "git", @@ -19,15 +19,15 @@ "expand-tilde": "^1.2.0", "bluebird": "^3.0.5", "graceful-fs": "^4.1.3", - "mkdirp": "~0.5.0", + "mkdirp": "^0.5.0", "netrc": "^0.1.3", - "request": "~2.53.0", - "rimraf": "~2.3.2", + "request": "^2.53.0", + "rimraf": "^2.3.2", "semver": "^5.0.1", "tar-fs": "^1.13.0" }, "homepage": "https://github.com/jspm/github", "devDependencies": { - "mocha": "~2.2.4" + "mocha": "^2.2.4" } } From ed544198092c6f000da05349893792af50963320 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 10:47:43 +1000 Subject: [PATCH 08/23] remove temporary code that was committed --- github.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/github.js b/github.js index 0a6ee8d..2b9c8a0 100644 --- a/github.js +++ b/github.js @@ -384,9 +384,8 @@ GithubLocation.prototype = { if (error.statusCode) { var headerSuffix = '\n' + JSON.stringify(error.headers, null, 2)); - if (error.api && (error.statusCode == 406 || error.statusCode == 401)) { + if (error.statusCode == 406 || error.statusCode == 401) { if (error.api) { - hasApiFailure // TODO: replace this with the api failure response error = new Error('api says invalid auth: ' error.statusCode + headerSuffix); } From 31dbc1d1d5fa22033a5b2cb1d66c134da49fdcb7 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 10:51:48 +1000 Subject: [PATCH 09/23] add travis --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..d59881c --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +language: node_js +sudo: false +node_js: + - "4" + - "5" + - "6" \ No newline at end of file From ddfcb6109558872fd272dc2152b5e342a978c103 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 11:06:41 +1000 Subject: [PATCH 10/23] fix another syntax error --- github.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/github.js b/github.js index 2b9c8a0..a53c595 100644 --- a/github.js +++ b/github.js @@ -379,19 +379,19 @@ GithubLocation.prototype = { }); return { versions: versions }; - } + }) .catch(function(error) { if (error.statusCode) { - var headerSuffix = '\n' + JSON.stringify(error.headers, null, 2)); + var headerSuffix = '\n' + JSON.stringify(error.headers, null, 2); if (error.statusCode == 406 || error.statusCode == 401) { if (error.api) { // TODO: replace this with the api failure response - error = new Error('api says invalid auth: ' error.statusCode + headerSuffix); + error = new Error('api says invalid auth: ' + error.statusCode + headerSuffix); } - else + else { error = new Error('Invalid authentication details.\n' + - 'Run %jspm registry config ' + self.name + '% to reconfigure the credentials, or update them in your ~/.netrc file.'; + 'Run %jspm registry config ' + self.name + '% to reconfigure the credentials, or update them in your ~/.netrc file.'); } } else if (error.statusCode == 404) From 2d1f4ef8dc2e5d5a639b3e03cff3787715b3fd78 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 11:19:52 +1000 Subject: [PATCH 11/23] add some fairly simple tests --- test/mocha.opts | 1 - test/pool.js | 35 ----------------------------------- test/setup.js | 2 +- test/test.js | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 37 deletions(-) delete mode 100644 test/pool.js create mode 100644 test/test.js diff --git a/test/mocha.opts b/test/mocha.opts index 7b826c8..46191c0 100644 --- a/test/mocha.opts +++ b/test/mocha.opts @@ -1,4 +1,3 @@ --require test/setup.js ---reporter dot --ui tdd --timeout 120000 diff --git a/test/pool.js b/test/pool.js deleted file mode 100644 index e5aa3f5..0000000 --- a/test/pool.js +++ /dev/null @@ -1,35 +0,0 @@ -var Pool = require('../pool'); -var exec = require('child_process').exec; - -test('Controls concurrency', function(done) { - var pool = new Pool(5); - - var count = 0; - var counts = []; - function f() { - return new Promise(function(resolve) { - count++; - counts.push(count); - exec('sleep 0.01', function() { - count--; - resolve(); - }); - }); - } - var calls = []; - for (var i=0; i<100; i++) - calls[i] = pool.execute(f); - - Promise.all(calls) - .then(function() { - var i; - // burn in, number running == number executed - for (i=0; i<5; i++) - assert.equal(counts[i],i + 1); - // maxed out, number running == pool max - for (i=4; i<100; i++) - assert.equal(counts[i], 5); - }) - .then(done) - .catch(done); -}); diff --git a/test/setup.js b/test/setup.js index 4883550..8ee950b 100644 --- a/test/setup.js +++ b/test/setup.js @@ -1,2 +1,2 @@ -global.Promise = require('rsvp').Promise; +global.Promise = require('bluebird'); global.assert = require('assert'); diff --git a/test/test.js b/test/test.js new file mode 100644 index 0000000..70d93a6 --- /dev/null +++ b/test/test.js @@ -0,0 +1,46 @@ +var githubRegistry = require('../github'); + + + +suite('Github', function() { + var github; + setup(function() { + github = new githubRegistry({ + baseDir: '.', + log: true, + tmpDir: '.', + auth: '', + name: 'github' + }); + }); + + suite('locate', function() { + test('jspm/github', function() { + return github.locate('jspm/github', 'jspm/github exists'); + }); + + test('6to5/6to5', function() { + return github.locate('6to5/6to5').then(function(result) { + assert(result.redirect, 'has redirect'); + assert.equal(result.redirect, 'github:babel/babel', 'redirect points to babel/babel'); + }); + }); + + test('jspm/thisdoesnotexist', function() { + return github.locate('jspm/thisdoesnotexist').then(function() { + // todo: this should throw + // assert(false); + }); + }); + }); + + suite('lookup', function() { + test('jspm/github@0.13.16', function() { + var tag = '0.13.16'; + var sha = '21d0a9aa00806bb7f67ef5cd98c876aa20e4d803'; + return github.lookup('jspm/github').then(function(result) { + assert(result.versions[tag].hash === sha); + }); + }); + }); +}); \ No newline at end of file From 845ae9deb34ed22345085c516187d011bf376069 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 11:20:47 +1000 Subject: [PATCH 12/23] add more languages --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index d59881c..bb314c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,9 @@ language: node_js sudo: false node_js: + - "0.10" + - "0.11" + - "0.12" - "4" - "5" - "6" \ No newline at end of file From 4f9e30f9b6e037127e2723a973de98ba9483ba60 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 11:24:39 +1000 Subject: [PATCH 13/23] update test --- test/test.js | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/test/test.js b/test/test.js index 70d93a6..c40d9a3 100644 --- a/test/test.js +++ b/test/test.js @@ -1,19 +1,14 @@ var githubRegistry = require('../github'); - - suite('Github', function() { - var github; - setup(function() { - github = new githubRegistry({ - baseDir: '.', - log: true, - tmpDir: '.', - auth: '', - name: 'github' - }); + var github = new githubRegistry({ + baseDir: '.', + log: true, + tmpDir: '.', + auth: '', + name: 'github' }); - + suite('locate', function() { test('jspm/github', function() { return github.locate('jspm/github', 'jspm/github exists'); From 4451b3a2ce96b53f4f37ae878a303956b28acdd3 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 12:08:09 +1000 Subject: [PATCH 14/23] comments --- github.js | 1 + test/test.js | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/github.js b/github.js index a53c595..a207ca7 100644 --- a/github.js +++ b/github.js @@ -321,6 +321,7 @@ GithubLocation.prototype = { .then(function() { if(this.auth && this.auth.token) { // use API to get branches/tags + // TODO: fallback to git protocol if the API has been rate-limited return Promise.all(['tags', 'heads'].map(function(type) { return asp(request)(extend({ uri: this.apiRemoteString + 'repos/' + repo + '/git/refs/' + type + this.authSuffix, diff --git a/test/test.js b/test/test.js index c40d9a3..f646270 100644 --- a/test/test.js +++ b/test/test.js @@ -8,7 +8,7 @@ suite('Github', function() { auth: '', name: 'github' }); - + suite('locate', function() { test('jspm/github', function() { return github.locate('jspm/github', 'jspm/github exists'); @@ -23,7 +23,7 @@ suite('Github', function() { test('jspm/thisdoesnotexist', function() { return github.locate('jspm/thisdoesnotexist').then(function() { - // todo: this should throw + // todo: this should throw if using the api // assert(false); }); }); From 40b887ef5d2cecf176d70141c528064a0c395f1a Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 12:23:35 +1000 Subject: [PATCH 15/23] change code to allow fallback to git on API rate limit --- github.js | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/github.js b/github.js index a207ca7..c2e7038 100644 --- a/github.js +++ b/github.js @@ -319,9 +319,8 @@ GithubLocation.prototype = { return Promise.resolve() .then(function() { - if(this.auth && this.auth.token) { + if (this.auth && this.auth.token) { // use API to get branches/tags - // TODO: fallback to git protocol if the API has been rate-limited return Promise.all(['tags', 'heads'].map(function(type) { return asp(request)(extend({ uri: this.apiRemoteString + 'repos/' + repo + '/git/refs/' + type + this.authSuffix, @@ -347,11 +346,30 @@ GithubLocation.prototype = { }); } else { // fallback to git-based approach - return lsRemote(extend({ - url: remoteString + repo + '.git' - }, self.defaultRequestOptions)); + return false; } }) + .catch(function(e) { + if (e.headers['x-ratelimit-remaining'] == '0') { + if (!apiWarned) { + ui.log('API ratelimit reached, falling back to slower git protocol'); + apiWarned = true; + } + // fallback to git-based approach + return false; + } else { + throw res; + } + }) + .then(function(refs) { + // API response + if (refs) return refs; + + // no API auth or API auth is rate-limited, use git + return lsRemote(extend({ + url: remoteString + repo + '.git' + }, self.defaultRequestOptions)); + }) .then(function(refs) { var versions = {}; refs.forEach(function(ref) { @@ -387,7 +405,7 @@ GithubLocation.prototype = { if (error.statusCode == 406 || error.statusCode == 401) { if (error.api) { - // TODO: replace this with the api failure response + // TODO: replace this with the api failure response code from below error = new Error('api says invalid auth: ' + error.statusCode + headerSuffix); } else { From 0413f9050214d6c995022a92a51f07b3e14062ec Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 12:24:11 +1000 Subject: [PATCH 16/23] style change to match --- github.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github.js b/github.js index c2e7038..99fafcc 100644 --- a/github.js +++ b/github.js @@ -15,7 +15,7 @@ var semver = require('semver'); function extend(dest, src) { for (var key in src) { - if(key in dest && typeof dest[key] === 'object') extend(dest[key], src[key]); + if (key in dest && typeof dest[key] === 'object') extend(dest[key], src[key]); else dest[key] = src[key] } @@ -419,7 +419,7 @@ GithubLocation.prototype = { error = new Error('invalid status code: ' + error.statusCode + headerSuffix); } - if(typeof error == 'string') { + if (typeof error == 'string') { error = new Error(error); } From 0a45da6b6c7dae74f2029356900aea7d3d6d3874 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 12:34:12 +1000 Subject: [PATCH 17/23] add token support --- test/test.js | 63 +++++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/test/test.js b/test/test.js index f646270..f3ed80f 100644 --- a/test/test.js +++ b/test/test.js @@ -1,41 +1,48 @@ var githubRegistry = require('../github'); -suite('Github', function() { - var github = new githubRegistry({ - baseDir: '.', - log: true, - tmpDir: '.', - auth: '', - name: 'github' - }); +testSuits('Github', ''); +if(process.env.token) + testSuits('Github with token auth', process.env.token); + - suite('locate', function() { - test('jspm/github', function() { - return github.locate('jspm/github', 'jspm/github exists'); +function testSuits(name, auth) { + suite(name, function() { + var github = new githubRegistry({ + baseDir: '.', + log: true, + tmpDir: '.', + auth: auth, + name: 'github' }); - test('6to5/6to5', function() { - return github.locate('6to5/6to5').then(function(result) { - assert(result.redirect, 'has redirect'); - assert.equal(result.redirect, 'github:babel/babel', 'redirect points to babel/babel'); + suite('locate', function() { + test('jspm/github', function() { + return github.locate('jspm/github', 'jspm/github exists'); }); - }); - test('jspm/thisdoesnotexist', function() { - return github.locate('jspm/thisdoesnotexist').then(function() { - // todo: this should throw if using the api - // assert(false); + test('6to5/6to5', function() { + return github.locate('6to5/6to5').then(function(result) { + assert(result.redirect, 'has redirect'); + assert.equal(result.redirect, 'github:babel/babel', 'redirect points to babel/babel'); + }); + }); + + test('jspm/thisdoesnotexist', function() { + return github.locate('jspm/thisdoesnotexist').then(function() { + // todo: this should throw if using the api + // assert(false); + }); }); }); - }); - suite('lookup', function() { - test('jspm/github@0.13.16', function() { - var tag = '0.13.16'; - var sha = '21d0a9aa00806bb7f67ef5cd98c876aa20e4d803'; - return github.lookup('jspm/github').then(function(result) { - assert(result.versions[tag].hash === sha); + suite('lookup', function() { + test('jspm/github@0.13.16', function() { + var tag = '0.13.16'; + var sha = '21d0a9aa00806bb7f67ef5cd98c876aa20e4d803'; + return github.lookup('jspm/github').then(function(result) { + assert(result.versions[tag].hash === sha); + }); }); }); }); -}); \ No newline at end of file +} \ No newline at end of file From ae07e0dc9e56c1682a9e7169681483ffa1ac4ed0 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 12:37:15 +1000 Subject: [PATCH 18/23] fix locate when using token --- github.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/github.js b/github.js index 99fafcc..1e8e451 100644 --- a/github.js +++ b/github.js @@ -286,8 +286,11 @@ GithubLocation.prototype = { }, self.defaultRequestOptions)) .on('response', function(res) { // redirect - if (res.statusCode == 301) - resolve({ redirect: self.name + ':' + res.headers.location.split('/').splice(3).join('/') }); + if (res.statusCode == 301) { + // strip access token + var path = require('url').parse(res.headers.location).pathname; + resolve({ redirect: self.name + ':' + path.substr(1) }); + } if (res.statusCode == 401) reject('Invalid authentication details.\n' + From f2c1d4af2eaf3b2a0261700907c885a3089c87df Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 13:12:39 +1000 Subject: [PATCH 19/23] fix API path --- github.js | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/github.js b/github.js index 1e8e451..163d167 100644 --- a/github.js +++ b/github.js @@ -322,29 +322,35 @@ GithubLocation.prototype = { return Promise.resolve() .then(function() { - if (this.auth && this.auth.token) { + if (self.auth && self.auth.token) { // use API to get branches/tags return Promise.all(['tags', 'heads'].map(function(type) { return asp(request)(extend({ - uri: this.apiRemoteString + 'repos/' + repo + '/git/refs/' + type + this.authSuffix, + uri: self.apiRemoteString + 'repos/' + repo + '/git/refs/' + type + self.authSuffix, headers: { 'Accept': 'application/vnd.github.v3.raw' - }, - qs: { - ref: version } }, self.defaultRequestOptions)); - })).then(function(tagRes, headRes) { - if (tagRes.statusCode != 200) - throw { statusCode: tagRes.statusCode, headers: tagRes.headers, api: true }; - else if (headRes.statusCode != 200) - throw { statusCode: headRes.statusCode, headers: tagRes.headers, api: true }; + })).then(function(responses) { + var tagRes = responses[0]; + var headRes = responses[1]; + + var refs = []; - var tags = JSON.parse(tagRes.body); - var heads = JSON.parse(headRes.body); + // there should always be heads + if (headRes.statusCode != 200) + throw { statusCode: headRes.statusCode, headers: headRes.headers, api: true }; + else + refs = refs.concat(JSON.parse(headRes.body)); + + // tag response can be 404, i.e. no tags + if (tagRes.statusCode == 200) + refs = refs.concat(JSON.parse(tagRes.body)); + else if (tagRes.statusCode != 404) + throw { statusCode: tagRes.statusCode, headers: tagRes.headers, api: true }; - return tags.concat(heads).map(function(obj) { - return { hash: obj.object.sha, name: obj.ref }; + return refs.map(function(obj) { + return { sha: obj.object.sha, name: obj.ref }; }); }); } else { @@ -353,7 +359,7 @@ GithubLocation.prototype = { } }) .catch(function(e) { - if (e.headers['x-ratelimit-remaining'] == '0') { + if (e.headers && e.headers['x-ratelimit-remaining'] == '0') { if (!apiWarned) { ui.log('API ratelimit reached, falling back to slower git protocol'); apiWarned = true; @@ -361,7 +367,7 @@ GithubLocation.prototype = { // fallback to git-based approach return false; } else { - throw res; + throw e; } }) .then(function(refs) { From fd64ea430404712fee7447a3f67f2a556d982853 Mon Sep 17 00:00:00 2001 From: AdamBurgess Date: Tue, 2 Aug 2016 13:16:50 +1000 Subject: [PATCH 20/23] add user auth test as well --- test/test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test.js b/test/test.js index f3ed80f..8dac109 100644 --- a/test/test.js +++ b/test/test.js @@ -1,9 +1,10 @@ var githubRegistry = require('../github'); testSuits('Github', ''); -if(process.env.token) +if (process.env.token) testSuits('Github with token auth', process.env.token); - +if (process.env.ghuser) + testSuits('Github with user:token auth', new Buffer(process.env.ghuser + ':' + process.env.token).toString('base64')); function testSuits(name, auth) { suite(name, function() { From cb7cac9af04308d07caf03f9e43498beea46b2b6 Mon Sep 17 00:00:00 2001 From: Adam Burgess Date: Tue, 2 Aug 2016 23:55:19 +1000 Subject: [PATCH 21/23] Revert "add more languages" This reverts commit 845ae9deb34ed22345085c516187d011bf376069. --- .travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index bb314c8..d59881c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,6 @@ language: node_js sudo: false node_js: - - "0.10" - - "0.11" - - "0.12" - "4" - "5" - "6" \ No newline at end of file From c395d2b5964e8fe427c37be8ff3cbb436e814190 Mon Sep 17 00:00:00 2001 From: Adam Burgess Date: Tue, 2 Aug 2016 23:56:39 +1000 Subject: [PATCH 22/23] promisify ls-remote --- github.js | 2 +- ls-remote.js | 30 +++++++++++++++++------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/github.js b/github.js index 163d167..8bcbc84 100644 --- a/github.js +++ b/github.js @@ -27,7 +27,7 @@ try { } catch(e) {} -var lsRemote = asp(require('./ls-remote')); +var lsRemote = require('./ls-remote'); function createRemoteStrings(auth, hostname) { var authString = auth.username ? (encodeURIComponent(auth.username) + ':' + encodeURIComponent(auth.password) + '@') : ''; diff --git a/ls-remote.js b/ls-remote.js index 402f0f1..3594c28 100644 --- a/ls-remote.js +++ b/ls-remote.js @@ -1,17 +1,20 @@ -var request = require('request'); +var asp = require('bluebird').Promise.promisify; +var request = asp(require('request')); -module.exports = function(opts, cb) { - opts.encoding = null; - opts.url += '/info/refs?service=git-upload-pack'; +module.exports = function(opts) { + return Promise.resolve() + .then(function() { + opts.encoding = null; + opts.url += '/info/refs?service=git-upload-pack'; - request(opts, function(err, response, body) { - // network errors - if (err) return cb(err); - - if (response.statusCode != 200) return cb({ statusCode: response.statusCode, headers: response.headers }); + return request(opts) + }) + .then(function(response) { + if (response.statusCode != 200) throw { statusCode: response.statusCode, headers: response.headers }; // per the specification, content-type must be this - if (response.headers['content-type'] != 'application/x-git-upload-pack-advertisement') return cb({ statusCode: 500, headers: response.statusCode }); + if (response.headers['content-type'] != 'application/x-git-upload-pack-advertisement') throw { statusCode: 500, headers: response.statusCode }; + var body = response.body; var lines = []; var pos = 0; while (pos < body.length) { @@ -25,11 +28,11 @@ module.exports = function(opts, cb) { } // verify the first line is git-upload-pack - if( lines.shift() != '# service=git-upload-pack') return cb({ statusCode: 500, headers: response.headers }); + if( lines.shift() != '# service=git-upload-pack') throw { statusCode: 500, headers: response.headers }; // verify the second line is the HEAD // e.g. 21d0a9aa00806bb7f67ef5cd98c876aa20e4d803 HEAD\u0000multi_ack thin-pack [...] - if (lines.shift().substr(41, 4) != 'HEAD') return cb({ statusCode: 500, headers: response.headers }); + if (lines.shift().substr(41, 4) != 'HEAD') throw { statusCode: 500, headers: response.headers }; // parse the remaining lines var refs = lines.map(function(line) { @@ -38,7 +41,8 @@ module.exports = function(opts, cb) { name: line.substr(41) }; }); - cb(null, refs); + + return refs; }); } From abb6dff053f16175b5b204c9dcb9b5aa8c9c76c9 Mon Sep 17 00:00:00 2001 From: Adam Burgess Date: Tue, 2 Aug 2016 23:57:18 +1000 Subject: [PATCH 23/23] changes to extend --- github.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/github.js b/github.js index 8bcbc84..0a57af6 100644 --- a/github.js +++ b/github.js @@ -15,8 +15,8 @@ var semver = require('semver'); function extend(dest, src) { for (var key in src) { - if (key in dest && typeof dest[key] === 'object') extend(dest[key], src[key]); - else dest[key] = src[key] + if (typeof dest[key] === 'object') extend(dest[key], src[key]); + else dest[key] = src[key]; } return dest;