diff --git a/config.schema.json b/config.schema.json index 3661d7464..2dc29021a 100644 --- a/config.schema.json +++ b/config.schema.json @@ -5,7 +5,11 @@ "description": "Configuration for customizing git-proxy", "type": "object", "properties": { - "proxyUrl": { "type": "string" }, + "proxyUrl": { + "type": "string", + "description": "Used in early versions of git proxy to configure the remote host that traffic is proxied to. In later versions, the repository URL is used to determine the domain proxied, allowing multiple hosts to be proxied by one instance.", + "deprecated": true + }, "cookieSecret": { "type": "string" }, "sessionMaxAgeHours": { "type": "number" }, "api": { diff --git a/cypress/e2e/repo.cy.js b/cypress/e2e/repo.cy.js index 411397128..8b0ac1a21 100644 --- a/cypress/e2e/repo.cy.js +++ b/cypress/e2e/repo.cy.js @@ -10,7 +10,7 @@ describe('Repo', () => { describe('Code button for repo row', () => { it('Opens tooltip with correct content and can copy', () => { - const cloneURL = 'http://localhost:8000/finos/test-repo.git'; + const cloneURL = 'http://localhost:8000/github.com/finos/test-repo.git'; const tooltipQuery = 'div[role="tooltip"]'; cy diff --git a/package-lock.json b/package-lock.json index af178aa47..2919561cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2715,6 +2715,19 @@ "eslint-scope": "5.1.1" } }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2864,6 +2877,16 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/@paralleldrive/cuid2": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@paralleldrive/cuid2/-/cuid2-2.2.2.tgz", + "integrity": "sha512-ZOBkgDwEdoYVlSeRbYYXs0S9MejQofiVYoTbKzy/6GQa39/q5tQU2IX46+shYnUkpEl3wc+J6wRlar7r2EK2xA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@noble/hashes": "^1.1.5" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -7492,13 +7515,14 @@ } }, "node_modules/formidable": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.2.tgz", - "integrity": "sha512-CM3GuJ57US06mlpQ47YcunuUZ9jpm8Vx+P2CGt2j7HpgkKZO/DJYQ0Bobim8G6PFQmK5lOqOOdUXboU+h73A4g==", + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/formidable/-/formidable-2.1.5.tgz", + "integrity": "sha512-Oz5Hwvwak/DCaXVVUtPn4oLMLLy1CdclLKO1LFgU7XzDpVMUU5UjlSLpGMocyQNNk8F6IJW9M/YdooSn2MRI+Q==", "dev": true, + "license": "MIT", "dependencies": { + "@paralleldrive/cuid2": "^2.2.2", "dezalgo": "^1.0.4", - "hexoid": "^1.0.0", "once": "^1.4.0", "qs": "^6.11.0" }, @@ -8053,15 +8077,6 @@ "he": "bin/he" } }, - "node_modules/hexoid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hexoid/-/hexoid-1.0.0.tgz", - "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/highlight.js": { "version": "11.9.0", "resolved": "https://registry.npmjs.org/highlight.js/-/highlight.js-11.9.0.tgz", @@ -14029,9 +14044,9 @@ } }, "node_modules/vite": { - "version": "4.5.13", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.13.tgz", - "integrity": "sha512-Hgp8IF/yZDzKsN1hQWOuQZbrKiaFsbQud+07jJ8h9m9PaHWkpvZ5u55Xw5yYjWRXwRQ4jwFlJvY7T7FUJG9MCA==", + "version": "4.5.14", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.14.tgz", + "integrity": "sha512-+v57oAaoYNnO3hIu5Z/tJRZjq5aHM2zDve9YZ8HngVHbhk66RStobhb1sqPMIPEleV6cNKYK4eGrAbE9Ulbl2g==", "dev": true, "license": "MIT", "dependencies": { diff --git a/package.json b/package.json index f05521aa0..64c9a0722 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "prepare": "node ./scripts/prepare.js", "lint": "eslint \"src/**/*.{js,jsx,ts,tsx,json}\" \"test/**/*.{js,jsx,ts,tsx,json}\"", "lint:fix": "eslint --fix \"src/**/*.{js,jsx,ts,tsx,json}\" \"test/**/*.{js,jsx,ts,tsx,json}\"", - "format": "prettier --write src/**/*.{js,jsx,ts,tsx,css,md,json,scss} test/**/*.{js,jsx,ts,tsx,json} --config ./.prettierrc", + "format": "prettier --write src/**/*.{js,jsx,ts,tsx,css,md,json,scss} test/**/*.{js,jsx,ts,tsx,json} packages/git-proxy-cli/test/**/*.{js,jsx,ts,tsx,json} packages/git-proxy-cli/index.js --config ./.prettierrc", "gen-schema-doc": "node ./scripts/doc-schema.js", "cypress:run": "cypress run" }, diff --git a/packages/git-proxy-cli/test/testCli.proxy.config.json b/packages/git-proxy-cli/test/testCli.proxy.config.json index 48073ef57..95b3266ec 100644 --- a/packages/git-proxy-cli/test/testCli.proxy.config.json +++ b/packages/git-proxy-cli/test/testCli.proxy.config.json @@ -1,8 +1,7 @@ { "tempPassword": { "sendEmail": false, - "emailConfig": { - } + "emailConfig": {} }, "authorisedList": [ { @@ -22,9 +21,9 @@ { "type": "mongo", "connectionString": "mongodb://localhost:27017/gitproxy", - "options": { + "options": { "useUnifiedTopology": true - }, + }, "enabled": false } ], diff --git a/packages/git-proxy-cli/test/testCli.test.js b/packages/git-proxy-cli/test/testCli.test.js index fbfce0fe3..33d8d7917 100644 --- a/packages/git-proxy-cli/test/testCli.test.js +++ b/packages/git-proxy-cli/test/testCli.test.js @@ -219,12 +219,12 @@ describe('test git-proxy-cli', function () { before(async function () { await helper.addRepoToDb(TEST_REPO_CONFIG); - await helper.addGitPushToDb(pushId, TEST_REPO); + await helper.addGitPushToDb(pushId, TEST_REPO_CONFIG.url); }); after(async function () { await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to authorise should fail when server is down', async function () { @@ -299,7 +299,7 @@ describe('test git-proxy-cli', function () { after(async function () { await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to cancel should fail when server is down', async function () { @@ -415,12 +415,12 @@ describe('test git-proxy-cli', function () { before(async function () { await helper.addRepoToDb(TEST_REPO_CONFIG); - await helper.addGitPushToDb(pushId, TEST_REPO); + await helper.addGitPushToDb(pushId, TEST_REPO_CONFIG.url); }); after(async function () { await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to reject should fail when server is down', async function () { @@ -492,13 +492,13 @@ describe('test git-proxy-cli', function () { before(async function () { await helper.addRepoToDb(TEST_REPO_CONFIG); await helper.addUserToDb('testuser1', 'testpassword', 'test@email.com', gitAccount); - await helper.addGitPushToDb(pushId, TEST_REPO, gitAccount); + await helper.addGitPushToDb(pushId, TEST_REPO_CONFIG.url, gitAccount); }); after(async function () { await helper.removeUserFromDb('testuser1'); await helper.removeGitPushFromDb(pushId); - await helper.removeRepoFromDb(TEST_REPO_CONFIG.name); + await helper.removeRepoFromDb(TEST_REPO_CONFIG.url); }); it('attempt to ls should list existing push', async function () { diff --git a/packages/git-proxy-cli/test/testCliUtils.js b/packages/git-proxy-cli/test/testCliUtils.js index 557857619..e47a21fd3 100644 --- a/packages/git-proxy-cli/test/testCliUtils.js +++ b/packages/git-proxy-cli/test/testCliUtils.js @@ -30,7 +30,7 @@ async function runCli( expectedExitCode = 0, expectedMessages = null, expectedErrorMessages = null, - debug = false, + debug = true, ) { try { console.log(`cli: '${cli}'`); @@ -152,8 +152,9 @@ async function addRepoToDb(newRepo, debug = false) { const found = repos.find((y) => y.project === newRepo.project && newRepo.name === y.name); if (!found) { await db.createRepo(newRepo); - await db.addUserCanPush(newRepo.name, 'admin'); - await db.addUserCanAuthorise(newRepo.name, 'admin'); + const repo = await db.getRepoByUrl(newRepo.url); + await db.addUserCanPush(repo._id, 'admin'); + await db.addUserCanAuthorise(repo._id, 'admin'); if (debug) { console.log(`New repo added to database: ${newRepo}`); } @@ -166,26 +167,27 @@ async function addRepoToDb(newRepo, debug = false) { /** * Removes a repo from the DB. - * @param {string} repoName The name of the repo to remove. + * @param {string} repoUrl The url of the repo to remove. */ -async function removeRepoFromDb(repoName) { - await db.deleteRepo(repoName); +async function removeRepoFromDb(repoUrl) { + const repo = await db.getRepoByUrl(repoUrl); + await db.deleteRepo(repo._id); } /** * Add a new git push record to the database. * @param {string} id The ID of the git push. - * @param {string} repo The repository of the git push. + * @param {string} repoUrl The repository URL of the git push. * @param {string} user The user who pushed the git push. * @param {boolean} debug Flag to enable logging for debugging. */ -async function addGitPushToDb(id, repo, user = null, debug = false) { +async function addGitPushToDb(id, repoUrl, user = null, debug = false) { const action = new actions.Action( id, 'push', // type 'get', // method Date.now(), // timestamp - repo, + repoUrl, ); action.user = user; const step = new steps.Step( diff --git a/proxy.config.json b/proxy.config.json index 618603a6a..e6e6b3a5a 100644 --- a/proxy.config.json +++ b/proxy.config.json @@ -1,5 +1,4 @@ { - "proxyUrl": "https://github.com", "cookieSecret": "cookie secret", "sessionMaxAgeHours": 12, "rateLimit": { diff --git a/src/config/env.ts b/src/config/env.ts index 85b8475b5..9bb0bad55 100644 --- a/src/config/env.ts +++ b/src/config/env.ts @@ -9,12 +9,12 @@ const { GIT_PROXY_SERVER_PORT = 8000, GIT_PROXY_HTTPS_SERVER_PORT = 8443, GIT_PROXY_UI_HOST = 'http://localhost', - GIT_PROXY_UI_PORT = 8080 + GIT_PROXY_UI_PORT = 8080, } = process.env; export const serverConfig: ServerConfig = { GIT_PROXY_SERVER_PORT, GIT_PROXY_HTTPS_SERVER_PORT, GIT_PROXY_UI_HOST, - GIT_PROXY_UI_PORT + GIT_PROXY_UI_PORT, }; diff --git a/src/config/index.ts b/src/config/index.ts index 63174a296..3265a8da7 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -21,7 +21,6 @@ let _database: Database[] = defaultSettings.sink; let _authentication: Authentication[] = defaultSettings.authentication; let _apiAuthentication: Authentication[] = defaultSettings.apiAuthentication; let _tempPassword: TempPasswordConfig = defaultSettings.tempPassword; -let _proxyUrl = defaultSettings.proxyUrl; let _api: Record = defaultSettings.api; let _cookieSecret: string = defaultSettings.cookieSecret; let _sessionMaxAgeHours: number = defaultSettings.sessionMaxAgeHours; @@ -47,15 +46,6 @@ let _config = { ...defaultSettings, ...(_userSettings || {}) } as Configuration; // Create config loader instance const configLoader = new ConfigLoader(_config); -// Get configured proxy URL -export const getProxyUrl = () => { - if (_userSettings !== null && _userSettings.proxyUrl) { - _proxyUrl = _userSettings.proxyUrl; - } - - return _proxyUrl; -}; - // Gets a list of authorised repositories export const getAuthorisedList = () => { if (_userSettings !== null && _userSettings.authorisedList) { @@ -87,12 +77,12 @@ export const getDatabase = () => { } } - throw Error('No database cofigured!'); + throw Error('No database configured!'); }; /** * Get the list of enabled authentication methods - * + * * At least one authentication method must be enabled. * @return {Array} List of enabled authentication methods */ @@ -104,7 +94,7 @@ export const getAuthMethods = () => { const enabledAuthMethods = _authentication.filter((auth) => auth.enabled); if (enabledAuthMethods.length === 0) { - throw new Error("No authentication method enabled"); + throw new Error('No authentication method enabled'); } return enabledAuthMethods; @@ -112,7 +102,7 @@ export const getAuthMethods = () => { /** * Get the list of enabled authentication methods for API endpoints - * + * * If no API authentication methods are enabled, all endpoints are public. * @return {Array} List of enabled authentication methods */ @@ -121,7 +111,7 @@ export const getAPIAuthMethods = () => { _apiAuthentication = _userSettings.apiAuthentication; } - const enabledAuthMethods = _apiAuthentication.filter(auth => auth.enabled); + const enabledAuthMethods = _apiAuthentication.filter((auth) => auth.enabled); return enabledAuthMethods; }; diff --git a/src/db/file/index.ts b/src/db/file/index.ts index 6ac1c2088..80276a7af 100644 --- a/src/db/file/index.ts +++ b/src/db/file/index.ts @@ -2,29 +2,19 @@ import * as users from './users'; import * as repo from './repo'; import * as pushes from './pushes'; -export const { - getPushes, - writeAudit, - getPush, - deletePush, - authorise, - cancel, - reject, - canUserCancelPush, - canUserApproveRejectPush, -} = pushes; +export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; export const { getRepos, getRepo, + getRepoByUrl, + getRepoById, createRepo, addUserCanPush, addUserCanAuthorise, removeUserCanPush, removeUserCanAuthorise, deleteRepo, - isUserPushAllowed, - canUserApproveRejectPushRepo, } = repo; export const { findUser, findUserByOIDC, getUsers, createUser, deleteUser, updateUser } = users; diff --git a/src/db/file/pushes.ts b/src/db/file/pushes.ts index 2a54314ea..38a3336f6 100644 --- a/src/db/file/pushes.ts +++ b/src/db/file/pushes.ts @@ -3,13 +3,19 @@ import _ from 'lodash'; import Datastore from '@seald-io/nedb'; import { Action } from '../../proxy/actions/Action'; import { toClass } from '../helper'; -import * as repo from './repo'; import { PushQuery } from '../types'; +const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day + +// these don't get coverage in tests as they have already been run once before the test +/* istanbul ignore if */ if (!fs.existsSync('./.data')) fs.mkdirSync('./.data'); +/* istanbul ignore if */ if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/pushes.db', autoload: true }); +db.ensureIndex({ fieldName: 'id', unique: true }); +db.setAutocompactionInterval(COMPACTION_INTERVAL); const defaultPushQuery: PushQuery = { error: false, @@ -18,10 +24,12 @@ const defaultPushQuery: PushQuery = { authorised: false, }; -export const getPushes = (query: PushQuery) => { +export const getPushes = (query: PushQuery): Promise => { if (!query) query = defaultPushQuery; return new Promise((resolve, reject) => { db.find(query, (err: Error, docs: Action[]) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -35,9 +43,11 @@ export const getPushes = (query: PushQuery) => { }); }; -export const getPush = async (id: string) => { +export const getPush = async (id: string): Promise => { return new Promise((resolve, reject) => { db.findOne({ id: id }, (err, doc) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -51,9 +61,11 @@ export const getPush = async (id: string) => { }); }; -export const deletePush = async (id: string) => { +export const deletePush = async (id: string): Promise => { return new Promise((resolve, reject) => { db.remove({ id }, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -63,20 +75,22 @@ export const deletePush = async (id: string) => { }); }; -export const writeAudit = async (action: Action) => { +export const writeAudit = async (action: Action): Promise => { return new Promise((resolve, reject) => { const options = { multi: false, upsert: true }; db.update({ id: action.id }, action, options, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const authorise = async (id: string, attestation: any) => { +export const authorise = async (id: string, attestation: any): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -90,7 +104,7 @@ export const authorise = async (id: string, attestation: any) => { return { message: `authorised ${id}` }; }; -export const reject = async (id: string) => { +export const reject = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -103,7 +117,7 @@ export const reject = async (id: string) => { return { message: `reject ${id}` }; }; -export const cancel = async (id: string) => { +export const cancel = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -114,36 +128,3 @@ export const cancel = async (id: string) => { await writeAudit(action); return { message: `cancel ${id}` }; }; - -export const canUserCancelPush = async (id: string, user: any) => { - return new Promise(async (resolve) => { - const pushDetail = await getPush(id); - if (!pushDetail) { - resolve(false); - return; - } - - const repoName = pushDetail.repoName.replace('.git', ''); - const isAllowed = await repo.isUserPushAllowed(repoName, user); - - if (isAllowed) { - resolve(true); - } else { - resolve(false); - } - }); -}; - -export const canUserApproveRejectPush = async (id: string, user: any) => { - return new Promise(async (resolve) => { - const action = await getPush(id); - if (!action) { - resolve(false); - return; - } - const repoName = action?.repoName.replace('.git', ''); - const isAllowed = await repo.canUserApproveRejectPushRepo(repoName, user); - - resolve(isAllowed); - }); -}; diff --git a/src/db/file/repo.ts b/src/db/file/repo.ts index 8686899f5..c7acdbe0a 100644 --- a/src/db/file/repo.ts +++ b/src/db/file/repo.ts @@ -1,108 +1,159 @@ import fs from 'fs'; -import Datastore from '@seald-io/nedb' +import Datastore from '@seald-io/nedb'; import { Repo } from '../types'; +import { toClass } from '../helper'; +import _ from 'lodash'; +const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day + +// these don't get coverage in tests as they have already been run once before the test +/* istanbul ignore if */ if (!fs.existsSync('./.data')) fs.mkdirSync('./.data'); +/* istanbul ignore if */ if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/repos.db', autoload: true }); +db.ensureIndex({ fieldName: 'url', unique: true }); +db.setAutocompactionInterval(COMPACTION_INTERVAL); -export const getRepos = async (query: any = {}) => { +export const getRepos = async (query: any = {}): Promise => { + if (query?.name) { + query.name = query.name.toLowerCase(); + } return new Promise((resolve, reject) => { - db.find({}, (err: Error, docs: Repo[]) => { + db.find(query, (err: Error, docs: Repo[]) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(docs); + resolve( + _.chain(docs) + .map((x) => toClass(x, Repo.prototype)) + .value(), + ); } }); }); }; -export const getRepo = async (name: string) => { +export const getRepo = async (name: string): Promise => { return new Promise((resolve, reject) => { - db.findOne({ name }, (err: Error | null, doc: Repo) => { + db.findOne({ name: name.toLowerCase() }, (err: Error | null, doc: Repo) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(doc); + resolve(doc ? toClass(doc, Repo.prototype) : null); } }); }); }; +export const getRepoByUrl = async (repoURL: string): Promise => { + return new Promise((resolve, reject) => { + db.findOne({ url: repoURL }, (err: Error | null, doc: Repo) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(doc ? toClass(doc, Repo.prototype) : null); + } + }); + }); +}; -export const createRepo = async (repo: Repo) => { - repo.users = { - canPush: [], - canAuthorise: [], - }; +export const getRepoById = async (_id: string): Promise => { + return new Promise((resolve, reject) => { + db.findOne({ _id: _id }, (err: Error | null, doc: Repo) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(doc ? toClass(doc, Repo.prototype) : null); + } + }); + }); +}; +export const createRepo = async (repo: Repo): Promise => { return new Promise((resolve, reject) => { db.insert(repo, (err, doc) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(doc); + resolve(doc ? toClass(doc, Repo.prototype) : null); } }); }); }; -export const addUserCanPush = async (name: string, user: string) => { +export const addUserCanPush = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; } if (repo.users.canPush.includes(user)) { - resolve(null); + resolve(); return; } repo.users.canPush.push(user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const addUserCanAuthorise = async (name: string, user: string) => { +export const addUserCanAuthorise = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; } if (repo.users.canAuthorise.includes(user)) { - resolve(null); + resolve(); return; } repo.users.canAuthorise.push(user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const removeUserCanAuthorise = async (name: string, user: string) => { +export const removeUserCanAuthorise = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; @@ -111,19 +162,22 @@ export const removeUserCanAuthorise = async (name: string, user: string) => { repo.users.canAuthorise = repo.users.canAuthorise.filter((x: string) => x != user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const removeUserCanPush = async (name: string, user: string) => { +export const removeUserCanPush = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); return new Promise(async (resolve, reject) => { - const repo = await getRepo(name); + const repo = await getRepoById(_id); if (!repo) { reject(new Error('Repo not found')); return; @@ -132,19 +186,23 @@ export const removeUserCanPush = async (name: string, user: string) => { repo.users.canPush = repo.users.canPush.filter((x) => x != user); const options = { multi: false, upsert: false }; - db.update({ name: name }, repo, options, (err) => { + db.update({ _id: _id }, repo, options, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(null); + resolve(); } }); }); }; -export const deleteRepo = async (name: string) => { +export const deleteRepo = async (_id: string): Promise => { return new Promise((resolve, reject) => { - db.remove({ name: name }, (err) => { + db.remove({ _id: _id }, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -153,43 +211,3 @@ export const deleteRepo = async (name: string) => { }); }); }; - -export const isUserPushAllowed = async (name: string, user: string) => { - name = name.toLowerCase(); - return new Promise(async (resolve) => { - const repo = await getRepo(name); - if (!repo) { - resolve(false); - return; - } - - console.log(repo.users.canPush); - console.log(repo.users.canAuthorise); - - if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { - resolve(true); - } else { - resolve(false); - } - }); -}; - -export const canUserApproveRejectPushRepo = async (name: string, user: string) => { - name = name.toLowerCase(); - console.log(`checking if user ${user} can approve/reject for ${name}`); - return new Promise(async (resolve) => { - const repo = await getRepo(name); - if (!repo) { - resolve(false); - return; - } - - if (repo.users.canAuthorise.includes(user)) { - console.log(`user ${user} can approve/reject to repo ${name}`); - resolve(true); - } else { - console.log(`user ${user} cannot approve/reject to repo ${name}`); - resolve(false); - } - }); -}; diff --git a/src/db/file/users.ts b/src/db/file/users.ts index d72443c97..25716f10b 100644 --- a/src/db/file/users.ts +++ b/src/db/file/users.ts @@ -2,14 +2,26 @@ import fs from 'fs'; import Datastore from '@seald-io/nedb'; import { User } from '../types'; +const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day + +// these don't get coverage in tests as they have already been run once before the test +/* istanbul ignore if */ if (!fs.existsSync('./.data')) fs.mkdirSync('./.data'); +/* istanbul ignore if */ if (!fs.existsSync('./.data/db')) fs.mkdirSync('./.data/db'); const db = new Datastore({ filename: './.data/db/users.db', autoload: true }); -export const findUser = (username: string) => { +// Using a unique constraint with the index +db.ensureIndex({ fieldName: 'username', unique: true }); +db.ensureIndex({ fieldName: 'email', unique: true }); +db.setAutocompactionInterval(COMPACTION_INTERVAL); + +export const findUser = (username: string): Promise => { return new Promise((resolve, reject) => { - db.findOne({ username: username }, (err: Error | null, doc: User) => { + db.findOne({ username: username.toLowerCase() }, (err: Error | null, doc: User) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -23,9 +35,11 @@ export const findUser = (username: string) => { }); }; -export const findUserByOIDC = function (oidcId: string) { - return new Promise((resolve, reject) => { - db.findOne({ oidcId: oidcId }, (err, doc) => { +export const findUserByOIDC = function (oidcId: string): Promise { + return new Promise((resolve, reject) => { + db.findOne({ oidcId: oidcId }, (err: Error | null, doc: User) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -39,21 +53,27 @@ export const findUserByOIDC = function (oidcId: string) { }); }; -export const createUser = function (user: User) { +export const createUser = function (user: User): Promise { + user.username = user.username.toLowerCase(); + user.email = user.email.toLowerCase(); return new Promise((resolve, reject) => { db.insert(user, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(user); + resolve(); } }); }); }; -export const deleteUser = (username: string) => { +export const deleteUser = (username: string): Promise => { return new Promise((resolve, reject) => { - db.remove({ username: username }, (err) => { + db.remove({ username: username.toLowerCase() }, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { @@ -63,22 +83,55 @@ export const deleteUser = (username: string) => { }); }; -export const updateUser = (user: User) => { +export const updateUser = (user: User): Promise => { + user.username = user.username.toLowerCase(); + if (user.email) { + user.email = user.email.toLowerCase(); + } return new Promise((resolve, reject) => { - const options = { multi: false, upsert: false }; - db.update({ username: user.username }, user, options, (err) => { + // The mongo db adaptor adds fields to existing documents, where this adaptor replaces the document + // hence, retrieve and merge documents to avoid dropping fields (such as the gitaccount) + let existingUser; + db.findOne({ username: user.username }, (err, doc) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { - resolve(null); + if (!doc) { + existingUser = {}; + } else { + existingUser = doc; + } + + Object.assign(existingUser, user); + + const options = { multi: false, upsert: true }; + db.update({ username: user.username }, existingUser, options, (err) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ + if (err) { + reject(err); + } else { + resolve(); + } + }); } }); }); }; -export const getUsers = (query: any = {}) => { +export const getUsers = (query: any = {}): Promise => { + if (query.username) { + query.username = query.username.toLowerCase(); + } + if (query.email) { + query.email = query.email.toLowerCase(); + } return new Promise((resolve, reject) => { db.find(query, (err: Error, docs: User[]) => { + // ignore for code coverage as neDB rarely returns errors even for an invalid query + /* istanbul ignore if */ if (err) { reject(err); } else { diff --git a/src/db/index.ts b/src/db/index.ts index ff1189f1b..518f64468 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -1,12 +1,23 @@ -const bcrypt = require('bcryptjs'); -const config = require('../config'); -let sink: any; +import { AuthorisedRepo } from '../config/types'; +import { PushQuery, Repo, Sink, User } from './types'; +import * as bcrypt from 'bcryptjs'; +import * as config from '../config'; +import * as mongo from './mongo'; +import * as neDb from './file'; +import { Action } from '../proxy/actions/Action'; +import MongoDBStore from 'connect-mongo'; + +let sink: Sink; if (config.getDatabase().type === 'mongo') { - sink = require('./mongo'); + sink = mongo; } else if (config.getDatabase().type === 'fs') { - sink = require('./file'); + sink = neDb; } +const isBlank = (str: string) => { + return !str || /^\s*$/.test(str); +}; + export const createUser = async ( username: string, password: string, @@ -32,17 +43,17 @@ export const createUser = async ( admin: admin, }; - if (username === undefined || username === null || username === '') { + if (isBlank(username)) { const errorMessage = `username ${username} cannot be empty`; throw new Error(errorMessage); } - if (gitAccount === undefined || gitAccount === null || gitAccount === '') { + if (isBlank(gitAccount)) { const errorMessage = `GitAccount ${gitAccount} cannot be empty`; throw new Error(errorMessage); } - if (email === undefined || email === null || email === '') { + if (isBlank(email)) { const errorMessage = `Email ${email} cannot be empty`; throw new Error(errorMessage); } @@ -56,30 +67,114 @@ export const createUser = async ( await sink.createUser(data); }; -export const { - authorise, - reject, - cancel, - getPushes, - writeAudit, - getPush, - deletePush, - findUser, - findUserByOIDC, - getUsers, - deleteUser, - updateUser, - getRepos, - getRepo, - createRepo, - addUserCanPush, - addUserCanAuthorise, - removeUserCanAuthorise, - removeUserCanPush, - deleteRepo, - isUserPushAllowed, - canUserApproveRejectPushRepo, - canUserApproveRejectPush, - canUserCancelPush, - getSessionStore, -} = sink; +export const createRepo = async (repo: AuthorisedRepo) => { + const toCreate = { + ...repo, + users: { + canPush: [], + canAuthorise: [], + }, + }; + toCreate.name = repo.name.toLowerCase(); + + console.log(`creating new repo ${JSON.stringify(toCreate)}`); + + if (isBlank(toCreate.project)) { + throw new Error('Project name cannot be empty'); + } + if (isBlank(toCreate.name)) { + throw new Error('Repository name cannot be empty'); + } + if (isBlank(toCreate.url)) { + throw new Error('URL cannot be empty'); + } + + return sink.createRepo(toCreate) as Promise>; +}; + +export const isUserPushAllowed = async (url: string, user: string) => { + user = user.toLowerCase(); + return new Promise(async (resolve) => { + const repo = await getRepoByUrl(url); + if (!repo) { + resolve(false); + return; + } + + console.log(repo.users.canPush); + console.log(repo.users.canAuthorise); + + if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { + resolve(true); + } else { + resolve(false); + } + }); +}; + +export const canUserApproveRejectPush = async (id: string, user: string) => { + return new Promise(async (resolve) => { + const action = await getPush(id); + if (!action) { + resolve(false); + return; + } + + const theRepo = await sink.getRepoByUrl(action.url); + + if (theRepo?.users?.canAuthorise?.includes(user)) { + console.log(`user ${user} can approve/reject for repo ${action.url}`); + resolve(true); + } else { + console.log(`user ${user} cannot approve/reject for repo ${action.url}`); + resolve(false); + } + }); +}; + +export const canUserCancelPush = async (id: string, user: string) => { + return new Promise(async (resolve) => { + const action = await getPush(id); + if (!action) { + resolve(false); + return; + } + + const isAllowed = await isUserPushAllowed(action.url, user); + + if (isAllowed) { + resolve(true); + } else { + resolve(false); + } + }); +}; + +export const getSessionStore = (): MongoDBStore | null => + sink.getSessionStore ? sink.getSessionStore() : null; +export const getPushes = (query: PushQuery): Promise => sink.getPushes(query); +export const writeAudit = (action: Action): Promise => sink.writeAudit(action); +export const getPush = (id: string): Promise => sink.getPush(id); +export const deletePush = (id: string): Promise => sink.deletePush(id); +export const authorise = (id: string, attestation: any): Promise<{ message: string }> => + sink.authorise(id, attestation); +export const cancel = (id: string): Promise<{ message: string }> => sink.cancel(id); +export const reject = (id: string): Promise<{ message: string }> => sink.reject(id); +export const getRepos = (query?: object): Promise => sink.getRepos(query); +export const getRepo = (name: string): Promise => sink.getRepo(name); +export const getRepoByUrl = (url: string): Promise => sink.getRepoByUrl(url); +export const getRepoById = (_id: string): Promise => sink.getRepoById(_id); +export const addUserCanPush = (_id: string, user: string): Promise => + sink.addUserCanPush(_id, user); +export const addUserCanAuthorise = (_id: string, user: string): Promise => + sink.addUserCanAuthorise(_id, user); +export const removeUserCanPush = (_id: string, user: string): Promise => + sink.removeUserCanPush(_id, user); +export const removeUserCanAuthorise = (_id: string, user: string): Promise => + sink.removeUserCanAuthorise(_id, user); +export const deleteRepo = (_id: string): Promise => sink.deleteRepo(_id); +export const findUser = (username: string): Promise => sink.findUser(username); +export const findUserByOIDC = (oidcId: string): Promise => sink.findUserByOIDC(oidcId); +export const getUsers = (query?: object): Promise => sink.getUsers(query); +export const deleteUser = (username: string): Promise => sink.deleteUser(username); +export const updateUser = (user: User): Promise => sink.updateUser(user); diff --git a/src/db/mongo/helper.ts b/src/db/mongo/helper.ts index 335434ebb..c4956de0f 100644 --- a/src/db/mongo/helper.ts +++ b/src/db/mongo/helper.ts @@ -25,7 +25,7 @@ export const connect = async (collectionName: string): Promise => { export const findDocuments = async ( collectionName: string, filter: Filter = {}, - options: FindOptions = {} + options: FindOptions = {}, ): Promise => { const collection = await connect(collectionName); return collection.find(filter, options).toArray() as Promise; @@ -34,7 +34,7 @@ export const findDocuments = async ( export const findOneDocument = async ( collectionName: string, filter: Filter = {}, - options: FindOptions = {} + options: FindOptions = {}, ): Promise => { const collection = await connect(collectionName); return (await collection.findOne(filter, options)) as T | null; diff --git a/src/db/mongo/index.ts b/src/db/mongo/index.ts index a6d7ce6b2..9b81720ad 100644 --- a/src/db/mongo/index.ts +++ b/src/db/mongo/index.ts @@ -5,29 +5,19 @@ import * as users from './users'; export const { getSessionStore } = helper; -export const { - getPushes, - writeAudit, - getPush, - deletePush, - authorise, - cancel, - reject, - canUserCancelPush, - canUserApproveRejectPush, -} = pushes; +export const { getPushes, writeAudit, getPush, deletePush, authorise, cancel, reject } = pushes; export const { getRepos, getRepo, + getRepoByUrl, + getRepoById, createRepo, addUserCanPush, addUserCanAuthorise, removeUserCanPush, removeUserCanAuthorise, deleteRepo, - isUserPushAllowed, - canUserApproveRejectPushRepo, } = repo; -export const { findUser, getUsers, createUser, deleteUser, updateUser } = users; +export const { findUser, findUserByOIDC, getUsers, createUser, deleteUser, updateUser } = users; diff --git a/src/db/mongo/pushes.ts b/src/db/mongo/pushes.ts index 8778bdf73..c64325755 100644 --- a/src/db/mongo/pushes.ts +++ b/src/db/mongo/pushes.ts @@ -1,8 +1,7 @@ import { connect, findDocuments, findOneDocument } from './helper'; import { Action } from '../../proxy/actions'; import { toClass } from '../helper'; -import * as repo from './repo'; -import { Push, PushQuery } from '../types'; +import { PushQuery } from '../types'; const collectionName = 'pushes'; @@ -13,8 +12,8 @@ const defaultPushQuery: PushQuery = { authorised: false, }; -export const getPushes = async (query: PushQuery = defaultPushQuery): Promise => { - return findDocuments(collectionName, query, { +export const getPushes = async (query: PushQuery = defaultPushQuery): Promise => { + return findDocuments(collectionName, query, { projection: { _id: 0, id: 1, @@ -45,12 +44,12 @@ export const getPush = async (id: string): Promise => { return doc ? (toClass(doc, Action.prototype) as Action) : null; }; -export const deletePush = async function (id: string) { +export const deletePush = async function (id: string): Promise { const collection = await connect(collectionName); - return collection.deleteOne({ id }); + await collection.deleteOne({ id }); }; -export const writeAudit = async (action: Action): Promise => { +export const writeAudit = async (action: Action): Promise => { const data = JSON.parse(JSON.stringify(action)); const options = { upsert: true }; const collection = await connect(collectionName); @@ -59,10 +58,9 @@ export const writeAudit = async (action: Action): Promise => { throw new Error('Invalid id'); } await collection.updateOne({ id: data.id }, { $set: data }, options); - return action; }; -export const authorise = async (id: string, attestation: any) => { +export const authorise = async (id: string, attestation: any): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -76,7 +74,7 @@ export const authorise = async (id: string, attestation: any) => { return { message: `authorised ${id}` }; }; -export const reject = async (id: string) => { +export const reject = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -88,7 +86,7 @@ export const reject = async (id: string) => { return { message: `reject ${id}` }; }; -export const cancel = async (id: string) => { +export const cancel = async (id: string): Promise<{ message: string }> => { const action = await getPush(id); if (!action) { throw new Error(`push ${id} not found`); @@ -99,37 +97,3 @@ export const cancel = async (id: string) => { await writeAudit(action); return { message: `canceled ${id}` }; }; - -export const canUserApproveRejectPush = async (id: string, user: string) => { - return new Promise(async (resolve) => { - const action = await getPush(id); - if (!action) { - resolve(false); - return; - } - - const repoName = action.repoName.replace('.git', ''); - const isAllowed = await repo.canUserApproveRejectPushRepo(repoName, user); - - resolve(isAllowed); - }); -}; - -export const canUserCancelPush = async (id: string, user: string) => { - return new Promise(async (resolve) => { - const pushDetail = await getPush(id); - if (!pushDetail) { - resolve(false); - return; - } - - const repoName = pushDetail.repoName.replace('.git', ''); - const isAllowed = await repo.isUserPushAllowed(repoName, user); - - if (isAllowed) { - resolve(true); - } else { - resolve(false); - } - }); -}; diff --git a/src/db/mongo/repo.ts b/src/db/mongo/repo.ts index 5e55d3d71..8a0311a63 100644 --- a/src/db/mongo/repo.ts +++ b/src/db/mongo/repo.ts @@ -1,100 +1,71 @@ -import { Repo } from "../types"; - -const connect = require('./helper').connect; +import _ from 'lodash'; +import { Repo } from '../types'; +import { connect } from './helper'; +import { toClass } from '../helper'; +import { ObjectId, OptionalId, Document } from 'mongodb'; const collectionName = 'repos'; -const isBlank = (str: string) => { - return !str || /^\s*$/.test(str); -}; - -export const getRepos = async (query: any = {}) => { +export const getRepos = async (query: any = {}): Promise => { const collection = await connect(collectionName); - return collection.find(query).toArray(); + const docs = collection.find(query).toArray(); + return _.chain(docs) + .map((x) => toClass(x, Repo.prototype)) + .value(); }; -export const getRepo = async (name: string) => { +export const getRepo = async (name: string): Promise => { + name = name.toLowerCase(); const collection = await connect(collectionName); - return collection.findOne({ name: { $eq: name } }); + const doc = collection.findOne({ name: { $eq: name } }); + return doc ? toClass(doc, Repo.prototype) : null; }; -export const createRepo = async (repo: Repo) => { - console.log(`creating new repo ${JSON.stringify(repo)}`); - - if (isBlank(repo.project)) { - throw new Error('Project name cannot be empty'); - } - if (isBlank(repo.name)) { - throw new Error('Repository name cannot be empty'); - } - if (isBlank(repo.url)) { - throw new Error('URL cannot be empty'); - } - - repo.users = { - canPush: [], - canAuthorise: [], - }; - +export const getRepoByUrl = async (repoUrl: string): Promise => { const collection = await connect(collectionName); - await collection.insertOne(repo); - console.log(`created new repo ${JSON.stringify(repo)}`); + const doc = collection.findOne({ name: { $eq: repoUrl.toLowerCase() } }); + return doc ? toClass(doc, Repo.prototype) : null; }; -export const addUserCanPush = async (name: string, user: string) => { - name = name.toLowerCase(); +export const getRepoById = async (_id: string): Promise => { const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $push: { 'users.canPush': user } }); + const doc = collection.findOne({ _id: new ObjectId(_id) }); + return doc ? toClass(doc, Repo.prototype) : null; }; -export const addUserCanAuthorise = async (name: string, user: string) => { - name = name.toLowerCase(); +export const createRepo = async (repo: Repo): Promise => { const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $push: { 'users.canAuthorise': user } }); + const response = await collection.insertOne(repo as OptionalId); + console.log(`created new repo ${JSON.stringify(repo)}`); + // add in the _id generated for the record + repo._id = response.insertedId.toString(); + return repo; }; -export const removeUserCanPush = async (name: string, user: string) => { - name = name.toLowerCase(); +export const addUserCanPush = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $pull: { 'users.canPush': user } }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $push: { 'users.canPush': user } }); }; -export const removeUserCanAuthorise = async (name: string, user: string) => { - name = name.toLowerCase(); +export const addUserCanAuthorise = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.updateOne({ name: name }, { $pull: { 'users.canAuthorise': user } }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $push: { 'users.canAuthorise': user } }); }; -export const deleteRepo = async (name: string) => { +export const removeUserCanPush = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); const collection = await connect(collectionName); - await collection.deleteMany({ name: name }); + await collection.updateOne({ _id: new ObjectId(_id) }, { $pull: { 'users.canPush': user } }); }; -export const isUserPushAllowed = async (name: string, user: string) => { - name = name.toLowerCase(); - return new Promise(async (resolve) => { - const repo = await exports.getRepo(name); - console.log(repo.users.canPush); - console.log(repo.users.canAuthorise); - - if (repo.users.canPush.includes(user) || repo.users.canAuthorise.includes(user)) { - resolve(true); - } else { - resolve(false); - } - }); +export const removeUserCanAuthorise = async (_id: string, user: string): Promise => { + user = user.toLowerCase(); + const collection = await connect(collectionName); + await collection.updateOne({ _id: new ObjectId(_id) }, { $pull: { 'users.canAuthorise': user } }); }; -export const canUserApproveRejectPushRepo = async (name: string, user: string) => { - name = name.toLowerCase(); - console.log(`checking if user ${user} can approve/reject for ${name}`); - return new Promise(async (resolve) => { - const repo = await exports.getRepo(name); - if (repo.users.canAuthorise.includes(user)) { - console.log(`user ${user} can approve/reject to repo ${name}`); - resolve(true); - } else { - console.log(`user ${user} cannot approve/reject to repo ${name}`); - resolve(false); - } - }); +export const deleteRepo = async (_id: string): Promise => { + const collection = await connect(collectionName); + await collection.deleteMany({ _id: new ObjectId(_id) }); }; diff --git a/src/db/mongo/users.ts b/src/db/mongo/users.ts index 0bfa1a941..623bcc9d1 100644 --- a/src/db/mongo/users.ts +++ b/src/db/mongo/users.ts @@ -1,32 +1,54 @@ -import { User } from "../types"; - -const connect = require('./helper').connect; +import { OptionalId, Document } from 'mongodb'; +import { toClass } from '../helper'; +import { User } from '../types'; +import { connect } from './helper'; +import _ from 'lodash'; const collectionName = 'users'; -export const findUser = async function (username: string) { +export const findUser = async function (username: string): Promise { + const collection = await connect(collectionName); + const doc = collection.findOne({ username: { $eq: username.toLowerCase() } }); + return doc ? toClass(doc, User.prototype) : null; +}; + +export const findUserByOIDC = async function (oidcId: string): Promise { const collection = await connect(collectionName); - return collection.findOne({ username: { $eq: username } }); + const doc = collection.findOne({ oidcId: { $eq: oidcId } }); + return doc ? toClass(doc, User.prototype) : null; }; -export const getUsers = async function (query: any = {}) { - console.log(`Getting users for query= ${JSON.stringify(query)}`); +export const getUsers = async function (query: any = {}): Promise { + if (query.username) { + query.username = query.username.toLowerCase(); + } + if (query.email) { + query.email = query.email.toLowerCase(); + } + console.log(`Getting users for query = ${JSON.stringify(query)}`); const collection = await connect(collectionName); - return collection.find(query, { password: 0 }).toArray(); + const docs = collection.find(query, { projection: { password: 0 } }).toArray(); + return _.chain(docs) + .map((x) => toClass(x, User.prototype)) + .value(); }; -export const deleteUser = async function (username: string) { +export const deleteUser = async function (username: string): Promise { const collection = await connect(collectionName); - return collection.deleteOne({ username: username }); + await collection.deleteOne({ username: username.toLowerCase() }); }; -export const createUser = async function (user: User) { +export const createUser = async function (user: User): Promise { user.username = user.username.toLowerCase(); + user.email = user.email.toLowerCase(); const collection = await connect(collectionName); - return collection.insertOne(user); + await collection.insertOne(user as OptionalId); }; -export const updateUser = async (user: User) => { +export const updateUser = async (user: User): Promise => { user.username = user.username.toLowerCase(); + if (user.email) { + user.email = user.email.toLowerCase(); + } const options = { upsert: true }; const collection = await connect(collectionName); await collection.updateOne({ username: user.username }, { $set: user }, options); diff --git a/src/db/types.ts b/src/db/types.ts index dba9bdf3a..fc9503701 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -1,48 +1,88 @@ +import { Action } from '../proxy/actions/Action'; +import MongoDBStore from 'connect-mongo'; + export type PushQuery = { error: boolean; - blocked: boolean, - allowPush: boolean, - authorised: boolean + blocked: boolean; + allowPush: boolean; + authorised: boolean; }; export type UserRole = 'canPush' | 'canAuthorise'; -export type Repo = { +export class Repo { project: string; name: string; url: string; users: Record; - _id: string; -}; + _id?: string; + + constructor( + project: string, + name: string, + url: string, + users?: Record, + _id?: string, + ) { + this.project = project; + this.name = name; + this.url = url; + this.users = users ?? { canPush: [], canAuthorise: [] }; + this._id = _id; + } +} -export type User = { - _id: string; +export class User { username: string; password: string | null; // null if oidcId is set gitAccount: string; email: string; admin: boolean; - oidcId: string | null; -}; + oidcId?: string | null; + _id?: string; -export type Push = { - id: string; - allowPush: boolean; - authorised: boolean; - blocked: boolean; - blockedMessage: string; - branch: string; - canceled: boolean; - commitData: object; - commitFrom: string; - commitTo: string; - error: boolean; - method: string; - project: string; - rejected: boolean; - repo: string; - repoName: string; - timepstamp: string; - type: string; - url: string; -}; + constructor( + username: string, + password: string, + gitAccount: string, + email: string, + admin: boolean, + oidcId: string | null = null, + _id?: string, + ) { + this.username = username; + this.password = password; + this.gitAccount = gitAccount; + this.email = email; + this.admin = admin; + this.oidcId = oidcId ?? null; + this._id = _id; + } +} + +export interface Sink { + getSessionStore?: () => MongoDBStore; + getPushes: (query: PushQuery) => Promise; + writeAudit: (action: Action) => Promise; + getPush: (id: string) => Promise; + deletePush: (id: string) => Promise; + authorise: (id: string, attestation: any) => Promise<{ message: string }>; + cancel: (id: string) => Promise<{ message: string }>; + reject: (id: string) => Promise<{ message: string }>; + getRepos: (query?: object) => Promise; + getRepo: (name: string) => Promise; + getRepoByUrl: (url: string) => Promise; + getRepoById: (_id: string) => Promise; + createRepo: (repo: Repo) => Promise; + addUserCanPush: (_id: string, user: string) => Promise; + addUserCanAuthorise: (_id: string, user: string) => Promise; + removeUserCanPush: (_id: string, user: string) => Promise; + removeUserCanAuthorise: (_id: string, user: string) => Promise; + deleteRepo: (_id: string) => Promise; + findUser: (username: string) => Promise; + findUserByOIDC: (oidcId: string) => Promise; + getUsers: (query?: object) => Promise; + createUser: (user: User) => Promise; + deleteUser: (username: string) => Promise; + updateUser: (user: User) => Promise; +} diff --git a/src/plugin.ts b/src/plugin.ts index f2bd8f26a..92fb9a99c 100644 --- a/src/plugin.ts +++ b/src/plugin.ts @@ -15,9 +15,11 @@ function isCompatiblePlugin(obj: any, propertyName: string = 'isGitProxyPlugin') // valid plugin objects will have the appropriate property set to true // if the prototype chain is exhausted, return false while (obj != null) { - if (Object.prototype.hasOwnProperty.call(obj, propertyName) && + if ( + Object.prototype.hasOwnProperty.call(obj, propertyName) && obj.isGitProxyPlugin && - Object.keys(obj).includes('exec')) { + Object.keys(obj).includes('exec') + ) { return true; } obj = Object.getPrototypeOf(obj); @@ -55,25 +57,28 @@ class PluginLoader { */ async load(): Promise { try { - const modulePromises = this.targets.map(target => - this._loadPluginModule(target).catch(error => { + const modulePromises = this.targets.map((target) => + this._loadPluginModule(target).catch((error) => { console.error(`Failed to load plugin: ${error}`); // TODO: log.error() return Promise.reject(error); // Or return an error object to handle it later - }) + }), ); const moduleResults = await Promise.allSettled(modulePromises); const loadedModules = moduleResults - .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled' && result.value !== null) - .map(result => result.value); + .filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled' && result.value !== null, + ) + .map((result) => result.value); console.log(`Found ${loadedModules.length} plugin modules`); // TODO: log.debug() - const pluginTypeResultPromises = loadedModules.map(mod => - this._getPluginObjects(mod).catch(error => { + const pluginTypeResultPromises = loadedModules.map((mod) => + this._getPluginObjects(mod).catch((error) => { console.error(`Failed to cast plugin objects: ${error}`); // TODO: log.error() return Promise.reject(error); // Or return an error object to handle it later - }) + }), ); const settledPluginTypeResults = await Promise.allSettled(pluginTypeResultPromises); @@ -81,16 +86,19 @@ class PluginLoader { * @type {PluginTypeResult[]} List of resolved PluginTypeResult objects */ const pluginTypeResults = settledPluginTypeResults - .filter((result): result is PromiseFulfilledResult => result.status === 'fulfilled' && result.value !== null) - .map(result => result.value); + .filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled' && result.value !== null, + ) + .map((result) => result.value); for (const result of pluginTypeResults) { - this.pushPlugins.push(...result.pushAction) - this.pullPlugins.push(...result.pullAction) + this.pushPlugins.push(...result.pushAction); + this.pullPlugins.push(...result.pullAction); } const combinedPlugins = [...this.pushPlugins, ...this.pullPlugins]; - combinedPlugins.forEach(plugin => { + combinedPlugins.forEach((plugin) => { console.log(`Loaded plugin: ${plugin.constructor.name}`); }); } catch (error) { @@ -128,7 +136,9 @@ class PluginLoader { console.log('found pull plugin', potentialModule.constructor.name); plugins.pullAction.push(potentialModule); } else { - console.error(`Error: Object ${potentialModule.constructor.name} does not seem to be a compatible plugin type`); + console.error( + `Error: Object ${potentialModule.constructor.name} does not seem to be a compatible plugin type`, + ); } } @@ -136,7 +146,7 @@ class PluginLoader { // `module.exports = new ProxyPlugin()` in CJS or `exports default new ProxyPlugin()` in ESM // the "module" is a single object that could be a plugin if (isCompatiblePlugin(pluginModule)) { - handlePlugin(pluginModule) + handlePlugin(pluginModule); } else { // handle the typical case of a module which exports multiple objects // module.exports = { x, y } (CJS) or multiple `export ...` statements (ESM) @@ -173,11 +183,11 @@ class PushActionPlugin extends ProxyPlugin { * Wrapper class which contains at least one function executed as part of the action chain for git push operations. * The function must be called `exec` and take in two parameters: an Express Request (req) and the current Action * executed in the chain (action). This function should return a Promise that resolves to an Action. - * + * * Optionally, child classes which extend this can simply define the `exec` function as their own property. * This is the preferred implementation when a custom plugin (subclass) has its own state or additional methods * that are required. - * + * * @param {function} exec - A function that: * - Takes in an Express Request object as the first parameter (`req`). * - Takes in an Action object as the second parameter (`action`). @@ -201,11 +211,11 @@ class PullActionPlugin extends ProxyPlugin { * Wrapper class which contains at least one function executed as part of the action chain for git pull operations. * The function must be called `exec` and take in two parameters: an Express Request (req) and the current Action * executed in the chain (action). This function should return a Promise that resolves to an Action. - * + * * Optionally, child classes which extend this can simply define the `exec` function as their own property. * This is the preferred implementation when a custom plugin (subclass) has its own state or additional methods * that are required. - * + * * @param {function} exec - A function that: * - Takes in an Express Request object as the first parameter (`req`). * - Takes in an Action object as the second parameter (`action`). @@ -218,9 +228,4 @@ class PullActionPlugin extends ProxyPlugin { } } -export { - PluginLoader, - PushActionPlugin, - PullActionPlugin, - isCompatiblePlugin, -} +export { PluginLoader, PushActionPlugin, PullActionPlugin, isCompatiblePlugin }; diff --git a/src/proxy/actions/Action.ts b/src/proxy/actions/Action.ts index 78dbc2ef0..b9d5e5ed3 100644 --- a/src/proxy/actions/Action.ts +++ b/src/proxy/actions/Action.ts @@ -1,5 +1,5 @@ -import { getProxyUrl } from "../../config"; -import { Step } from "./Step"; +import { processGitURLForNameAndOrg, processUrlPath } from '../routes/helper'; +import { Step } from './Step'; /** * Represents a commit. @@ -55,22 +55,36 @@ class Action { * @param {string} type The type of the action * @param {string} method The method of the action * @param {number} timestamp The timestamp of the action - * @param {string} repo The repo of the action + * @param {string} url The URL to the repo that should be proxied (with protocol, origin, repo path, but not the path for the git operation). */ - constructor(id: string, type: string, method: string, timestamp: number, repo: string) { + constructor(id: string, type: string, method: string, timestamp: number, url: string) { this.id = id; this.type = type; this.method = method; this.timestamp = timestamp; - this.project = repo.split("/")[0]; - this.repoName = repo.split("/")[1]; - this.url = `${getProxyUrl()}/${repo}`; - this.repo = repo; + this.url = url; + + const urlBreakdown = processUrlPath(url); + if (urlBreakdown) { + this.repo = urlBreakdown.repoPath; + const repoBreakdown = processGitURLForNameAndOrg(urlBreakdown.repoPath); + if (repoBreakdown) { + this.project = repoBreakdown.project ?? ''; + this.repoName = repoBreakdown.repoName; + } else { + this.project = 'UNKNOWN'; + this.repoName = 'UNKNOWN'; + } + } else { + this.repo = 'NOT-FOUND'; + this.project = 'UNKNOWN'; + this.repoName = 'UNKNOWN'; + } } /** * Add a step to the action. - * @param {Step} step + * @param {Step} step */ addStep(step: Step): void { this.steps.push(step); diff --git a/src/proxy/actions/Step.ts b/src/proxy/actions/Step.ts index cdb07bf95..6eb114e9c 100644 --- a/src/proxy/actions/Step.ts +++ b/src/proxy/actions/Step.ts @@ -1,4 +1,4 @@ -import { v4 as uuidv4 } from "uuid"; +import { v4 as uuidv4 } from 'uuid'; /** Class representing a Push Step. */ class Step { @@ -17,7 +17,7 @@ class Step { errorMessage: string | null = null, blocked: boolean = false, blockedMessage: string | null = null, - content: any = null + content: any = null, ) { this.id = uuidv4(); this.stepName = stepName; @@ -35,12 +35,12 @@ class Step { } setContent(content: any): void { - this.log("setting content"); + this.log('setting content'); this.content = content; } setAsyncBlock(message: string): void { - this.log("setting blocked"); + this.log('setting blocked'); this.blocked = true; this.blockedMessage = message; } diff --git a/src/proxy/actions/autoActions.ts b/src/proxy/actions/autoActions.ts index 03ed0529a..450c97d80 100644 --- a/src/proxy/actions/autoActions.ts +++ b/src/proxy/actions/autoActions.ts @@ -33,7 +33,4 @@ const attemptAutoRejection = async (action: Action) => { } }; -export { - attemptAutoApproval, - attemptAutoRejection, -}; +export { attemptAutoApproval, attemptAutoRejection }; diff --git a/src/proxy/actions/index.ts b/src/proxy/actions/index.ts index 72aa2918a..13f35276c 100644 --- a/src/proxy/actions/index.ts +++ b/src/proxy/actions/index.ts @@ -1,7 +1,4 @@ import { Action } from './Action'; import { Step } from './Step'; -export { - Action, - Step -} +export { Action, Step }; diff --git a/src/proxy/index.ts b/src/proxy/index.ts index 4cfcda986..afd2d7923 100644 --- a/src/proxy/index.ts +++ b/src/proxy/index.ts @@ -47,9 +47,9 @@ export const proxyPreparations = async () => { defaultAuthorisedRepoList.forEach(async (x) => { const found = allowedList.find((y) => y.project === x.project && x.name === y.name); if (!found) { - await createRepo(x); - await addUserCanPush(x.name, 'admin'); - await addUserCanAuthorise(x.name, 'admin'); + const repo = await createRepo(x); + await addUserCanPush(repo._id!, 'admin'); + await addUserCanAuthorise(repo._id!, 'admin'); } }); }; diff --git a/src/proxy/processors/pre-processor/parseAction.ts b/src/proxy/processors/pre-processor/parseAction.ts index ed610d9d1..ddf854caf 100644 --- a/src/proxy/processors/pre-processor/parseAction.ts +++ b/src/proxy/processors/pre-processor/parseAction.ts @@ -1,36 +1,48 @@ import { Action } from '../../actions'; +import { processUrlPath } from '../../routes/helper'; +import * as db from '../../../db'; -const exec = async (req: { originalUrl: string; method: string; headers: Record }) => { +const exec = async (req: { + originalUrl: string; + method: string; + headers: Record; +}) => { const id = Date.now(); const timestamp = id; - const repoName = getRepoNameFromUrl(req.originalUrl); - const paths = req.originalUrl.split('/'); - + const pathBreakdown = processUrlPath(req.originalUrl); let type = 'default'; + if (pathBreakdown) { + if (pathBreakdown.gitPath.endsWith('git-upload-pack') && req.method === 'GET') { + type = 'pull'; + } + if ( + pathBreakdown.gitPath.includes('git-receive-pack') && + req.method === 'POST' && + req.headers['content-type'] === 'application/x-git-receive-pack-request' + ) { + type = 'push'; + } + } // else failed to parse proxy URL path - which is logged in the parsing util - if (paths[paths.length - 1].endsWith('git-upload-pack') && req.method === 'GET') { - type = 'pull'; - } - if ( - paths[paths.length - 1] === 'git-receive-pack' && - req.method === 'POST' && - req.headers['content-type'] === 'application/x-git-receive-pack-request' - ) { - type = 'push'; - } + // Proxy URLs take the form https://:// + // e.g. https://git-proxy-instance.com:8443/github.com/finos/git-proxy.git + // We'll receive /github.com/finos/git-proxy.git as the req.url / req.originalUrl + // Add protocol (assume SSL) to reconstruct full URL - noting path will start with a / + let url = 'https:/' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); - return new Action(id.toString(), type, req.method, timestamp, repoName); -}; + console.log(`Parse action calculated repo URL: ${url} for inbound URL path: ${req.originalUrl}`); -const getRepoNameFromUrl = (url: string): string => { - const parts = url.split('/'); - for (let i = 0, len = parts.length; i < len; i++) { - const part = parts[i]; - if (part.endsWith('.git')) { - return `${parts[i - 1]}/${part}`.trim(); - } + if (!(await db.getRepoByUrl(url))) { + // fallback for legacy proxy URLs + // legacy git proxy paths took the form: https://:/ + // by assuming the host was github.com + url = 'https://github.com' + (pathBreakdown?.repoPath ?? 'NOT-FOUND'); + console.log( + `Parse action fallback calculated repo URL: ${url} for inbound URL path: ${req.originalUrl}`, + ); } - return 'NOT-FOUND'; + + return new Action(id.toString(), type, req.method, timestamp, url); }; exec.displayName = 'parseAction.exec'; diff --git a/src/proxy/processors/push-action/checkAuthorEmails.ts b/src/proxy/processors/push-action/checkAuthorEmails.ts index a25a0e6c9..367514e16 100644 --- a/src/proxy/processors/push-action/checkAuthorEmails.ts +++ b/src/proxy/processors/push-action/checkAuthorEmails.ts @@ -27,14 +27,16 @@ const isEmailAllowed = (email: string): boolean => { } return true; -} +}; const exec = async (req: any, action: Action): Promise => { console.log({ req, action }); const step = new Step('checkAuthorEmails'); - const uniqueAuthorEmails = [...new Set(action.commitData?.map((commit: Commit) => commit.authorEmail))]; + const uniqueAuthorEmails = [ + ...new Set(action.commitData?.map((commit: Commit) => commit.authorEmail)), + ]; console.log({ uniqueAuthorEmails }); const illegalEmails = uniqueAuthorEmails.filter((email) => !isEmailAllowed(email)); diff --git a/src/proxy/processors/push-action/checkCommitMessages.ts b/src/proxy/processors/push-action/checkCommitMessages.ts index 7a95f6c12..01517539d 100644 --- a/src/proxy/processors/push-action/checkCommitMessages.ts +++ b/src/proxy/processors/push-action/checkCommitMessages.ts @@ -47,7 +47,7 @@ const isMessageAllowed = (commitMessage: string): boolean => { } return true; -} +}; // Execute if the repo is approved const exec = async (req: any, action: Action): Promise => { diff --git a/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts b/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts index 9560dc58d..a06a7d995 100644 --- a/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts +++ b/src/proxy/processors/push-action/checkRepoInAuthorisedList.ts @@ -1,29 +1,15 @@ import { Action, Step } from '../../actions'; -import { getRepos } from '../../../db'; -import { Repo } from '../../../db/types'; +import { getRepoByUrl } from '../../../db'; // Execute if the repo is approved -const exec = async ( - req: any, - action: Action, - authorisedList: () => Promise = getRepos, -): Promise => { +const exec = async (req: any, action: Action): Promise => { const step = new Step('checkRepoInAuthorisedList'); - const list = await authorisedList(); - console.log(list); - - const found = list.find((x: Repo) => { - const targetName = action.repo.replace('.git', '').toLowerCase(); - const allowedName = `${x.project}/${x.name}`.replace('.git', '').toLowerCase(); - console.log(`${targetName} = ${allowedName}`); - return targetName === allowedName; - }); - - console.log(found); + // console.log(found); + const found = (await getRepoByUrl(action.url)) !== null; if (!found) { - console.log('not found'); + console.log(`Repository url '${action.url}' not found`); step.error = true; step.log(`repo ${action.repo} is not in the authorisedList, ending`); console.log('setting error'); @@ -33,7 +19,7 @@ const exec = async ( } console.log('found'); - step.log(`repo ${action.repo} is in the authorisedList`); + step.log(`repo ${action.url} is in the authorisedList`); action.addStep(step); return action; }; diff --git a/src/proxy/processors/push-action/checkUserPushPermission.ts b/src/proxy/processors/push-action/checkUserPushPermission.ts index c712693e5..1cd57538c 100644 --- a/src/proxy/processors/push-action/checkUserPushPermission.ts +++ b/src/proxy/processors/push-action/checkUserPushPermission.ts @@ -5,7 +5,6 @@ import { getUsers, isUserPushAllowed } from '../../../db'; const exec = async (req: any, action: Action): Promise => { const step = new Step('checkUserPushPermission'); - const repoName = action.repo.split('/')[1].replace('.git', ''); let isUserAllowed = false; let user = action.user; @@ -16,28 +15,28 @@ const exec = async (req: any, action: Action): Promise => { if (list.length == 1) { user = list[0].username; - isUserAllowed = await isUserPushAllowed(repoName, user); + isUserAllowed = await isUserPushAllowed(action.url, user!); } - console.log(`User ${user} permission on Repo ${repoName} : ${isUserAllowed}`); + console.log(`User ${user} permission on Repo ${action.url} : ${isUserAllowed}`); if (!isUserAllowed) { console.log('User not allowed to Push'); step.error = true; - step.log(`User ${user} is not allowed to push on repo ${action.repo}, ending`); + step.log(`User ${user} is not allowed to push on repo ${action.url}, ending`); console.log('setting error'); step.setError( `Rejecting push as user ${action.user} ` + - `is not allowed to push on repo ` + - `${action.repo}`, + `is not allowed to push on repo ` + + `${action.repo}`, ); action.addStep(step); return action; } - step.log(`User ${user} is allowed to push on repo ${action.repo}`); + step.log(`User ${user} is allowed to push on repo ${action.url}`); action.addStep(step); return action; }; diff --git a/src/proxy/processors/push-action/preReceive.ts b/src/proxy/processors/push-action/preReceive.ts index 1a8b23d89..1c3ad36b9 100644 --- a/src/proxy/processors/push-action/preReceive.ts +++ b/src/proxy/processors/push-action/preReceive.ts @@ -10,7 +10,7 @@ const sanitizeInput = (_req: any, action: Action): string => { const exec = async ( req: any, action: Action, - hookFilePath: string = './hooks/pre-receive.sh' + hookFilePath: string = './hooks/pre-receive.sh', ): Promise => { const step = new Step('executeExternalPreReceiveHook'); let stderrTrimmed = ''; diff --git a/src/proxy/processors/push-action/pullRemote.ts b/src/proxy/processors/push-action/pullRemote.ts index c7559643f..991a62ca9 100644 --- a/src/proxy/processors/push-action/pullRemote.ts +++ b/src/proxy/processors/push-action/pullRemote.ts @@ -1,5 +1,5 @@ import { Action, Step } from '../../actions'; -import fs from 'fs' +import fs from 'fs'; import git from 'isomorphic-git'; import gitHttpClient from 'isomorphic-git/http/node'; @@ -22,24 +22,23 @@ const exec = async (req: any, action: Action): Promise => { } const cmd = `git clone ${action.url}`; - step.log(`Exectuting ${cmd}`); + step.log(`Executing ${cmd}`); const authHeader = req.headers?.authorization; const [username, password] = Buffer.from(authHeader.split(' ')[1], 'base64') .toString() .split(':'); - await git - .clone({ - fs, - http: gitHttpClient, - url: action.url, - onAuth: () => ({ - username, - password, - }), - dir: `${action.proxyGitPath}/${action.repoName}`, - }); + await git.clone({ + fs, + http: gitHttpClient, + url: action.url, + onAuth: () => ({ + username, + password, + }), + dir: `${action.proxyGitPath}/${action.repoName}`, + }); console.log('Clone Success: ', action.url); diff --git a/src/proxy/processors/push-action/scanDiff.ts b/src/proxy/processors/push-action/scanDiff.ts index 296c8b404..899fa6442 100644 --- a/src/proxy/processors/push-action/scanDiff.ts +++ b/src/proxy/processors/push-action/scanDiff.ts @@ -8,13 +8,13 @@ const privateOrganizations = getPrivateOrganizations(); const BLOCK_TYPE = { LITERAL: 'Offending Literal', PATTERN: 'Offending Pattern', - PROVIDER: 'PROVIDER' -} + PROVIDER: 'PROVIDER', +}; type CombinedMatch = { type: string; match: RegExp; -} +}; type RawMatch = { type: string; @@ -22,7 +22,7 @@ type RawMatch = { file?: string; lines: number[]; content: string; -} +}; type Match = { type: string; @@ -30,7 +30,7 @@ type Match = { file?: string; lines: string; content: string; -} +}; const getDiffViolations = (diff: string, organization: string): Match[] | string | null => { // Commit diff is empty, i.e. '', null or undefined @@ -53,7 +53,7 @@ const getDiffViolations = (diff: string, organization: string): Match[] | string if (res.length > 0) { console.log('Diff is blocked via configured literals/patterns/providers...'); // combining matches with file and line number - return res + return res; } return null; @@ -67,80 +67,78 @@ const combineMatches = (organization: string) => { const blockedPatterns: string[] = commitConfig.diff.block.patterns; // Configured blocked providers - const blockedProviders: [string, string][] = organization && privateOrganizations.includes(organization) ? [] : - Object.entries(commitConfig.diff.block.providers); + const blockedProviders: [string, string][] = + organization && privateOrganizations.includes(organization) + ? [] + : Object.entries(commitConfig.diff.block.providers); - // Combine all matches (literals, paterns) + // Combine all matches (literals, patterns) const combinedMatches = [ - ...blockedLiterals.map(literal => ({ + ...blockedLiterals.map((literal) => ({ type: BLOCK_TYPE.LITERAL, - match: new RegExp(literal, 'gi') + match: new RegExp(literal, 'gi'), })), - ...blockedPatterns.map(pattern => ({ + ...blockedPatterns.map((pattern) => ({ type: BLOCK_TYPE.PATTERN, - match: new RegExp(pattern, 'gi') + match: new RegExp(pattern, 'gi'), })), ...blockedProviders.map(([key, value]) => ({ type: key, - match: new RegExp(value, 'gi') + match: new RegExp(value, 'gi'), })), ]; return combinedMatches; -} +}; -const collectMatches = ( - parsedDiff: File[], - combinedMatches: CombinedMatch[] -): Match[] => { +const collectMatches = (parsedDiff: File[], combinedMatches: CombinedMatch[]): Match[] => { const allMatches: Record = {}; - parsedDiff.forEach(file => { + parsedDiff.forEach((file) => { const fileName = file.to || file.from; - console.log("CHANGE", file.chunks) + console.log('CHANGE', file.chunks); - file.chunks.forEach(chunk => { - chunk.changes.forEach(change => { - console.log("CHANGE", change) + file.chunks.forEach((chunk) => { + chunk.changes.forEach((change) => { + console.log('CHANGE', change); if (change.type === 'add') { // store line number const lineNumber = change.ln; // Iterate through each match types - literal, patterns, providers combinedMatches.forEach(({ type, match }) => { - // using Match all to find all occurences of the pattern in the line - const matches = [...change.content.matchAll(match)] + // using Match all to find all occurrences of the pattern in the line + const matches = [...change.content.matchAll(match)]; - matches.forEach(matchInstance => { + matches.forEach((matchInstance) => { const matchLiteral = matchInstance[0]; const matchKey = `${type}_${matchLiteral}_${fileName}`; // unique key - if (!allMatches[matchKey]) { - // match entry + // match entry allMatches[matchKey] = { type, literal: matchLiteral, file: fileName, lines: [], - content: change.content.trim() + content: change.content.trim(), }; } - // apend line numbers to the list of lines - allMatches[matchKey].lines.push(lineNumber) - }) + // append line numbers to the list of lines + allMatches[matchKey].lines.push(lineNumber); + }); }); } }); }); }); - // convert matches into a final result array, joining line numbers - const result = Object.values(allMatches).map(match => ({ + // convert matches into a final result array, joining line numbers + const result = Object.values(allMatches).map((match) => ({ ...match, - lines: match.lines.join(',') // join the line numbers into a comma-separated string - })) + lines: match.lines.join(','), // join the line numbers into a comma-separated string + })); return result; -} +}; const formatMatches = (matches: Match[]) => { return matches.map((match, index) => { @@ -148,9 +146,9 @@ const formatMatches = (matches: Match[]) => { Policy Exception Type: ${match.type} DETECTED: ${match.literal} FILE(S) LOCATED: ${match.file} - Line(s) of code: ${match.lines}` + Line(s) of code: ${match.lines}`; }); -} +}; const exec = async (req: any, action: Action): Promise => { const step = new Step('scanDiff'); @@ -160,24 +158,24 @@ const exec = async (req: any, action: Action): Promise => { const diff = steps.find((s) => s.stepName === 'diff')?.content; - console.log(diff) + console.log(diff); const diffViolations = getDiffViolations(diff, action.project); if (diffViolations) { - const formattedMatches = Array.isArray(diffViolations) ? formatMatches(diffViolations).join('\n\n') : diffViolations; + const formattedMatches = Array.isArray(diffViolations) + ? formatMatches(diffViolations).join('\n\n') + : diffViolations; const errorMsg = []; errorMsg.push(`\n\n\n\nYour push has been blocked.\n`); errorMsg.push(`Please ensure your code does not contain sensitive information or URLs.\n\n`); - errorMsg.push(formattedMatches) - errorMsg.push('\n') + errorMsg.push(formattedMatches); + errorMsg.push('\n'); console.log(`The following diff is illegal: ${commitFrom}:${commitTo}`); step.error = true; step.log(`The following diff is illegal: ${commitFrom}:${commitTo}`); - step.setError( - errorMsg.join('\n') - ); + step.setError(errorMsg.join('\n')); action.addStep(step); return action; diff --git a/src/proxy/processors/types.ts b/src/proxy/processors/types.ts index bb267ce90..ae92ba48c 100644 --- a/src/proxy/processors/types.ts +++ b/src/proxy/processors/types.ts @@ -1,4 +1,4 @@ -import { Action } from "../actions"; +import { Action } from '../actions'; export interface Processor { exec(req: any, action: Action): Promise; @@ -17,4 +17,4 @@ export type CommitContent = { deflatedSize: number; objectRef: any; content: string; -} +}; diff --git a/src/proxy/routes/helper.ts b/src/proxy/routes/helper.ts new file mode 100644 index 000000000..1db075b9e --- /dev/null +++ b/src/proxy/routes/helper.ts @@ -0,0 +1,165 @@ +import * as db from '../../db'; + +/** Regex used to analyze un-proxied Git URLs */ +const GIT_URL_REGEX = /(.+:\/\/)([^/]+)(\/.+\.git)(\/.+)*/; + +/** Type representing a breakdown of Git URL (un-proxied)*/ +export type GitUrlBreakdown = { protocol: string; host: string; repoPath: string }; + +/** Function that processes Git URLs to extract the protocol, host, path to the + * git endpoint and discarding any git path (specific operation) that comes after + * the .git element. + * + * E.g. Processing https://github.com/finos/git-proxy.git/info/refs?service=git-upload-pack + * would produce: + * - protocol: https:// + * - host: github.com + * - repoPath: /finos/git-proxy.git + * + * and processing https://someOtherHost.com:8080/repo.git + * would produce: + * - protocol: https:// + * - host: someOtherHost.com:8080 + * - repoPath: /repo.git + * + * @param {string} url The URL to process + * @return {GitUrlBreakdown | null} A breakdown of the components of the URL. + */ +export const processGitUrl = (url: string): GitUrlBreakdown | null => { + const components = url.match(GIT_URL_REGEX); + if (components && components.length >= 5) { + return { + protocol: components[1], + host: components[2], + repoPath: components[3], + // component [4] would be any git path, but isn't needed for repo URLs + }; + } else { + console.error(`Failed to parse git URL: ${url}`); + return null; + } +}; + +/** Regex used to analyze url paths for requests to the proxy and split them + * into the embedded git end point and path for the git operation. */ +const PROXIED_URL_PATH_REGEX = /(.+\.git)(\/.*)?/; + +/** Type representing a breakdown of paths requested from the proxy server */ +export type UrlPathBreakdown = { repoPath: string; gitPath: string }; + +/** Function that processes URL paths (URL with origin removed) of requests to the proxy + * to extract the embedded repository path and path for the specific git operation to be + * proxied. + * + * E.g. Processing /finos/git-proxy.git/info/refs?service=git-upload-pack + * would produce: + * - repoPath: /finos/git-proxy.git + * - gitPath: /info/refs?service=git-upload-pack + * + * and processing /github.com/finos/git-proxy.git/info/refs?service=git-upload-pack + * would produce: + * - repoPath: /github.com/finos/git-proxy.git + * - gitPath: /info/refs?service=git-upload-pack + * + * @param {string} requestPath The URL path to process. + * @return {GitUrlBreakdown | null} A breakdown of the components of the URL path. + */ +export const processUrlPath = (requestPath: string): UrlPathBreakdown | null => { + const components = requestPath.match(PROXIED_URL_PATH_REGEX); + if (components && components.length >= 3) { + return { + repoPath: components[1], + gitPath: components[2] ?? '/', + }; + } else { + console.error(`Failed to parse proxy url path: ${requestPath}`); + return null; + } +}; + +/** Regex used to analyze repo URLs (with protocol and origin) to extract the repository name and + * any path or organisation that proceeds and drop the origin and protocol if present. */ +const GIT_URL_NAME_ORG_REGEX = /(.+:\/\/)?([^/]+)\/(?:(.*)\/)?([^/]+\.git)/; + +/** Type representing a breakdown Git URL into repository name and organisation (project). */ +export type GitNameBreakdown = { project: string | null; repoName: string }; + +/** Function that processes git URLs embedded in proxy request URLs to extract + * the repository name and any path or organisation. + * + * E.g. Processing https://github.com/finos/git-proxy.git + * would produce: + * - project: finos + * - repoName: git-proxy.git + * + * Processing https://someGitHost.com/repo.git + * would produce: + * - project: null + * - repoName: repo.git + * + * Processing https://anotherGitHost.com/project/subProject/subSubProject/repo.git + * would produce: + * - project: project/subProject/subSubProject + * - repoName: repo.git + * + * @param {string} gitUrl The URL to process. + * @return {GitNameBreakdown | null} A breakdown of the components of the URL. + */ +export const processGitURLForNameAndOrg = (gitUrl: string): GitNameBreakdown | null => { + const components = gitUrl.match(GIT_URL_NAME_ORG_REGEX); + if (components && components.length >= 5) { + return { + project: components[3] ?? null, // there may be no project or path for standalone git repo + repoName: components[4], + }; + } else { + console.error(`Failed to parse git URL: ${gitUrl}`); + return null; + } +}; + +/** + * Check whether an HTTP request has the expected properties of a + * Git HTTP request. The URL is expected to be "sanitized", stripped of + * specific paths such as the GitHub {owner}/{repo}.git parts. + * @param {string} gitPath Sanitized URL path which only includes the path + * specific to git (everything after .git/) + * @param {*} headers Request headers (TODO: Fix JSDoc linting and refer to + * node:http.IncomingHttpHeaders) + * @return {boolean} If true, this is a valid and expected git request. + * Otherwise, false. + */ +export const validGitRequest = (gitPath: string, headers: any): boolean => { + const { 'user-agent': agent, accept } = headers; + if ( + ['/info/refs?service=git-upload-pack', '/info/refs?service=git-receive-pack'].includes(gitPath) + ) { + // https://www.git-scm.com/docs/http-protocol#_discovering_references + // We can only filter based on User-Agent since the Accept header is not + // sent in this request + return agent.startsWith('git/'); + } + if (['/git-upload-pack', '/git-receive-pack'].includes(gitPath)) { + // https://www.git-scm.com/docs/http-protocol#_uploading_data + return agent.startsWith('git/') && accept.startsWith('application/x-git-'); + } + return false; +}; + +/** + * Collect the Set of all host (host and port if specified) that + * will be proxying requests for, to be used to initialize the proxy. + * + * @return {string[]} an array of origins + */ +export const getAllProxiedHosts = async (): Promise => { + const repos = await db.getRepos(); + const origins = new Set(); + repos.forEach((repo) => { + const parsedUrl = processGitUrl(repo.url); + if (parsedUrl) { + origins.add(parsedUrl.host); + } // failures are logged by parsing util fn + }); + return Array.from(origins); +}; diff --git a/src/proxy/routes/index.ts b/src/proxy/routes/index.ts index c49853376..27dd59fdc 100644 --- a/src/proxy/routes/index.ts +++ b/src/proxy/routes/index.ts @@ -1,140 +1,140 @@ import { Router } from 'express'; import proxy from 'express-http-proxy'; import { executeChain } from '../chain'; -import { getProxyUrl } from '../../config'; +import { processUrlPath, validGitRequest, getAllProxiedHosts } from './helper'; +import { ProxyOptions } from 'express-http-proxy'; + +const proxyFilter: ProxyOptions['filter'] = async (req, res) => { + try { + console.log('request url: ', req.url); + console.log('host: ', req.headers.host); + console.log('user-agent: ', req.headers['user-agent']); + + const urlComponents = processUrlPath(req.url); + + if ( + !urlComponents || + urlComponents.gitPath === undefined || + !validGitRequest(urlComponents.gitPath, req.headers) + ) { + res.status(400).send('Invalid request received'); + console.log('action blocked'); + return false; + } + + const action = await executeChain(req, res); + console.log('action processed'); + + if (action.error || action.blocked) { + res.set('content-type', 'application/x-git-receive-pack-result'); + res.set('transfer-encoding', 'chunked'); + res.set('expires', 'Fri, 01 Jan 1980 00:00:00 GMT'); + res.set('pragma', 'no-cache'); + res.set('cache-control', 'no-cache, max-age=0, must-revalidate'); + res.set('vary', 'Accept-Encoding'); + res.set('x-frame-options', 'DENY'); + res.set('connection', 'close'); + + let message = ''; + + if (action.error) { + message = action.errorMessage!; + console.error(message); + } + if (action.blocked) { + message = action.blockedMessage!; + } -// eslint-disable-next-line new-cap -const router = Router(); + const packetMessage = handleMessage(message); -/** - * For a given Git HTTP request destined for a GitHub repo, - * remove the GitHub specific components of the URL. - * @param {string} url URL path of the request - * @return {string} Modified path which removes the {owner}/{repo} parts - */ -const stripGitHubFromGitPath = (url: string): string | undefined => { - const parts = url.split('/'); - // url = '/{owner}/{repo}.git/{git-path}' - // url.split('/') = ['', '{owner}', '{repo}.git', '{git-path}'] - if (parts.length !== 4 && parts.length !== 5) { - console.error('unexpected url received: ', url); - return undefined; - } - parts.splice(1, 2); // remove the {owner} and {repo} from the array - return parts.join('/'); -}; + console.log(req.headers); -/** - * Check whether an HTTP request has the expected properties of a - * Git HTTP request. The URL is expected to be "sanitized", stripped of - * specific paths such as the GitHub {owner}/{repo}.git parts. - * @param {string} url Sanitized URL which only includes the path specific to git - * @param {*} headers Request headers (TODO: Fix JSDoc linting and refer to node:http.IncomingHttpHeaders) - * @return {boolean} If true, this is a valid and expected git request. Otherwise, false. - */ -const validGitRequest = (url: string, headers: any): boolean => { - const { 'user-agent': agent, accept } = headers; - if (['/info/refs?service=git-upload-pack', '/info/refs?service=git-receive-pack'].includes(url)) { - // https://www.git-scm.com/docs/http-protocol#_discovering_references - // We can only filter based on User-Agent since the Accept header is not - // sent in this request - return agent.startsWith('git/'); - } - if (['/git-upload-pack', '/git-receive-pack'].includes(url)) { - // https://www.git-scm.com/docs/http-protocol#_uploading_data - return agent.startsWith('git/') && accept.startsWith('application/x-git-'); - } - return false; -}; + res.status(200).send(packetMessage); -router.use( - '/', - proxy(getProxyUrl(), { - preserveHostHdr: false, - filter: async function (req, res) { - try { - console.log('request url: ', req.url); - console.log('host: ', req.headers.host); - console.log('user-agent: ', req.headers['user-agent']); - const gitPath = stripGitHubFromGitPath(req.url); - if (gitPath === undefined || !validGitRequest(gitPath, req.headers)) { - res.status(400).send('Invalid request received'); - return false; - } - - const action = await executeChain(req, res); - console.log('action processed'); - - if (action.error || action.blocked) { - res.set('content-type', 'application/x-git-receive-pack-result'); - res.set('transfer-encoding', 'chunked'); - res.set('expires', 'Fri, 01 Jan 1980 00:00:00 GMT'); - res.set('pragma', 'no-cache'); - res.set('cache-control', 'no-cache, max-age=0, must-revalidate'); - res.set('vary', 'Accept-Encoding'); - res.set('x-frame-options', 'DENY'); - res.set('connection', 'close'); - - let message = ''; - - if (action.error) { - message = action.errorMessage!; - console.error(message); - } - if (action.blocked) { - message = action.blockedMessage!; - } - - const packetMessage = handleMessage(message); - - console.log(req.headers); - - res.status(200).send(packetMessage); - - return false; - } - - return true; - } catch (e) { - console.error(e); - return false; - } - }, - proxyReqPathResolver: (req) => { - const url = getProxyUrl() + req.originalUrl; - console.log('Sending request to ' + url); - return url; - }, - proxyReqOptDecorator: function (proxyReqOpts) { - return proxyReqOpts; - }, - - proxyReqBodyDecorator: function (bodyContent, srcReq) { - if (srcReq.method === 'GET') { - return ''; - } - return bodyContent; - }, + return false; + } - proxyErrorHandler: function (err, res, next) { - console.log(`ERROR=${err}`); - next(err); - }, - }), -); + return true; + } catch (e) { + console.error('Error occurred in proxy filter function ', e); + return false; + } +}; const handleMessage = (message: string): string => { const errorMessage = `\t${message}`; const len = 6 + new TextEncoder().encode(errorMessage).length; - const prefix = len.toString(16); const packetMessage = `${prefix.padStart(4, '0')}\x02${errorMessage}\n0000`; return packetMessage; }; -export { - router, - handleMessage, - validGitRequest, - stripGitHubFromGitPath, +const getRequestPathResolver: (prefix: string) => ProxyOptions['proxyReqPathResolver'] = ( + prefix, +) => { + return (req) => { + let url; + // try to prevent too many slashes in the URL + if (prefix.endsWith('/') && req.originalUrl.startsWith('/')) { + url = prefix.substring(0, prefix.length - 1) + req.originalUrl; + } else { + url = prefix + req.originalUrl; + } + + console.log(`Sending request to ${url}`); + return url; + }; +}; + +const proxyReqOptDecorator: ProxyOptions['proxyReqOptDecorator'] = (proxyReqOpts) => proxyReqOpts; + +const proxyReqBodyDecorator: ProxyOptions['proxyReqBodyDecorator'] = (bodyContent, srcReq) => { + if (srcReq.method === 'GET') { + return ''; + } + return bodyContent; +}; + +const proxyErrorHandler: ProxyOptions['proxyErrorHandler'] = (err, res, next) => { + console.log(`ERROR=${err}`); + next(err); }; + +// eslint-disable-next-line new-cap +const router = Router(); + +getAllProxiedHosts().then((originsToProxy) => { + // TODO: this will only happen on startup. We'll need to add routes at runtime when new origins are added? Or force a restart for the proxy to work + + // Middlewares are processed in the order that they are added, if one applies and then doesn't call `next` then subsequent ones are not applied. + // Hence, we define known origins first, then a catch all route for backwards compatibility + originsToProxy.forEach((origin) => { + console.log(`setting up origin '${origin}'`); + router.use( + '/' + origin, + proxy('https://' + origin, { + preserveHostHdr: false, + filter: proxyFilter, + proxyReqPathResolver: getRequestPathResolver('https://'), // no need to add host as it's in the URL + proxyReqOptDecorator: proxyReqOptDecorator, + proxyReqBodyDecorator: proxyReqBodyDecorator, + proxyErrorHandler: proxyErrorHandler, + }), + ); + }); + + // Catch-all route for backwards compatibility + router.use( + '/', + proxy('https://github.com', { + preserveHostHdr: false, + filter: proxyFilter, + proxyReqPathResolver: getRequestPathResolver('https://github.com'), + proxyReqOptDecorator: proxyReqOptDecorator, + proxyReqBodyDecorator: proxyReqBodyDecorator, + proxyErrorHandler: proxyErrorHandler, + }), + ); +}); + +export { router, handleMessage, validGitRequest }; diff --git a/src/service/passport/oidc.js b/src/service/passport/oidc.js index 18fdf7de9..52928460b 100644 --- a/src/service/passport/oidc.js +++ b/src/service/passport/oidc.js @@ -7,11 +7,13 @@ const configure = async (passport) => { const { discovery, fetchUserInfo } = await import('openid-client'); const { Strategy } = await import('openid-client/passport'); const authMethods = require('../../config').getAuthMethods(); - const oidcConfig = authMethods.find((method) => method.type.toLowerCase() === "openidconnect")?.oidcConfig; + const oidcConfig = authMethods.find( + (method) => method.type.toLowerCase() === 'openidconnect', + )?.oidcConfig; const { issuer, clientID, clientSecret, callbackURL, scope } = oidcConfig; if (!oidcConfig || !oidcConfig.issuer) { - throw new Error('Missing OIDC issuer in configuration') + throw new Error('Missing OIDC issuer in configuration'); } const server = new URL(issuer); @@ -26,7 +28,7 @@ const configure = async (passport) => { const userInfo = await fetchUserInfo(config, tokenSet.access_token, expectedSub); handleUserAuthentication(userInfo, done); }); - + // currentUrl must be overridden to match the callback URL strategy.currentUrl = function (request) { const callbackUrl = new URL(callbackURL); @@ -41,7 +43,7 @@ const configure = async (passport) => { passport.serializeUser((user, done) => { done(null, user.oidcId || user.username); - }) + }); passport.deserializeUser(async (id, done) => { try { @@ -50,18 +52,18 @@ const configure = async (passport) => { } catch (err) { done(err); } - }) + }); return passport; } catch (error) { console.error('OIDC configuration failed:', error); throw error; } -} +}; /** * Handles user authentication with OIDC. - * @param {Object} userInfo the OIDC user info object + * @param {Object} userInfo the OIDC user info object * @param {Function} done the callback function * @return {Promise} a promise with the authenticated user or an error */ @@ -97,7 +99,9 @@ const handleUserAuthentication = async (userInfo, done) => { * @return {string | null} the email address */ const safelyExtractEmail = (profile) => { - return profile.email || (profile.emails && profile.emails.length > 0 ? profile.emails[0].value : null); + return ( + profile.email || (profile.emails && profile.emails.length > 0 ? profile.emails[0].value : null) + ); }; /** diff --git a/src/service/routes/auth.js b/src/service/routes/auth.js index aaf2efa26..9c5db2578 100644 --- a/src/service/routes/auth.js +++ b/src/service/routes/auth.js @@ -3,7 +3,8 @@ const router = new express.Router(); const passport = require('../passport').getPassport(); const authStrategies = require('../passport').authStrategies; const db = require('../../db'); -const { GIT_PROXY_UI_HOST: uiHost = 'http://localhost', GIT_PROXY_UI_PORT: uiPort = 3000 } = process.env; +const { GIT_PROXY_UI_HOST: uiHost = 'http://localhost', GIT_PROXY_UI_PORT: uiPort = 3000 } = + process.env; router.get('/', (req, res) => { res.status(200).json({ @@ -27,7 +28,7 @@ router.post('/login', passport.authenticate(authStrategies['local'].type), async const currentUser = { ...req.user }; delete currentUser.password; console.log( - `serivce.routes.auth.login: user logged in, username=${ + `service.routes.auth.login: user logged in, username=${ currentUser.username } profile=${JSON.stringify(currentUser)}`, ); diff --git a/src/service/routes/repo.js b/src/service/routes/repo.js index 1f6698e3b..b5ab5cd7d 100644 --- a/src/service/routes/repo.js +++ b/src/service/routes/repo.js @@ -5,9 +5,7 @@ const { getProxyURL } = require('../urls'); router.get('/', async (req, res) => { const proxyURL = getProxyURL(req); - const query = { - type: 'push', - }; + const query = {}; for (const k in req.query) { if (!k) continue; @@ -24,16 +22,16 @@ router.get('/', async (req, res) => { res.send(qd.map((d) => ({ ...d, proxyURL }))); }); -router.get('/:name', async (req, res) => { +router.get('/:id', async (req, res) => { const proxyURL = getProxyURL(req); - const name = req.params.name; - const qd = await db.getRepo(name); + const _id = req.params.id; + const qd = await db.getRepoById(_id); res.send({ ...qd, proxyURL }); }); -router.patch('/:name/user/push', async (req, res) => { +router.patch('/:id/user/push', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.body.username.toLowerCase(); const user = await db.findUser(username); @@ -42,7 +40,7 @@ router.patch('/:name/user/push', async (req, res) => { return; } - await db.addUserCanPush(repoName, username); + await db.addUserCanPush(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -51,9 +49,9 @@ router.patch('/:name/user/push', async (req, res) => { } }); -router.patch('/:name/user/authorise', async (req, res) => { +router.patch('/:id/user/authorise', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.body.username; const user = await db.findUser(username); @@ -62,7 +60,7 @@ router.patch('/:name/user/authorise', async (req, res) => { return; } - await db.addUserCanAuthorise(repoName, username); + await db.addUserCanAuthorise(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -71,9 +69,9 @@ router.patch('/:name/user/authorise', async (req, res) => { } }); -router.delete('/:name/user/authorise/:username', async (req, res) => { +router.delete('/:id/user/authorise/:username', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.params.username; const user = await db.findUser(username); @@ -82,7 +80,7 @@ router.delete('/:name/user/authorise/:username', async (req, res) => { return; } - await db.removeUserCanAuthorise(repoName, username); + await db.removeUserCanAuthorise(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -91,9 +89,9 @@ router.delete('/:name/user/authorise/:username', async (req, res) => { } }); -router.delete('/:name/user/push/:username', async (req, res) => { +router.delete('/:id/user/push/:username', async (req, res) => { if (req.user && req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; const username = req.params.username; const user = await db.findUser(username); @@ -102,7 +100,7 @@ router.delete('/:name/user/push/:username', async (req, res) => { return; } - await db.removeUserCanPush(repoName, username); + await db.removeUserCanPush(_id, username); res.send({ message: 'created' }); } else { res.status(401).send({ @@ -111,11 +109,11 @@ router.delete('/:name/user/push/:username', async (req, res) => { } }); -router.delete('/:name/delete', async (req, res) => { +router.delete('/:id/delete', async (req, res) => { if (req.user.admin) { - const repoName = req.params.name; + const _id = req.params.id; - await db.deleteRepo(repoName); + await db.deleteRepo(_id); res.send({ message: 'deleted' }); } else { res.status(401).send({ @@ -126,10 +124,10 @@ router.delete('/:name/delete', async (req, res) => { router.post('/', async (req, res) => { if (req.user && req.user.admin) { - const repo = await db.getRepo(req.body.name); + const repo = await db.getRepoByUrl(req.body.url); if (repo) { res.status(409).send({ - message: 'Repository already exists!', + message: `Repository ${req.body.url} already exists!`, }); } else { try { diff --git a/src/ui/components/Filtering/Filtering.css b/src/ui/components/Filtering/Filtering.css index 84f9258e0..a83724cb2 100644 --- a/src/ui/components/Filtering/Filtering.css +++ b/src/ui/components/Filtering/Filtering.css @@ -52,4 +52,4 @@ .dropdown-item:hover { background-color: #f0f0f0; -} \ No newline at end of file +} diff --git a/src/ui/components/Filtering/Filtering.jsx b/src/ui/components/Filtering/Filtering.jsx index aa9d26c78..1c9e2fcc7 100644 --- a/src/ui/components/Filtering/Filtering.jsx +++ b/src/ui/components/Filtering/Filtering.jsx @@ -27,29 +27,34 @@ const Filtering = ({ onFilterChange }) => { }; return ( -
-
+
+
{/* Make the entire button clickable for toggling dropdown */} - {isOpen && ( -
-
handleOptionClick('Date Modified')} className="dropdown-item"> +
+
handleOptionClick('Date Modified')} className='dropdown-item'> Date Modified
-
handleOptionClick('Date Created')} className="dropdown-item"> +
handleOptionClick('Date Created')} className='dropdown-item'> Date Created
-
handleOptionClick('Alphabetical')} className="dropdown-item"> +
handleOptionClick('Alphabetical')} className='dropdown-item'> Alphabetical
@@ -60,7 +65,3 @@ const Filtering = ({ onFilterChange }) => { }; export default Filtering; - - - - diff --git a/src/ui/components/Pagination/Pagination.jsx b/src/ui/components/Pagination/Pagination.jsx index e87e43c17..777807343 100644 --- a/src/ui/components/Pagination/Pagination.jsx +++ b/src/ui/components/Pagination/Pagination.jsx @@ -1,8 +1,7 @@ import React from 'react'; -import './Pagination.css'; +import './Pagination.css'; export default function Pagination({ currentPage, totalItems = 0, itemsPerPage, onPageChange }) { - const totalPages = Math.ceil(totalItems / itemsPerPage); const handlePageClick = (page) => { diff --git a/src/ui/components/Search/Search.css b/src/ui/components/Search/Search.css index db87dc8c0..d4c650d13 100644 --- a/src/ui/components/Search/Search.css +++ b/src/ui/components/Search/Search.css @@ -1,7 +1,7 @@ .search-bar { width: 100%; - max-width:100%; - margin: 0 auto 20px auto; + max-width: 100%; + margin: 0 auto 20px auto; } .search-input { @@ -10,9 +10,9 @@ font-size: 16px; border: 1px solid #ccc; border-radius: 4px; - box-sizing: border-box; + box-sizing: border-box; } .search-input:focus { - border-color: #007bff; + border-color: #007bff; } diff --git a/src/ui/components/Search/Search.jsx b/src/ui/components/Search/Search.jsx index 5e1cbf6b4..e774ea0f2 100644 --- a/src/ui/components/Search/Search.jsx +++ b/src/ui/components/Search/Search.jsx @@ -2,27 +2,26 @@ import React from 'react'; import { TextField } from '@material-ui/core'; import './Search.css'; import InputAdornment from '@material-ui/core/InputAdornment'; -import SearchIcon from '@material-ui/icons/Search'; - +import SearchIcon from '@material-ui/icons/Search'; export default function Search({ onSearch }) { const handleSearchChange = (event) => { const query = event.target.value; - onSearch(query); + onSearch(query); }; return (
+ ), @@ -31,7 +30,3 @@ export default function Search({ onSearch }) {
); } - - - - diff --git a/src/ui/services/repo.js b/src/ui/services/repo.js index 2ac0bc98d..8f5ab3343 100644 --- a/src/ui/services/repo.js +++ b/src/ui/services/repo.js @@ -9,8 +9,8 @@ const config = { withCredentials: true, }; -const canAddUser = (repoName, user, action) => { - const url = new URL(`${baseUrl}/repo/${repoName}`); +const canAddUser = (repoId, user, action) => { + const url = new URL(`${baseUrl}/repo/${repoId}`); return axios .get(url.toString(), config) .then((response) => { @@ -87,10 +87,10 @@ const addRepo = async (onClose, setError, data) => { }); }; -const addUser = async (repoName, user, action) => { - const canAdd = await canAddUser(repoName, user, action); +const addUser = async (repoId, user, action) => { + const canAdd = await canAddUser(repoId, user, action); if (canAdd) { - const url = new URL(`${baseUrl}/repo/${repoName}/user/${action}`); + const url = new URL(`${baseUrl}/repo/${repoId}/user/${action}`); const data = { username: user }; await axios .patch(url, data, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) @@ -104,8 +104,8 @@ const addUser = async (repoName, user, action) => { } }; -const deleteUser = async (user, repoName, action) => { - const url = new URL(`${baseUrl}/repo/${repoName}/user/${action}/${user}`); +const deleteUser = async (user, repoId, action) => { + const url = new URL(`${baseUrl}/repo/${repoId}/user/${action}/${user}`); await axios .delete(url, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) @@ -115,8 +115,8 @@ const deleteUser = async (user, repoName, action) => { }); }; -const deleteRepo = async (repoName) => { - const url = new URL(`${baseUrl}/repo/${repoName}/delete`); +const deleteRepo = async (repoId) => { + const url = new URL(`${baseUrl}/repo/${repoId}/delete`); await axios .delete(url, { withCredentials: true, headers: { 'X-CSRF-TOKEN': getCookie('csrf') } }) diff --git a/src/ui/views/OpenPushRequests/components/PushesTable.jsx b/src/ui/views/OpenPushRequests/components/PushesTable.jsx index 2a3a7f33a..309b38ed0 100644 --- a/src/ui/views/OpenPushRequests/components/PushesTable.jsx +++ b/src/ui/views/OpenPushRequests/components/PushesTable.jsx @@ -94,6 +94,9 @@ export default function PushesTable(props) { {currentItems.reverse().map((row) => { const repoFullName = row.repo.replace('.git', ''); const repoBranch = row.branch.replace('refs/heads/', ''); + const repoUrl = row.url; + const repoWebUrl = repoUrl.replace('.git', ''); + const isGitHub = repoUrl.startsWith('https://github.com'); return ( @@ -103,22 +106,18 @@ export default function PushesTable(props) { .toString()} - + {repoFullName} - + {repoBranch} @@ -126,22 +125,28 @@ export default function PushesTable(props) { - - {row.commitData[0].committer} - + {isGitHub && ( + + {row.commitData[0].committer} + + )} + {!isGitHub && {row.commitData[0].committer}} - - {row.commitData[0].author} - + {isGitHub && ( + + {row.commitData[0].author} + + )} + {!isGitHub && {row.commitData[0].author}} {row.commitData[0].authorEmail ? ( diff --git a/src/ui/views/PushDetails/PushDetails.jsx b/src/ui/views/PushDetails/PushDetails.jsx index cb094ab1b..9fb38de12 100644 --- a/src/ui/views/PushDetails/PushDetails.jsx +++ b/src/ui/views/PushDetails/PushDetails.jsx @@ -95,6 +95,9 @@ export default function Dashboard() { const repoFullName = data.repo.replace('.git', ''); const repoBranch = data.branch.replace('refs/heads/', ''); + const repoUrl = data.url; + const repoWebUrl = repoUrl.replace('.git', ''); + const isGitHub = repoUrl.startsWith('https://github.com'); const generateIcon = (title) => { switch (title) { @@ -197,17 +200,26 @@ export default function Dashboard() { ) : ( <> - - - + {isGitHub && ( + + + + )}

- - {data.attestation.reviewer.gitAccount} - {' '} + {isGitHub && ( + + {data.attestation.reviewer.gitAccount} + + )} + {!isGitHub && ( + + {data.attestation.reviewer.username} + + )}{' '} approved this contribution

@@ -247,7 +259,7 @@ export default function Dashboard() {

Remote Head

@@ -259,7 +271,7 @@ export default function Dashboard() {

Commit SHA

@@ -270,7 +282,7 @@ export default function Dashboard() {

Repository

- + {repoFullName}

@@ -278,11 +290,7 @@ export default function Dashboard() {

Branch

- + {repoBranch}

@@ -310,18 +318,28 @@ export default function Dashboard() { {moment.unix(c.commitTs || c.commitTimestamp).toString()} - - {c.committer} - + {isGitHub && ( + + {c.committer} + + )} + {!isGitHub && {c.committer}} - - {c.author} - + {isGitHub && ( + + {c.author} + + )} + {!isGitHub && {c.author}} {c.authorEmail ? ( diff --git a/src/ui/views/RepoDetails/Components/AddUser.jsx b/src/ui/views/RepoDetails/Components/AddUser.jsx index afab44a53..a4836a322 100644 --- a/src/ui/views/RepoDetails/Components/AddUser.jsx +++ b/src/ui/views/RepoDetails/Components/AddUser.jsx @@ -19,7 +19,7 @@ import { getUsers } from '../../../services/user'; import { PersonAdd } from '@material-ui/icons'; function AddUserDialog(props) { - const repoName = props.repoName; + const repoId = props.repoId; const type = props.type; const refreshFn = props.refreshFn; const [username, setUsername] = useState(''); @@ -48,7 +48,7 @@ function AddUserDialog(props) { const add = async () => { try { setIsLoading(true); - await addUser(repoName, username, type); + await addUser(repoId, username, type); handleSuccess(); handleClose(); } catch (e) { @@ -145,7 +145,7 @@ AddUserDialog.propTypes = { export default function AddUser(props) { const [open, setOpen] = React.useState(false); - const repoName = props.repoName; + const repoId = props.repoId; const type = props.type; const refreshFn = props.refreshFn; @@ -163,7 +163,7 @@ export default function AddUser(props) { { - getRepo(setIsLoading, setData, setAuth, setIsError, repoName); + getRepo(setIsLoading, setData, setAuth, setIsError, repoId); }, []); const removeUser = async (userToRemove, action) => { - await deleteUser(userToRemove, repoName, action); - getRepo(setIsLoading, setData, setAuth, setIsError, repoName); + await deleteUser(userToRemove, repoId, action); + getRepo(setIsLoading, setData, setAuth, setIsError, repoId); }; - const removeRepository = async (name) => { - await deleteRepo(name); + const removeRepository = async (id) => { + await deleteRepo(id); navigate('/dashboard/repo', { replace: true }); }; - const refresh = () => getRepo(setIsLoading, setData, setAuth, setIsError, repoName); + const refresh = () => getRepo(setIsLoading, setData, setAuth, setIsError, repoId); if (isLoading) return
Loading...
; if (isError) return
Something went wrong ...
; - const { project: org, name, proxyURL } = data || {}; - const cloneURL = `${proxyURL}/${org}/${name}.git`; + const { url: remoteUrl, proxyURL } = data || {}; + const parsedUrl = new URL(remoteUrl); + const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; return ( @@ -72,7 +73,7 @@ export default function RepoDetails() { @@ -134,7 +135,7 @@ export default function RepoDetails() { {user.admin && (
- +
)} @@ -179,7 +180,7 @@ export default function RepoDetails() { {user.admin && (
- +
)} diff --git a/src/ui/views/RepoList/Components/NewRepo.jsx b/src/ui/views/RepoList/Components/NewRepo.jsx index e1c912069..b5e93f1c4 100644 --- a/src/ui/views/RepoList/Components/NewRepo.jsx +++ b/src/ui/views/RepoList/Components/NewRepo.jsx @@ -53,8 +53,8 @@ function AddRepositoryDialog(props) { maxUser: 1, }; - if (data.project.trim().length == 0 || data.project.length > 100) { - setError('project name length unexpected'); + if (data.project.length > 100) { + setError('organisation name is too long'); return; } @@ -64,7 +64,11 @@ function AddRepositoryDialog(props) { } try { - new URL(data.url); + const parsedUrl = new URL(data.url); + if (!parsedUrl.pathname.endsWith('.git')) { + setError('Invalid git URL - Git URLs should end with .git'); + return; + } } catch { setError('Invalid URL'); return; @@ -73,6 +77,7 @@ function AddRepositoryDialog(props) { try { await addRepo(onClose, setError, data); handleSuccess(data); + handleClose(); } catch (e) { if (e.message) { @@ -124,7 +129,7 @@ function AddRepositoryDialog(props) { aria-describedby='project-helper-text' onChange={(e) => setProject(e.target.value)} /> - GitHub Organization + Organization or path @@ -136,7 +141,7 @@ function AddRepositoryDialog(props) { aria-describedby='name-helper-text' onChange={(e) => setName(e.target.value)} /> - GitHub Repository Name + Git Repository Name @@ -149,7 +154,7 @@ function AddRepositoryDialog(props) { aria-describedby='url-helper-text' onChange={(e) => setUrl(e.target.value)} /> - GitHub Repository URL + Git Repository URL diff --git a/src/ui/views/RepoList/Components/RepoOverview.jsx b/src/ui/views/RepoList/Components/RepoOverview.jsx index 826f78c97..a808b4054 100644 --- a/src/ui/views/RepoList/Components/RepoOverview.jsx +++ b/src/ui/views/RepoList/Components/RepoOverview.jsx @@ -578,6 +578,7 @@ export default function Repositories(props) { getGitHubRepository(); }, [props.data.project, props.data.name]); + // TODO add support for GitLab API: https://docs.gitlab.com/api/projects/#get-a-single-project const getGitHubRepository = async () => { await axios .get(`https://api.github.com/repos/${props.data.project}/${props.data.name}`) @@ -585,13 +586,16 @@ export default function Repositories(props) { setGitHub(res.data); }) .catch((error) => { - setErrorMessage(`Error fetching GitHub repository ${props.data.project}/${props.data.name}: ${error}`); + setErrorMessage( + `Error fetching GitHub repository ${props.data.project}/${props.data.name}: ${error}`, + ); setSnackbarOpen(true); }); }; - const { project: org, name, proxyURL } = props?.data || {}; - const cloneURL = `${proxyURL}/${org}/${name}.git`; + const { url: remoteUrl, proxyURL } = props?.data || {}; + const parsedUrl = new URL(remoteUrl); + const cloneURL = `${proxyURL}/${parsedUrl.host}${parsedUrl.port ? `:${parsedUrl.port}` : ''}${parsedUrl.pathname}`; return ( diff --git a/src/ui/views/RepoList/Components/Repositories.jsx b/src/ui/views/RepoList/Components/Repositories.jsx index ac9663423..248a27ae5 100644 --- a/src/ui/views/RepoList/Components/Repositories.jsx +++ b/src/ui/views/RepoList/Components/Repositories.jsx @@ -14,8 +14,7 @@ import { UserContext } from '../../../../context'; import PropTypes from 'prop-types'; import Search from '../../../components/Search/Search'; import Pagination from '../../../components/Pagination/Pagination'; -import Filtering from '../../../components/Filtering/Filtering'; - +import Filtering from '../../../components/Filtering/Filtering'; export default function Repositories(props) { const useStyles = makeStyles(styles); @@ -26,7 +25,7 @@ export default function Repositories(props) { const [isLoading, setIsLoading] = useState(false); const [isError, setIsError] = useState(false); const [currentPage, setCurrentPage] = useState(1); - const itemsPerPage = 5; + const itemsPerPage = 5; const navigate = useNavigate(); const { user } = useContext(UserContext); const openRepo = (repo) => navigate(`/dashboard/repo/${repo}`, { replace: true }); @@ -37,10 +36,16 @@ export default function Repositories(props) { if (!k) continue; query[k] = props[k]; } - getRepos(setIsLoading, (data) => { - setData(data); - setFilteredData(data); - }, setAuth, setIsError, query); + getRepos( + setIsLoading, + (data) => { + setData(data); + setFilteredData(data); + }, + setAuth, + setIsError, + query, + ); }, [props]); const refresh = async (repo) => { @@ -50,16 +55,17 @@ export default function Repositories(props) { }; const handleSearch = (query) => { - setCurrentPage(1); + setCurrentPage(1); if (!query) { setFilteredData(data); } else { const lowercasedQuery = query.toLowerCase(); setFilteredData( - data.filter(repo => - repo.name.toLowerCase().includes(lowercasedQuery) || - repo.project.toLowerCase().includes(lowercasedQuery) - ) + data.filter( + (repo) => + repo.name.toLowerCase().includes(lowercasedQuery) || + repo.project.toLowerCase().includes(lowercasedQuery), + ), ); } }; @@ -88,8 +94,7 @@ export default function Repositories(props) { setFilteredData(sortedData); }; - - const handlePageChange = (page) => setCurrentPage(page); + const handlePageChange = (page) => setCurrentPage(page); const startIdx = (currentPage - 1) * itemsPerPage; const paginatedData = filteredData.slice(startIdx, startIdx + itemsPerPage); @@ -109,14 +114,14 @@ export default function Repositories(props) { key='x' classes={classes} openRepo={openRepo} - data={paginatedData} + data={paginatedData} repoButton={addrepoButton} - onSearch={handleSearch} - currentPage={currentPage} - totalItems={filteredData.length} - itemsPerPage={itemsPerPage} - onPageChange={handlePageChange} - onFilterChange={handleFilterChange} // Pass handleFilterChange as prop + onSearch={handleSearch} + currentPage={currentPage} + totalItems={filteredData.length} + itemsPerPage={itemsPerPage} + onPageChange={handlePageChange} + onFilterChange={handleFilterChange} // Pass handleFilterChange as prop /> ); } @@ -138,16 +143,15 @@ function GetGridContainerLayOut(props) { {props.repoButton} - - {/* Include the Filtering component */} + {/* Include the Filtering component */} {props.data.map((row) => { - if (row.project && row.name) { + if (row.url) { return ; } })} @@ -166,4 +170,3 @@ function GetGridContainerLayOut(props) { ); } - diff --git a/src/ui/views/UserList/Components/UserList.jsx b/src/ui/views/UserList/Components/UserList.jsx index ee6812485..36aef89e6 100644 --- a/src/ui/views/UserList/Components/UserList.jsx +++ b/src/ui/views/UserList/Components/UserList.jsx @@ -20,7 +20,6 @@ import Search from '../../../components/Search/Search'; const useStyles = makeStyles(styles); export default function UserList(props) { - const classes = useStyles(); const [data, setData] = useState([]); const [, setAuth] = useState(true); @@ -28,12 +27,11 @@ export default function UserList(props) { const [isError, setIsError] = useState(false); const navigate = useNavigate(); const [currentPage, setCurrentPage] = useState(1); - const itemsPerPage = 5; + const itemsPerPage = 5; const [searchQuery, setSearchQuery] = useState(''); const openUser = (username) => navigate(`/dashboard/admin/user/${username}`, { replace: true }); - useEffect(() => { const query = {}; @@ -47,32 +45,30 @@ export default function UserList(props) { if (isLoading) return
Loading...
; if (isError) return
Something went wrong...
; - - const filteredUsers = data.filter(user => - user.displayName && user.displayName.toLowerCase().includes(searchQuery.toLowerCase()) || - user.username && user.username.toLowerCase().includes(searchQuery.toLowerCase()) -); + const filteredUsers = data.filter( + (user) => + (user.displayName && user.displayName.toLowerCase().includes(searchQuery.toLowerCase())) || + (user.username && user.username.toLowerCase().includes(searchQuery.toLowerCase())), + ); const indexOfLastItem = currentPage * itemsPerPage; const indexOfFirstItem = indexOfLastItem - itemsPerPage; const currentItems = filteredUsers.slice(indexOfFirstItem, indexOfLastItem); const totalItems = filteredUsers.length; - const handlePageChange = (page) => { setCurrentPage(page); }; - const handleSearch = (query) => { setSearchQuery(query); - setCurrentPage(1); + setCurrentPage(1); }; return ( - +
@@ -94,12 +90,20 @@ export default function UserList(props) { {row.email} - + {row.gitAccount} - {row.admin ? : } + {row.admin ? ( + + ) : ( + + )}