From 1605d74c1d7ba98c958f996da1d95a345e7aeb67 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 18:31:06 +0100 Subject: [PATCH 01/11] Add support for server connecting and id assign --- src/drivers/example.driver.ts | 1 - src/server/websocket/defs.ts | 2 +- src/server/websocket/handlers.ts | 33 ++++++++++++++++++-------------- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/drivers/example.driver.ts b/src/drivers/example.driver.ts index 0777dd5..e055dbb 100644 --- a/src/drivers/example.driver.ts +++ b/src/drivers/example.driver.ts @@ -4,7 +4,6 @@ import Portal from '../models/portal' import { createClient } from '../config/providers/example.config' import { closePortal } from './portal.driver' - export const openPortalInstance = async (portal: Portal) => { const client = createClient(), name = `portal-${portal.id}` diff --git a/src/server/websocket/defs.ts b/src/server/websocket/defs.ts index 1382a9c..d3e6e55 100644 --- a/src/server/websocket/defs.ts +++ b/src/server/websocket/defs.ts @@ -1,4 +1,4 @@ -export type ClientType = 'portal' +export type ClientType = 'server' export default interface WSEvent { op: number diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 52d2f87..545566f 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -4,8 +4,9 @@ import { verify } from 'jsonwebtoken' import Portal from '../../models/portal' import WSEvent, { ClientType } from './defs' +import { generateFlake } from '../../utils/generate.utils' -const ACCEPTABLE_CLIENT_TYPES: ClientType[] = ['portal'], +const ACCEPTABLE_CLIENT_TYPES: ClientType[] = ['server'], isClientWithIdAndType = (id: string, type: ClientType) => (client: WebSocket) => client['id'] === id && client['type'] === type /** @@ -16,39 +17,43 @@ const handleMessage = async (message: WSEvent, socket: WebSocket) => { clientId = socket['id'], clientType = socket['type'] - console.log(`recieved message from ${clientType} (${clientId || 'unknown'}) over ws`, op, t) + console.log(`recieved ${JSON.stringify(d)} from ${clientType || 'unknown'} (${clientId || 'unknown'}) over ws`, op, t) if(op === 2) { try { - const { token, type } = d, { id } = verify(token, process.env.PORTAL_KEY) as { id: string } + const { token, type } = d, + payload = verify(token, process.env.PORTAL_KEY) + + if(!payload) return socket.close(1013) if(ACCEPTABLE_CLIENT_TYPES.indexOf(type) === -1) return socket.close(1013) - socket['id'] = id socket['type'] = type - if(type === 'portal') { - const portal = await new Portal().load(id) - await portal.updateStatus('open') - } + if(type === 'server') { + const id = generateFlake() + socket.send(JSON.stringify({ op: 10, d: { id } })) - console.log('recieved auth from', type, id) + socket['id'] = id + console.log('recieved auth from', type, id) + } else return socket.close(1013) } catch(error) { socket.close(1013) console.error('authentication error', error) } } } + export default handleMessage /** * Message incoming from API or Portals for Portal */ export const routeMessage = async (message: WSEvent, clients: WebSocket[]) => { - const { op, d, t } = message, { t: targetId } = d - console.log('recieved internal portal message to be routed to portal with id', targetId, JSON.stringify(message)) + // const { op, d, t } = message, { t: targetId } = d + // console.log('recieved internal portal message to be routed to portal with id', targetId, JSON.stringify(message)) - const target = clients.find(isClientWithIdAndType(targetId, 'portal')) - if(!target) return console.log('target not found for internal message to portal; aborting') + // const target = clients.find(isClientWithIdAndType(targetId, 'portal')) + // if(!target) return console.log('target not found for internal message to portal; aborting') - target.send(JSON.stringify({ op, d: { ...d, t: undefined }, t })) + // target.send(JSON.stringify({ op, d: { ...d, t: undefined }, t })) } From 62798badd583ce39e5bd1a759540961b216856e4 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 18:34:04 +0100 Subject: [PATCH 02/11] Add support for storing connected server ids in Redis --- src/server/websocket/handlers.ts | 4 +++- src/server/websocket/index.ts | 11 +++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 545566f..124313c 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -1,7 +1,7 @@ import WebSocket from 'ws' import { verify } from 'jsonwebtoken' -import Portal from '../../models/portal' +import client from '../../config/redis.config' import WSEvent, { ClientType } from './defs' import { generateFlake } from '../../utils/generate.utils' @@ -33,6 +33,8 @@ const handleMessage = async (message: WSEvent, socket: WebSocket) => { const id = generateFlake() socket.send(JSON.stringify({ op: 10, d: { id } })) + client.lpush('servers', id) + socket['id'] = id console.log('recieved auth from', type, id) } else return socket.close(1013) diff --git a/src/server/websocket/index.ts b/src/server/websocket/index.ts index d63d785..a919d02 100644 --- a/src/server/websocket/index.ts +++ b/src/server/websocket/index.ts @@ -1,9 +1,7 @@ import { Server } from 'ws' -import Portal from '../../models/portal' - import WSEvent from './defs' -import { createPubSubClient } from '../../config/redis.config' +import client, { createPubSubClient } from '../../config/redis.config' import handleMessage, { routeMessage } from './handlers' const sub = createPubSubClient() @@ -40,16 +38,13 @@ export default (wss: Server) => { }) socket.on('close', async () => { - const id = socket['id'], type = socket['type'] if(!id) return console.log('unknown socket closed') console.log('socket closed', id, type) - if(type === 'portal') { - const portal = await new Portal().load(id) - portal.updateStatus('closed') - } + if(type === 'server') + client.lrem('servers', 1, id) }) }) } From e4a8a6645f91e0f65bf160ce2fa1aeaa64469bf3 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 19:14:11 +0100 Subject: [PATCH 03/11] Add server model and support for reconnecting servers --- src/drivers/example.driver.ts | 12 ++--- src/drivers/portal.driver.ts | 2 +- src/models/portal/defs.ts | 5 +- src/models/portal/index.ts | 39 ++++----------- src/models/server/defs.ts | 12 +++++ src/models/server/index.ts | 85 ++++++++++++++++++++++++++++++++ src/schemas/portal.schema.ts | 4 +- src/schemas/server.schema.ts | 15 ++++++ src/server/websocket/handlers.ts | 17 ++++--- src/server/websocket/index.ts | 14 ++++-- src/utils/helpers.utils.ts | 3 ++ tsconfig.json | 3 +- 12 files changed, 156 insertions(+), 55 deletions(-) create mode 100644 src/models/server/defs.ts create mode 100644 src/models/server/index.ts create mode 100644 src/schemas/server.schema.ts create mode 100644 src/utils/helpers.utils.ts diff --git a/src/drivers/example.driver.ts b/src/drivers/example.driver.ts index e055dbb..1406c4c 100644 --- a/src/drivers/example.driver.ts +++ b/src/drivers/example.driver.ts @@ -10,8 +10,8 @@ export const openPortalInstance = async (portal: Portal) => { try { // Create the server using the API & Provider of your choice - await client.createServer() - await portal.updateStatus('starting') + // await client.createServer() + // await portal.updateStatus('starting') console.log(`opened portal with name ${name}`) } catch(error) { @@ -22,15 +22,13 @@ export const openPortalInstance = async (portal: Portal) => { } export const closePortalInstance = async (portal: Portal) => { - const client = createClient(), - name = `portal-${portal.id}`, - { serverId } = portal + const client = createClient() try { // Destroy the server using the id of the server you stored when creating the server - await client.destroyServer() + // await client.destroyServer() - console.log(`closed portal with name ${name}`) + // console.log(`closed portal with name ${name}`) } catch(error) { console.error('error while closing portal', error.response ? error.response.body : error) } diff --git a/src/drivers/portal.driver.ts b/src/drivers/portal.driver.ts index 3bf80bc..a9038b1 100644 --- a/src/drivers/portal.driver.ts +++ b/src/drivers/portal.driver.ts @@ -22,7 +22,7 @@ export const closePortal = (portalId: string) => new Promise(async (resolve, rej closePortalInstance(portal) - if(portal.status === 'open') + if(portal.status === 'connected') checkNextQueueItem() console.log('closing portal with status', portal.status) diff --git a/src/models/portal/defs.ts b/src/models/portal/defs.ts index 68ff85c..e361e19 100644 --- a/src/models/portal/defs.ts +++ b/src/models/portal/defs.ts @@ -9,11 +9,10 @@ export default interface IPortal { recievedAt: number room: string + server?: string + status: PortalStatus } - data: { - serverId?: string - } } export interface IStoredPortal extends IPortal, Document {} diff --git a/src/models/portal/index.ts b/src/models/portal/index.ts index 5e8f7d1..663091b 100644 --- a/src/models/portal/index.ts +++ b/src/models/portal/index.ts @@ -11,18 +11,18 @@ import { createPubSubClient } from '../../config/redis.config' const pub = createPubSubClient() -export type PortalStatus = 'waiting' | 'requested' | 'in-queue' | 'creating' | 'starting' | 'open' | 'closed' | 'error' +export type PortalStatus = 'connected' | 'starting' | 'in-queue' | 'waiting' | 'closed' | 'error' +export type PortalResolvable = Portal | string export default class Portal { id: string createdAt: number recievedAt: number - serverId: string - - status: PortalStatus - room: string + server?: string + + status: PortalStatus load = (id: string) => new Promise(async (resolve, reject) => { try { @@ -48,9 +48,8 @@ export default class Portal { recievedAt, room: roomId, - status: 'creating' - }, - data: {} + status: 'starting' + } } const stored = new StoredPortal(json) @@ -115,24 +114,6 @@ export default class Portal { } }) - updateServerId = (serverId: string) => new Promise(async (resolve, reject) => { - try { - await StoredPortal.updateOne({ - 'info.id': this.id - }, { - $set: { - 'data.serverId': serverId - } - }) - - this.serverId = serverId - - resolve(this) - } catch(error) { - reject(error) - } - }) - setup = (json: IPortal) => { this.id = json.info.id this.createdAt = json.info.createdAt @@ -140,9 +121,7 @@ export default class Portal { this.room = json.info.room this.status = json.info.status - - if(json.data) - if(json.data.serverId) - this.serverId = json.data.serverId + + this.server = json.info.server } } diff --git a/src/models/server/defs.ts b/src/models/server/defs.ts new file mode 100644 index 0000000..7569adb --- /dev/null +++ b/src/models/server/defs.ts @@ -0,0 +1,12 @@ +import { Document } from 'mongoose' + +export default interface IServer { + info: { + id: string + connectedAt: number + + portal?: string + } +} + +export interface IStoredServer extends IServer, Document {} diff --git a/src/models/server/index.ts b/src/models/server/index.ts new file mode 100644 index 0000000..0290fe2 --- /dev/null +++ b/src/models/server/index.ts @@ -0,0 +1,85 @@ +import Portal, { PortalResolvable } from '../portal' + +import IServer from './defs' +import StoredServer from '../../schemas/server.schema' + +import client from '../../config/redis.config' +import { generateFlake } from '../../utils/generate.utils' +import { extractPortalId } from '../../utils/helpers.utils' + +export default class Server { + id: string + connectedAt: number + + portal: PortalResolvable + + constructor(json?: IServer) { + if(!json) return + + this.setup(json) + } + + load = (id: string) => new Promise(async (resolve, reject) => { + try { + const doc = await StoredServer.findOne({ 'info.id': id }) + if(!doc) return reject('ServerNotFound') + + this.setup(doc) + + resolve(this) + } catch(error) { + reject(error) + } + }) + + create = () => new Promise(async (resolve, reject) => { + try { + const id = generateFlake(), + json: IServer = { + info: { + id, + connectedAt: Date.now() + } + } + + const stored = new StoredServer(json) + await stored.save() + + client.sadd('servers', id) + + this.setup(json) + + resolve(this) + } catch(error) { + reject(error) + } + }) + + destroy = () => new Promise(async (resolve, reject) => { + try { + await StoredServer.deleteOne({ + 'info.id': this.id + }) + + client.srem('servers', this.id) + + if(this.portal) { + const portalId = extractPortalId(this.portal), + portal = await new Portal().load(portalId) + + portal.updateStatus('closed') + } + + resolve() + } catch(error) { + reject(error) + } + }) + + setup = (json: IServer) => { + this.id = json.info.id + this.connectedAt = json.info.connectedAt + + this.portal = json.info.portal + } +} diff --git a/src/schemas/portal.schema.ts b/src/schemas/portal.schema.ts index e1f4d8f..8a93150 100644 --- a/src/schemas/portal.schema.ts +++ b/src/schemas/portal.schema.ts @@ -2,7 +2,7 @@ import { Schema, model } from 'mongoose' import { IStoredPortal } from '../models/portal/defs' -const ModelSchema = new Schema({ +const PortalSchema = new Schema({ info: { id: String, createdAt: Number, @@ -16,5 +16,5 @@ const ModelSchema = new Schema({ } }) -const StoredPortal = model('Portal', ModelSchema) +const StoredPortal = model('Portal', PortalSchema) export default StoredPortal diff --git a/src/schemas/server.schema.ts b/src/schemas/server.schema.ts new file mode 100644 index 0000000..7e6688a --- /dev/null +++ b/src/schemas/server.schema.ts @@ -0,0 +1,15 @@ +import { Schema, model } from 'mongoose' + +import { IStoredServer } from '../models/server/defs' + +const ServerSchema = new Schema({ + info: { + id: String, + connectedAt: Number, + + portal: String + } +}) + +const StoredServer = model('Server', ServerSchema) +export default StoredServer diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 124313c..8103128 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -5,6 +5,7 @@ import client from '../../config/redis.config' import WSEvent, { ClientType } from './defs' import { generateFlake } from '../../utils/generate.utils' +import Server from '../../models/server' const ACCEPTABLE_CLIENT_TYPES: ClientType[] = ['server'], isClientWithIdAndType = (id: string, type: ClientType) => (client: WebSocket) => client['id'] === id && client['type'] === type @@ -22,7 +23,7 @@ const handleMessage = async (message: WSEvent, socket: WebSocket) => { if(op === 2) { try { const { token, type } = d, - payload = verify(token, process.env.PORTAL_KEY) + payload = verify(token, process.env.PORTAL_KEY) as { id?: string } if(!payload) return socket.close(1013) if(ACCEPTABLE_CLIENT_TYPES.indexOf(type) === -1) return socket.close(1013) @@ -30,13 +31,17 @@ const handleMessage = async (message: WSEvent, socket: WebSocket) => { socket['type'] = type if(type === 'server') { - const id = generateFlake() - socket.send(JSON.stringify({ op: 10, d: { id } })) + let server: Server - client.lpush('servers', id) + if(payload.id && await client.sismember('servers', payload.id) === 1) + server = await new Server().load(payload.id) + else + server = await new Server().create() - socket['id'] = id - console.log('recieved auth from', type, id) + socket['id'] = server.id + socket.send(JSON.stringify({ op: 10, d: { id: server.id } })) + + console.log('recieved auth from', type, server.id) } else return socket.close(1013) } catch(error) { socket.close(1013) diff --git a/src/server/websocket/index.ts b/src/server/websocket/index.ts index a919d02..f446341 100644 --- a/src/server/websocket/index.ts +++ b/src/server/websocket/index.ts @@ -1,12 +1,14 @@ -import { Server } from 'ws' +import { Server as WSS } from 'ws' + +import Server from '../../models/server' import WSEvent from './defs' -import client, { createPubSubClient } from '../../config/redis.config' +import { createPubSubClient } from '../../config/redis.config' import handleMessage, { routeMessage } from './handlers' const sub = createPubSubClient() -export default (wss: Server) => { +export default (wss: WSS) => { sub.on('message', (channel, data) => { console.log('recieved message on channel', channel, 'data', data) @@ -43,8 +45,10 @@ export default (wss: Server) => { console.log('socket closed', id, type) - if(type === 'server') - client.lrem('servers', 1, id) + if(type === 'server') { + const server = await new Server().load(id) + server.destroy() + } }) }) } diff --git a/src/utils/helpers.utils.ts b/src/utils/helpers.utils.ts new file mode 100644 index 0000000..b276ecc --- /dev/null +++ b/src/utils/helpers.utils.ts @@ -0,0 +1,3 @@ +import { PortalResolvable } from '../models/portal' + +export const extractPortalId = (portal: PortalResolvable) => portal ? (typeof portal === 'string' ? portal : portal.id) : null \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 7f2903c..59e2d6f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,8 @@ "allowSyntheticDefaultImports": true, "outDir": "./dist", "allowJs": true, - "resolveJsonModule": true + "resolveJsonModule": true, + "newLine": "LF" }, "include": ["./src"] } From c58408bb9be62d160420a072820ca1fccf9bc9f2 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 20:24:20 +0100 Subject: [PATCH 04/11] Add support for assigning/unassigning portals to/from servers --- src/controllers/index.controller.ts | 31 ++++++--- src/drivers/portal.driver.ts | 5 +- src/models/portal/index.ts | 30 +++++++-- src/models/server/index.ts | 89 +++++++++++++++++++++++-- src/schemas/portal.schema.ts | 2 + src/services/queue.service.ts | 100 ---------------------------- 6 files changed, 132 insertions(+), 125 deletions(-) delete mode 100644 src/services/queue.service.ts diff --git a/src/controllers/index.controller.ts b/src/controllers/index.controller.ts index f6e11ad..fd29781 100644 --- a/src/controllers/index.controller.ts +++ b/src/controllers/index.controller.ts @@ -1,25 +1,38 @@ import express from 'express' +import Portal from '../models/portal' import PortalRequest from '../models/request/defs' -import { pushQueue } from '../services/queue.service' -import { closePortal } from '../drivers/portal.driver' import authenticate from '../server/middleware/authenticate.middleware' const app = express() -app.post('/create', authenticate, (req, res) => { +app.post('/create', authenticate, async (req, res) => { const { roomId } = req.body, request: PortalRequest = { roomId, recievedAt: Date.now() } - pushQueue(request) - - res.send(request) + + try { + const portal = await new Portal().create(request) + + res.send(portal) + } catch(error) { + console.error(error) + res.sendStatus(500) + } }) -app.delete('/:id', authenticate, (req, res) => { +app.delete('/:id', authenticate, async (req, res) => { const { id: portalId } = req.params - closePortal(portalId) + console.log('recieved request to close portal', portalId) + + try { + const portal = await new Portal().load(portalId) + await portal.destroy() - res.sendStatus(200) + res.sendStatus(200) + } catch(error) { + console.error(error) + res.sendStatus(500) + } }) export default app diff --git a/src/drivers/portal.driver.ts b/src/drivers/portal.driver.ts index a9038b1..f089424 100644 --- a/src/drivers/portal.driver.ts +++ b/src/drivers/portal.driver.ts @@ -1,7 +1,6 @@ import Portal from '../models/portal' import PortalRequest from '../models/request/defs' -import { checkNextQueueItem } from '../services/queue.service' import { openPortalInstance, closePortalInstance } from './router' export const createPortal = (request: PortalRequest) => new Promise(async (resolve, reject) => { @@ -22,8 +21,8 @@ export const closePortal = (portalId: string) => new Promise(async (resolve, rej closePortalInstance(portal) - if(portal.status === 'connected') - checkNextQueueItem() + // if(portal.status === 'connected') + // checkNextQueueItem() console.log('closing portal with status', portal.status) diff --git a/src/models/portal/index.ts b/src/models/portal/index.ts index 663091b..2fc0c87 100644 --- a/src/models/portal/index.ts +++ b/src/models/portal/index.ts @@ -1,6 +1,8 @@ import axios from 'axios' import { sign } from 'jsonwebtoken' +import Server from '../server' + import PortalRequest from '../request/defs' import IPortal from './defs' @@ -8,10 +10,11 @@ import StoredPortal from '../../schemas/portal.schema' import { generateFlake } from '../../utils/generate.utils' import { createPubSubClient } from '../../config/redis.config' +import StoredServer from '../../schemas/server.schema' const pub = createPubSubClient() -export type PortalStatus = 'connected' | 'starting' | 'in-queue' | 'waiting' | 'closed' | 'error' +export type PortalStatus = 'open' | 'starting' | 'in-queue' | 'closed' | 'error' export type PortalResolvable = Portal | string export default class Portal { @@ -24,6 +27,12 @@ export default class Portal { status: PortalStatus + constructor(json?: IPortal) { + if(!json) return + + this.setup(json) + } + load = (id: string) => new Promise(async (resolve, reject) => { try { const doc = await StoredPortal.findOne({ 'info.id': id }) @@ -48,7 +57,7 @@ export default class Portal { recievedAt, room: roomId, - status: 'starting' + status: 'in-queue' } } @@ -56,15 +65,22 @@ export default class Portal { await stored.save() this.setup(json) + + const serverDoc = await StoredServer.findOne({ 'info.portal': { $exists: false } }) + if(serverDoc) { + const server = new Server(serverDoc) + console.log('Assigning portal to server', server.id) + await server.assign(this) + } else console.log('Could not assign portal to server') /** * Inform API of new portal with room id */ - await axios.post(`${process.env.API_URL}/internal/portal`, { id: this.id, roomId }, { - headers: { - authorization: `Valve ${sign({}, process.env.API_KEY)}` - } - }) + // await axios.post(`${process.env.API_URL}/internal/portal`, { id: this.id, roomId }, { + // headers: { + // authorization: `Valve ${sign({}, process.env.API_KEY)}` + // } + // }) resolve(this) } catch(error) { diff --git a/src/models/server/index.ts b/src/models/server/index.ts index 0290fe2..101a6a7 100644 --- a/src/models/server/index.ts +++ b/src/models/server/index.ts @@ -3,6 +3,8 @@ import Portal, { PortalResolvable } from '../portal' import IServer from './defs' import StoredServer from '../../schemas/server.schema' +import StoredPortal from '../../schemas/portal.schema' + import client from '../../config/redis.config' import { generateFlake } from '../../utils/generate.utils' import { extractPortalId } from '../../utils/helpers.utils' @@ -49,6 +51,12 @@ export default class Server { this.setup(json) + const portalDoc = await StoredPortal.findOne({ 'info.server': { $exists: false } }) + if(portalDoc) { + const portal = new Portal(portalDoc) + this.assign(portal) + } + resolve(this) } catch(error) { reject(error) @@ -63,12 +71,8 @@ export default class Server { client.srem('servers', this.id) - if(this.portal) { - const portalId = extractPortalId(this.portal), - portal = await new Portal().load(portalId) - - portal.updateStatus('closed') - } + if(this.portal) + await this.unassign() resolve() } catch(error) { @@ -76,6 +80,79 @@ export default class Server { } }) + assign = (portal: PortalResolvable) => new Promise(async (resolve, reject) => { + if(this.portal) return reject('PortalAlreadyAssigned') + + const portalId = extractPortalId(portal) + + try { + + await StoredServer.updateOne({ + 'info.id': this.id + }, { + $set: { + 'info.portal': extractPortalId(portal) + } + }) + + await StoredPortal.updateOne({ + 'info.id': portalId + }, { + $set: { + 'info.server': this.id + } + }) + + if(typeof portal !== 'string') + portal.updateStatus('open') + + this.portal = portal + + resolve(this) + } catch(error) { + reject(error) + } + }) + + unassign = () => new Promise(async (resolve, reject) => { + if(!this.portal) return reject('NoPortalAssigned') + + const portalId = extractPortalId(this.portal) + + try { + await StoredServer.updateOne({ + 'info.id': this.id + }, { + $unset: { + 'info.portal': '' + } + }) + + await StoredPortal.updateOne({ + 'info.id': portalId + }, { + $unset: { + 'info.server': '' + } + }) + + let portal: Portal + if(typeof this.portal === 'string') + portal = await new Portal().load(portalId) + else + portal = this.portal + + if(portal) + portal.updateStatus('in-queue') + + delete this.portal + + resolve(this) + } catch(error) { + reject(error) + } + }) + setup = (json: IServer) => { this.id = json.info.id this.connectedAt = json.info.connectedAt diff --git a/src/schemas/portal.schema.ts b/src/schemas/portal.schema.ts index 8a93150..94d06aa 100644 --- a/src/schemas/portal.schema.ts +++ b/src/schemas/portal.schema.ts @@ -9,6 +9,8 @@ const PortalSchema = new Schema({ recievedAt: Number, room: String, + server: String, + status: String }, data: { diff --git a/src/services/queue.service.ts b/src/services/queue.service.ts deleted file mode 100644 index d4e622e..0000000 --- a/src/services/queue.service.ts +++ /dev/null @@ -1,100 +0,0 @@ -import client from '../config/redis.config' - -import PortalRequest from '../models/request/defs' - -import { fetchCurrentDriver } from '../drivers/router' -import { createPortal } from '../drivers/portal.driver' -import { fetchAvailableNode } from '../drivers/kubernetes.driver' - -/** - * This method is responsible for fetching the queue length. - */ -const fetchQueueLength = () => client.llen('portal_queue') - -/** - * This method is responsible for fetching the next queue item that needs to be - * handled. - */ -const fetchNextQueueItem = () => new Promise(async (resolve, reject) => { - try { - const queueItem: PortalRequest = await client.lrange('portal_queue', 0, 1)[0] - if(!queueItem) return resolve() - - resolve(queueItem) - } catch(error) { - reject(error) - } -}) - -/** - * This method is responsible for pulling the first queue item, and then - * returning the new length of the queue. - * - * It will first get the value of the item at the index given in the call, - * if there is no value then it will fetch the queue length and then return. - * - * If there is a value at that index, then it will remove the value at that index - * in the queue, and then return the length of the queue. - */ -const pullQueueItem = async (index: number = 0) => new Promise(async (resolve, reject) => { - try { - const value = await client.lindex('portal_queue', index) - if(!value) return resolve(await fetchQueueLength()) - - const length = await client.lrem('portal_queue', index, value) - resolve(length) - } catch(error) { - reject(error) - } -}) - -/** - * This method is called when a request is completed. It first checks if - * there are any available Kubernetes nodes for a new Portal to be deployed on. - * If not, the function does not complete. It checks if there is any items in the queue. - * If not, the function does not complete. If there is another queue item, it'll fetch - * the queue item and them call the method responsible for handling that queue item. - */ -export const checkNextQueueItem = async () => { - /** - * If there are no available nodes, don't resume with creating a new queue item - * This only applies if we're using the Kubernetes driver - * - * TODO: Move this to Kubernetes driver - */ - const driver = await fetchCurrentDriver() - if(driver == 'kubernetes' && !await fetchAvailableNode()) return - - const queueLength = await fetchQueueLength() - if(queueLength === 0) return - - handleQueueItem(await fetchNextQueueItem(), true) -} - -/** - * This method is called when a request needs to be handled. - * It is responsible for calling the service that creates a portal, - * and then calling the method that checks the next queue item - */ -const handleQueueItem = async (request: PortalRequest, didPullFromQueue: boolean) => { - if(didPullFromQueue) await pullQueueItem() - await createPortal(request) - - checkNextQueueItem() -} - -/** - * This method handles when a new request is recieved by the API. - * - * It checks if there are any existing queue items. If there are, - * then it pushes the new request to the queue. If not, then it - * will deal with the request immediately. - */ -export const pushQueue = async (request: PortalRequest) => { - const queueLength = await fetchQueueLength() - - if(queueLength === 0) - handleQueueItem(request, false) - else - client.lpush('portal_queue', JSON.stringify(request)) -} From 896df66503a0cf8ddcbac3f579c4c161ace8bc57 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 20:27:50 +0100 Subject: [PATCH 05/11] Add message routing support --- src/models/server/index.ts | 4 ++++ src/server/websocket/handlers.ts | 10 +++++----- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/models/server/index.ts b/src/models/server/index.ts index 101a6a7..1c2120e 100644 --- a/src/models/server/index.ts +++ b/src/models/server/index.ts @@ -106,6 +106,8 @@ export default class Server { if(typeof portal !== 'string') portal.updateStatus('open') + client.hset('portals', portalId, this.id) + this.portal = portal resolve(this) @@ -136,6 +138,8 @@ export default class Server { } }) + client.hdel('portals', portalId) + let portal: Portal if(typeof this.portal === 'string') portal = await new Portal().load(portalId) diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 8103128..55504b0 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -56,11 +56,11 @@ export default handleMessage * Message incoming from API or Portals for Portal */ export const routeMessage = async (message: WSEvent, clients: WebSocket[]) => { - // const { op, d, t } = message, { t: targetId } = d - // console.log('recieved internal portal message to be routed to portal with id', targetId, JSON.stringify(message)) + const { op, d, t } = message, { t: targetId } = d + console.log('recieved internal portal message to be routed to portal with id', targetId, JSON.stringify(message)) - // const target = clients.find(isClientWithIdAndType(targetId, 'portal')) - // if(!target) return console.log('target not found for internal message to portal; aborting') + const target = clients.find(isClientWithIdAndType(await client.hget('portals', targetId), 'server')) + if(!target) return console.log('target not found for internal message to portal; aborting') - // target.send(JSON.stringify({ op, d: { ...d, t: undefined }, t })) + target.send(JSON.stringify({ op, d: { ...d, t: undefined }, t })) } From 8a7a88016c0af5982a3056342021a354f10fa1c6 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 20:28:16 +0100 Subject: [PATCH 06/11] Remove route logging when not on development --- src/server/websocket/handlers.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 55504b0..80686e9 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -57,7 +57,9 @@ export default handleMessage */ export const routeMessage = async (message: WSEvent, clients: WebSocket[]) => { const { op, d, t } = message, { t: targetId } = d - console.log('recieved internal portal message to be routed to portal with id', targetId, JSON.stringify(message)) + + if(process.env.NODE_ENV === 'development') + console.log('recieved internal portal message to be routed to portal with id', targetId, JSON.stringify(message)) const target = clients.find(isClientWithIdAndType(await client.hget('portals', targetId), 'server')) if(!target) return console.log('target not found for internal message to portal; aborting') From 76001e4ee81e7496fed60c245c475b1c5d71c1e0 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 20:34:45 +0100 Subject: [PATCH 07/11] Import fixes --- .env.example | 3 +++ src/config/defaults.ts | 6 ++++++ src/server/websocket/handlers.ts | 7 +++---- 3 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 src/config/defaults.ts diff --git a/.env.example b/.env.example index dc44857..b3bfef0 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,9 @@ REDIS_URI= # Make sure to uncomment the below line and uncomment the REDIS_URI line above # REDIS_SENTINELS= +# Enable dynamic spawning of VM instances when none are available +ENABLE_DYNAMIC_VMS=false + # Optional: the JSON Google Application Credentials used for gcloud.driver # GOOGLE_APPLICATION_CREDENTIALS= diff --git a/src/config/defaults.ts b/src/config/defaults.ts new file mode 100644 index 0000000..a6133f9 --- /dev/null +++ b/src/config/defaults.ts @@ -0,0 +1,6 @@ +export default { + /** + * Boolean dependant on if dynamic VMs are enabled + */ + dynamic_vms_enabled: process.env.ENABLE_DYNAMIC_VMS === 'true' +} \ No newline at end of file diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 80686e9..0ff76f4 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -1,11 +1,10 @@ import WebSocket from 'ws' import { verify } from 'jsonwebtoken' -import client from '../../config/redis.config' - -import WSEvent, { ClientType } from './defs' -import { generateFlake } from '../../utils/generate.utils' import Server from '../../models/server' +import WSEvent, { ClientType } from './defs' + +import client from '../../config/redis.config' const ACCEPTABLE_CLIENT_TYPES: ClientType[] = ['server'], isClientWithIdAndType = (id: string, type: ClientType) => (client: WebSocket) => client['id'] === id && client['type'] === type From 36508cea5c4783cdb60eea21ec6c434f394f08ff Mon Sep 17 00:00:00 2001 From: William Gibson Date: Tue, 22 Oct 2019 21:06:52 +0100 Subject: [PATCH 08/11] Add early support for dynamic VMs --- README.md | 21 +++++++------- src/drivers/example.driver.ts | 21 +++++--------- src/drivers/gcloud.driver.ts | 29 ++++++------------- src/drivers/kubernetes.driver.ts | 20 +++++-------- src/drivers/manual.driver.ts | 48 -------------------------------- src/drivers/portal.driver.ts | 33 ---------------------- src/drivers/router.ts | 43 ++++++++++++---------------- src/models/portal/index.ts | 9 ++++-- src/schemas/portal.schema.ts | 3 -- 9 files changed, 58 insertions(+), 169 deletions(-) delete mode 100644 src/drivers/manual.driver.ts delete mode 100644 src/drivers/portal.driver.ts diff --git a/README.md b/README.md index 7f874f4..eeb6806 100644 --- a/README.md +++ b/README.md @@ -84,39 +84,39 @@ To run `@cryb/portals` in development mode, run `yarn dev`. It is recommended that in production you run `yarn build`, then `yarn start`. ### Adding a custom provider -`@cryb/portals` makes it easy to add a custom cloud provider to deploy Portal instances onto. +`@cryb/portals` makes it easy to add a custom cloud provider to deploy Server instances onto. 1. First, make a config file under `src/config/providers`. You want to call this `foo.config.ts`. This file should export a method that returns the API of the provider you want to use. See `example.config.ts` or `gcloud.config.ts` for an example of how Google Cloud intergration is handled. 2. Next, make a file under `src/drivers`. You want to call this `foo.driver.ts`. You can copy the code in `example.driver.ts` as a starting point. 3. Import your `foo.config.ts` file then create an instance of the client using the `createClient` method exported in the config file. -4. Then add the code to create a cloud deployment with the desired config under the `try {` clause in the `openPortalInstance` method. -5. *Optional, but recommended* Add the method under the `try {` clause in `closePortalInstance` to destroy the VM instance. This will be called when a Room no longer needs a portal, such as when all members have gone offline. +4. Then add the code to create a cloud deployment with the desired config under the `try {` clause in the `openServerInstance` method. +5. *Optional, but recommended* Add the method under the `try {` clause in `closeServerInstance` to destroy the VM instance. This will be called when the server is no longer needed, such as when there are no Rooms running. 6. Now, under `src/drivers/router.ts`, import your driver and rename its methods so they don't conflict when any other drivers. See below: ```ts import { - openPortalInstance as openFooPortalInstance, - closePortalInstance as closeFooPortalInstance + openServerInstance as openFooServerInstance, + closeServerInstance as closeFooServerInstance } from './foo.driver' ``` 7. *If you're not using TypeScript, skip this step* Make sure you have added the name of your driver to the `Driver` type. See below: ```ts -type Driver = 'gcloud' | 'kubernetes' | 'foo' +type Driver = 'gcloud' | 'kubernetes' | 'foo' | null ``` -8. Add a `case` to the `switch` statement under `openPortalInstance` with the name of your driver methods. See below: +8. Add a `case` to the `switch` statement under `openServerInstance` with the name of your driver methods. See below: ```ts switch(driver) { ... case 'foo': - openFooPortalInstance(portal) + openFooServerInstance(portal) break } ``` -9. *Optional, but recommended* If you added a `closePortalInstance` handler in your driver, add a `case` to the `switch` statement under `closePortalInstance` with the name of your driver methods. See below: +9. *Optional, but recommended* If you added a `closeServerInstance` handler in your driver, add a `case` to the `switch` statement under `closeServerInstance` with the name of your driver methods. See below: ```ts switch(driver) { ... case 'foo': - closeFooPortalInstance(portal) + closeFooServerInstance(portal) break } ``` @@ -128,5 +128,4 @@ const fetchCurrentDriver = () => 'foo' as Driver Done! Enjoy using `@cryb/portals` with the cloud provider of your preferred choice. For any help, view [here](#questions-/-issues). If you're feeling generous, create a [PR request](https://github.com/crybapp/portals) with your driver so the community can use it. Be sure to follow our [Guidelines](https://github.com/crybapp/guidelines) before submitting a PR. ## Questions / Issues - If you have an issues with `@cryb/portals`, please either open a GitHub issue, contact a maintainer or join the [Cryb Discord Server](https://discord.gg/ShTATH4) and ask in #tech-support. diff --git a/src/drivers/example.driver.ts b/src/drivers/example.driver.ts index 1406c4c..215bc19 100644 --- a/src/drivers/example.driver.ts +++ b/src/drivers/example.driver.ts @@ -1,34 +1,27 @@ -import Portal from '../models/portal' - // Import the API you wish to use import { createClient } from '../config/providers/example.config' -import { closePortal } from './portal.driver' -export const openPortalInstance = async (portal: Portal) => { - const client = createClient(), - name = `portal-${portal.id}` +export const openServerInstance = async () => { + const client = createClient() try { // Create the server using the API & Provider of your choice - // await client.createServer() - // await portal.updateStatus('starting') + await client.createServer() - console.log(`opened portal with name ${name}`) + console.log(`opened portal using example.driver`) } catch(error) { - closePortal(portal.id) - console.error('error while opening portal', error) } } -export const closePortalInstance = async (portal: Portal) => { +export const closeServerInstance = async () => { const client = createClient() try { // Destroy the server using the id of the server you stored when creating the server - // await client.destroyServer() + await client.destroyServer() - // console.log(`closed portal with name ${name}`) + console.log(`closed portal using example.driver`) } catch(error) { console.error('error while closing portal', error.response ? error.response.body : error) } diff --git a/src/drivers/gcloud.driver.ts b/src/drivers/gcloud.driver.ts index 9f8ada5..8146846 100644 --- a/src/drivers/gcloud.driver.ts +++ b/src/drivers/gcloud.driver.ts @@ -1,18 +1,13 @@ -import Portal from '../models/portal' - import { createClient, fetchCredentials } from '../config/providers/gcloud.config' -import { closePortal } from './portal.driver' const { project_id: projectId } = fetchCredentials() || { project_id: null }, zoneId = 'us-east1-b', baseUrl = `https://www.googleapis.com/compute/v1/projects/${projectId}/zones/${zoneId}/` -export const openPortalInstance = async (portal: Portal) => { +export const openServerInstance = async () => { const client = createClient() if(!client) throw 'The Google Cloud driver is incorrect. This may be due to improper ENV variables, please try again' - const portalName = `portal-${portal.id}` - try { const instanceTemplate = `https://www.googleapis.com/compute/v1/projects/${projectId}/global/instanceTemplates/portal-template` @@ -21,33 +16,27 @@ export const openPortalInstance = async (portal: Portal) => { url: `${baseUrl}instances?sourceInstanceTemplate=${instanceTemplate}`, method: 'POST', data: { - name: portalName + name: `server-${Date.now()}` } }) - await portal.updateStatus('starting') - - console.log(`opened portal with name ${portalName}`) + console.log('opened server using gcloud.driver') } catch(error) { - closePortal(portal.id) - console.error('error while opening portal', error) } } -export const closePortalInstance = async (portal: Portal) => { +export const closeServerInstance = async () => { const client = createClient() if(!client) throw 'The Google Cloud driver is incorrect. This may be due to improper ENV variables, please try again' - const portalName = `portal-${portal.id}` - try { - await client.request({ - url: `${baseUrl}instances/${portalName}`, - method: 'DELETE' - }) + // await client.request({ + // url: `${baseUrl}instances/${portalName}`, + // method: 'DELETE' + // }) - console.log(`closed portal with name ${portalName}`) + console.log('closed server using gcloud.driver') } catch(error) { console.error('error while closing portal', error.response ? error.response.body : error) } diff --git a/src/drivers/kubernetes.driver.ts b/src/drivers/kubernetes.driver.ts index 048b499..2a395ea 100644 --- a/src/drivers/kubernetes.driver.ts +++ b/src/drivers/kubernetes.driver.ts @@ -3,7 +3,6 @@ import { V1Pod, V1Node, CoreV1Api } from '@kubernetes/client-node' import Portal from '../models/portal' import { createClient } from '../config/providers/kubernetes.config' -import { closePortal } from './portal.driver' type Nodemap = { [key in string]: number @@ -55,11 +54,11 @@ export const fetchAvailableNode = (client?: CoreV1Api) => new Promise(as } }) -export const openPortalInstance = async (portal: Portal) => { +export const openServerInstance = async () => { const client = createClient() if(!client) throw 'The Kubernetes driver is incorrect. This may be due to improper ENV variables, please try again' - const name = `portal-${portal.id}` + const name = `server-${Date.now()}` try { const _pod = { @@ -102,32 +101,27 @@ export const openPortalInstance = async (portal: Portal) => { envFrom: [{ secretRef: { name: process.env.K8S_PORTAL_ENV_SECRET } }], - ports: [{ containerPort: 80 }], - args: ['--portalId', portal.id] + ports: [{ containerPort: 80 }] } ], imagePullSecrets: [{ name: process.env.K8S_PORTAL_IMAGE_PULL_SECRET }] } } as V1Pod, { body: pod } = await client.createNamespacedPod('portals', _pod) - console.log(`opened portal with name ${pod.metadata.name} in namespace ${pod.metadata.namespace}`) + console.log(`opened server using kubernetes.driver`) } catch(error) { - closePortal(portal.id) - console.error('error while opening portal', error.response ? error.response.body : error) } } -export const closePortalInstance = async (portal: Portal) => { +export const closeServerInstance = async () => { const client = createClient() if(!client) throw 'The Kubernetes driver is incorrect. This may be due to improper ENV variables, please try again' - const podName = `portal-${portal.id}` - try { - const { body: status } = await client.deleteNamespacedPod(podName, 'portals') + // const { body: status } = await client.deleteNamespacedPod(podName, 'portals') - console.log(`closed portal with name ${podName} which has been running since ${new Date(status.status['startTime'])}`) + console.log(`closed portal using kubernetes.driver`) } catch(error) { console.error('error while closing portal', error.response ? error.response.body : error) } diff --git a/src/drivers/manual.driver.ts b/src/drivers/manual.driver.ts deleted file mode 100644 index a9d76bc..0000000 --- a/src/drivers/manual.driver.ts +++ /dev/null @@ -1,48 +0,0 @@ -import Portal from '../models/portal' - -import { closePortal } from './portal.driver' - -const manualLogHeaders = [ - '--- IMPORTANT ---', - 'You\'re using the manual driver, which is intended for development.', -], manualLogFooters = [ - '------' -] - -export const openPortalInstance = async (portal: Portal) => { - const name = `portal-${portal.id}` - - try { - console.log([ - ...manualLogHeaders, - 'When starting @cryb/portal, use one of the following commands:', - `yarn docker:dev --portalId ${portal.id}`, - 'OR', - `npm run docker:dev --portalId ${portal.id}`, - ...manualLogFooters - ].join('\n')) - await portal.updateStatus('starting') - - console.log(`opened portal with name ${name}`) - } catch(error) { - closePortal(portal.id) - - console.error('error while opening portal', error) - } -} - -export const closePortalInstance = async (portal: Portal) => { - const name = `portal-${portal.id}` - - try { - console.log([ - ...manualLogHeaders, - `The Docker container running @cryb/portal with the portal id of ${portal.id} should now be terminated.`, - ...manualLogFooters - ].join('\n')) - - console.log(`closed portal with name ${name}`) - } catch(error) { - console.error('error while closing portal', error.response ? error.response.body : error) - } -} diff --git a/src/drivers/portal.driver.ts b/src/drivers/portal.driver.ts deleted file mode 100644 index f089424..0000000 --- a/src/drivers/portal.driver.ts +++ /dev/null @@ -1,33 +0,0 @@ -import Portal from '../models/portal' -import PortalRequest from '../models/request/defs' - -import { openPortalInstance, closePortalInstance } from './router' - -export const createPortal = (request: PortalRequest) => new Promise(async (resolve, reject) => { - try { - const portal = await new Portal().create(request) - openPortalInstance(portal) - - resolve(portal) - } catch(error) { - reject(error) - } -}) - -export const closePortal = (portalId: string) => new Promise(async (resolve, reject) => { - try { - const portal = await new Portal().load(portalId) - await portal.destroy() - - closePortalInstance(portal) - - // if(portal.status === 'connected') - // checkNextQueueItem() - - console.log('closing portal with status', portal.status) - - resolve() - } catch(error) { - reject(error) - } -}) diff --git a/src/drivers/router.ts b/src/drivers/router.ts index 0cc11dd..3073ea8 100644 --- a/src/drivers/router.ts +++ b/src/drivers/router.ts @@ -1,54 +1,47 @@ -import Portal from '../models/portal' - import { - openPortalInstance as openGCloudPortalInstance, - closePortalInstance as closeGCloudPortalInstance + openServerInstance as openGCloudServerInstance, + openServerInstance as closeGCloudServerInstance } from './gcloud.driver' import { - openPortalInstance as openK8SPortalInstance, - closePortalInstance as closeK8SPortalInstance + openServerInstance as openK8SServerInstance, + openServerInstance as closeK8SServerInstance } from './kubernetes.driver' -import { - openPortalInstance as openManualPortalInstance, - closePortalInstance as closeManualPortalInstance -} from './manual.driver' - -type Driver = 'gcloud' | 'kubernetes' | 'manual' +type Driver = 'gcloud' | 'kubernetes' | null -export const fetchCurrentDriver = () => 'manual' as Driver +export const fetchCurrentDriver = () => null as Driver -export const openPortalInstance = async (portal: Portal) => { +export const openServerInstance = async () => { const driver = await fetchCurrentDriver() - console.log('using driver', driver, 'to open portal with id', portal.id) + console.log('using driver', driver, 'to open server') switch(driver) { case 'gcloud': - openGCloudPortalInstance(portal) + openGCloudServerInstance() break case 'kubernetes': - openK8SPortalInstance(portal) + openK8SServerInstance() break - case 'manual': - openManualPortalInstance(portal) + default: + console.log('driver not found') break } } -export const closePortalInstance = async (portal: Portal) => { +export const closeServerInstance = async () => { const driver = await fetchCurrentDriver() - console.log('using driver', driver, 'to open portal with id', portal.id) + console.log('using driver', driver, 'to close server') switch(driver) { case 'gcloud': - closeGCloudPortalInstance(portal) + closeGCloudServerInstance() break case 'kubernetes': - closeK8SPortalInstance(portal) + closeK8SServerInstance() break - case 'manual': - closeManualPortalInstance(portal) + default: + console.log('driver not found') break } } diff --git a/src/models/portal/index.ts b/src/models/portal/index.ts index 2fc0c87..72f5cef 100644 --- a/src/models/portal/index.ts +++ b/src/models/portal/index.ts @@ -8,9 +8,13 @@ import PortalRequest from '../request/defs' import IPortal from './defs' import StoredPortal from '../../schemas/portal.schema' +import config from '../../config/defaults' +import StoredServer from '../../schemas/server.schema' + +import { openServerInstance } from '../../drivers/router' + import { generateFlake } from '../../utils/generate.utils' import { createPubSubClient } from '../../config/redis.config' -import StoredServer from '../../schemas/server.schema' const pub = createPubSubClient() @@ -71,7 +75,8 @@ export default class Portal { const server = new Server(serverDoc) console.log('Assigning portal to server', server.id) await server.assign(this) - } else console.log('Could not assign portal to server') + } else if(config.dynamic_vms_enabled) + openServerInstance() /** * Inform API of new portal with room id diff --git a/src/schemas/portal.schema.ts b/src/schemas/portal.schema.ts index 94d06aa..abe85db 100644 --- a/src/schemas/portal.schema.ts +++ b/src/schemas/portal.schema.ts @@ -12,9 +12,6 @@ const PortalSchema = new Schema({ server: String, status: String - }, - data: { - serverId: String } }) From bf1a2b563a5c610b0a3439339611680f2fd62b97 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Wed, 23 Oct 2019 01:05:28 +0100 Subject: [PATCH 09/11] Fix server unassign on portal destroy --- src/models/portal/index.ts | 27 ++++++++++++++++----------- src/models/server/index.ts | 2 ++ src/utils/helpers.utils.ts | 4 +++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/models/portal/index.ts b/src/models/portal/index.ts index 72f5cef..34746a9 100644 --- a/src/models/portal/index.ts +++ b/src/models/portal/index.ts @@ -1,7 +1,7 @@ import axios from 'axios' import { sign } from 'jsonwebtoken' -import Server from '../server' +import Server, { ServerResolvable } from '../server' import PortalRequest from '../request/defs' @@ -14,6 +14,7 @@ import StoredServer from '../../schemas/server.schema' import { openServerInstance } from '../../drivers/router' import { generateFlake } from '../../utils/generate.utils' +import { extractServerId } from '../../utils/helpers.utils' import { createPubSubClient } from '../../config/redis.config' const pub = createPubSubClient() @@ -27,7 +28,7 @@ export default class Portal { recievedAt: number room: string - server?: string + server?: ServerResolvable status: PortalStatus @@ -77,15 +78,7 @@ export default class Portal { await server.assign(this) } else if(config.dynamic_vms_enabled) openServerInstance() - - /** - * Inform API of new portal with room id - */ - // await axios.post(`${process.env.API_URL}/internal/portal`, { id: this.id, roomId }, { - // headers: { - // authorization: `Valve ${sign({}, process.env.API_KEY)}` - // } - // }) + else console.log('Could not assign portal to server') resolve(this) } catch(error) { @@ -95,6 +88,18 @@ export default class Portal { destroy = (error?: string) => new Promise(async (resolve, reject) => { try { + if(this.server) { + const serverId = extractServerId(this.server) + + let server: Server + if(typeof this.server === 'string') + server = await new Server().load(serverId) + else + server = this.server + + await server.unassign() + } + await StoredPortal.deleteOne({ 'info.id': this.id }) diff --git a/src/models/server/index.ts b/src/models/server/index.ts index 1c2120e..b0e5962 100644 --- a/src/models/server/index.ts +++ b/src/models/server/index.ts @@ -9,6 +9,8 @@ import client from '../../config/redis.config' import { generateFlake } from '../../utils/generate.utils' import { extractPortalId } from '../../utils/helpers.utils' +export type ServerResolvable = Server | string + export default class Server { id: string connectedAt: number diff --git a/src/utils/helpers.utils.ts b/src/utils/helpers.utils.ts index b276ecc..672601a 100644 --- a/src/utils/helpers.utils.ts +++ b/src/utils/helpers.utils.ts @@ -1,3 +1,5 @@ import { PortalResolvable } from '../models/portal' +import { ServerResolvable } from '../models/server' -export const extractPortalId = (portal: PortalResolvable) => portal ? (typeof portal === 'string' ? portal : portal.id) : null \ No newline at end of file +export const extractPortalId = (portal: PortalResolvable) => portal ? (typeof portal === 'string' ? portal : portal.id) : null +export const extractServerId = (server: ServerResolvable) => server ? (typeof server === 'string' ? server : server.id) : null \ No newline at end of file From 778250b298c2e61575057f80b8c5afde74957b56 Mon Sep 17 00:00:00 2001 From: William Gibson Date: Sun, 27 Oct 2019 18:34:56 +0000 Subject: [PATCH 10/11] Add deployment logic --- .env.example | 4 + src/config/defaults.ts | 6 +- src/config/providers/example.config.ts | 4 +- src/config/words.config.ts | 28 ++++ src/drivers/example.driver.ts | 9 +- src/drivers/gcloud.driver.ts | 19 ++- src/drivers/kubernetes.driver.ts | 94 ++++++------- src/drivers/router.ts | 2 +- src/drivers/utils/register.utils.driver.ts | 37 ++++++ src/models/deployment/defs.ts | 22 +++ src/models/deployment/index.ts | 148 +++++++++++++++++++++ src/schemas/deployment.schema.ts | 21 +++ src/server/websocket/handlers.ts | 17 ++- src/server/websocket/index.ts | 8 ++ src/utils/generate.utils.ts | 1 + 15 files changed, 356 insertions(+), 64 deletions(-) create mode 100644 src/config/words.config.ts create mode 100644 src/drivers/utils/register.utils.driver.ts create mode 100644 src/models/deployment/defs.ts create mode 100644 src/models/deployment/index.ts create mode 100644 src/schemas/deployment.schema.ts diff --git a/.env.example b/.env.example index b3bfef0..8e428b5 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,10 @@ REDIS_URI= # Enable dynamic spawning of VM instances when none are available ENABLE_DYNAMIC_VMS=false +# The amount of random words that should be chosen for a dynamic vm name. Defaults to 2 +DYNAMIC_VM_NAME_WORD_COUNT=2 +# The maximum amount of servers that can be deployed dynamically +MAX_VM_COUNT=10 # Optional: the JSON Google Application Credentials used for gcloud.driver # GOOGLE_APPLICATION_CREDENTIALS= diff --git a/src/config/defaults.ts b/src/config/defaults.ts index a6133f9..896932a 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -2,5 +2,9 @@ export default { /** * Boolean dependant on if dynamic VMs are enabled */ - dynamic_vms_enabled: process.env.ENABLE_DYNAMIC_VMS === 'true' + dynamic_vms_enabled: process.env.ENABLE_DYNAMIC_VMS === 'true', + /** + * The amount of words that should be generated for a dynamic vm deployment name + */ + dynamic_vm_word_count: parseInt(process.env.DYNAMIC_VM_NAME_WORD_COUNT || '2') || 2 } \ No newline at end of file diff --git a/src/config/providers/example.config.ts b/src/config/providers/example.config.ts index e584077..0bc2a22 100644 --- a/src/config/providers/example.config.ts +++ b/src/config/providers/example.config.ts @@ -1,6 +1,6 @@ export const createClient = () => { return { - createServer: async () => {}, - destroyServer: async () => {} + createServer: async (name: string) => {}, + destroyServer: async (name: string) => {} } } diff --git a/src/config/words.config.ts b/src/config/words.config.ts new file mode 100644 index 0000000..553385f --- /dev/null +++ b/src/config/words.config.ts @@ -0,0 +1,28 @@ +export default [ + 'alfa', + 'bravo', + 'charlie', + 'delta', + 'echo', + 'foxtrot', + 'golf', + 'hotel', + 'india', + 'juliett', + 'kilo', + 'lima', + 'mike', + 'november', + 'oscar', + 'papa', + 'quebec', + 'romeo', + 'sierra', + 'tango', + 'uniform', + 'victor', + 'whiskey', + 'xray', + 'yankee', + 'zulu' +] diff --git a/src/drivers/example.driver.ts b/src/drivers/example.driver.ts index 215bc19..afe1aa8 100644 --- a/src/drivers/example.driver.ts +++ b/src/drivers/example.driver.ts @@ -1,12 +1,14 @@ // Import the API you wish to use import { createClient } from '../config/providers/example.config' +import { registerDeployment, deregisterDeployment } from './utils/register.utils.driver' export const openServerInstance = async () => { const client = createClient() try { // Create the server using the API & Provider of your choice - await client.createServer() + const { name } = await registerDeployment('example') + await client.createServer(name) console.log(`opened portal using example.driver`) } catch(error) { @@ -14,12 +16,13 @@ export const openServerInstance = async () => { } } -export const closeServerInstance = async () => { +export const closeServerInstance = async (name: string) => { const client = createClient() try { // Destroy the server using the id of the server you stored when creating the server - await client.destroyServer() + await deregisterDeployment(name) + await client.destroyServer(name) console.log(`closed portal using example.driver`) } catch(error) { diff --git a/src/drivers/gcloud.driver.ts b/src/drivers/gcloud.driver.ts index 8146846..baefd79 100644 --- a/src/drivers/gcloud.driver.ts +++ b/src/drivers/gcloud.driver.ts @@ -1,5 +1,7 @@ import { createClient, fetchCredentials } from '../config/providers/gcloud.config' +import { registerDeployment, deregisterDeployment } from './utils/register.utils.driver' + const { project_id: projectId } = fetchCredentials() || { project_id: null }, zoneId = 'us-east1-b', baseUrl = `https://www.googleapis.com/compute/v1/projects/${projectId}/zones/${zoneId}/` @@ -9,32 +11,29 @@ export const openServerInstance = async () => { if(!client) throw 'The Google Cloud driver is incorrect. This may be due to improper ENV variables, please try again' try { - const instanceTemplate = `https://www.googleapis.com/compute/v1/projects/${projectId}/global/instanceTemplates/portal-template` + const { name } = await registerDeployment('gcloud'), + instanceTemplate = `https://www.googleapis.com/compute/v1/projects/${projectId}/global/instanceTemplates/portal-template` // Create a VM under the template 'portal-template' with the name 'portal-{id}' await client.request({ url: `${baseUrl}instances?sourceInstanceTemplate=${instanceTemplate}`, method: 'POST', - data: { - name: `server-${Date.now()}` - } + data: { name } }) - console.log('opened server using gcloud.driver') + console.log('opened server using gcloud.driver with name', name) } catch(error) { console.error('error while opening portal', error) } } -export const closeServerInstance = async () => { +export const closeServerInstance = async (name: string) => { const client = createClient() if(!client) throw 'The Google Cloud driver is incorrect. This may be due to improper ENV variables, please try again' try { - // await client.request({ - // url: `${baseUrl}instances/${portalName}`, - // method: 'DELETE' - // }) + await deregisterDeployment(name) + await client.request({ url: `${baseUrl}instances/${name}`, method: 'DELETE' }) console.log('closed server using gcloud.driver') } catch(error) { diff --git a/src/drivers/kubernetes.driver.ts b/src/drivers/kubernetes.driver.ts index 2a395ea..318d0b2 100644 --- a/src/drivers/kubernetes.driver.ts +++ b/src/drivers/kubernetes.driver.ts @@ -3,6 +3,7 @@ import { V1Pod, V1Node, CoreV1Api } from '@kubernetes/client-node' import Portal from '../models/portal' import { createClient } from '../config/providers/kubernetes.config' +import { registerDeployment, deregisterDeployment } from './utils/register.utils.driver' type Nodemap = { [key in string]: number @@ -58,55 +59,55 @@ export const openServerInstance = async () => { const client = createClient() if(!client) throw 'The Kubernetes driver is incorrect. This may be due to improper ENV variables, please try again' - const name = `server-${Date.now()}` - try { - const _pod = { - apiVersion: 'v1', - kind: 'Pod', - metadata: { - name, - labels: { - name, - kind: 'portal' - }, - namespace: 'portals' - }, - spec: { - volumes: [{ - name: 'dshm', - emptyDir: { - medium: 'Memory' - } - }], - containers: [ - { + const { name } = await registerDeployment('kubernetes'), + _pod = { + apiVersion: 'v1', + kind: 'Pod', + metadata: { name, - image: process.env.K8S_PORTAL_IMAGE_REGISTRY_URL, - resources: { - limits: { - cpu: process.env.K8S_PORTAL_CPU_LIMIT, - memory: process.env.K8S_PORTAL_MEMORY_LIMIT - }, - requests: { - cpu: process.env.K8S_PORTAL_CPU_REQUESTED, - memory: process.env.K8S_PORTAL_MEMORY_REQUESTED - } + labels: { + name, + kind: 'portal' }, - volumeMounts: [{ - mountPath: '/dev/shm', - name: 'dshm' - }], - imagePullPolicy: 'Always', - envFrom: [{ - secretRef: { name: process.env.K8S_PORTAL_ENV_SECRET } + namespace: 'portals' + }, + spec: { + volumes: [{ + name: 'dshm', + emptyDir: { + medium: 'Memory' + } }], - ports: [{ containerPort: 80 }] + containers: [ + { + name, + image: process.env.K8S_PORTAL_IMAGE_REGISTRY_URL, + resources: { + limits: { + cpu: process.env.K8S_PORTAL_CPU_LIMIT, + memory: process.env.K8S_PORTAL_MEMORY_LIMIT + }, + requests: { + cpu: process.env.K8S_PORTAL_CPU_REQUESTED, + memory: process.env.K8S_PORTAL_MEMORY_REQUESTED + } + }, + volumeMounts: [{ + mountPath: '/dev/shm', + name: 'dshm' + }], + imagePullPolicy: 'Always', + envFrom: [{ + secretRef: { name: process.env.K8S_PORTAL_ENV_SECRET } + }], + ports: [{ containerPort: 80 }] + } + ], + imagePullSecrets: [{ name: process.env.K8S_PORTAL_IMAGE_PULL_SECRET }] } - ], - imagePullSecrets: [{ name: process.env.K8S_PORTAL_IMAGE_PULL_SECRET }] - } - } as V1Pod, { body: pod } = await client.createNamespacedPod('portals', _pod) + } as V1Pod, + { body: pod } = await client.createNamespacedPod('portals', _pod) console.log(`opened server using kubernetes.driver`) } catch(error) { @@ -114,12 +115,13 @@ export const openServerInstance = async () => { } } -export const closeServerInstance = async () => { +export const closeServerInstance = async (name: string) => { const client = createClient() if(!client) throw 'The Kubernetes driver is incorrect. This may be due to improper ENV variables, please try again' try { - // const { body: status } = await client.deleteNamespacedPod(podName, 'portals') + await deregisterDeployment(name) + await client.deleteNamespacedPod(name, 'portals') console.log(`closed portal using kubernetes.driver`) } catch(error) { diff --git a/src/drivers/router.ts b/src/drivers/router.ts index 3073ea8..1657c3f 100644 --- a/src/drivers/router.ts +++ b/src/drivers/router.ts @@ -8,7 +8,7 @@ import { openServerInstance as closeK8SServerInstance } from './kubernetes.driver' -type Driver = 'gcloud' | 'kubernetes' | null +export type Driver = 'gcloud' | 'kubernetes' | 'example' | null export const fetchCurrentDriver = () => null as Driver diff --git a/src/drivers/utils/register.utils.driver.ts b/src/drivers/utils/register.utils.driver.ts new file mode 100644 index 0000000..94827ec --- /dev/null +++ b/src/drivers/utils/register.utils.driver.ts @@ -0,0 +1,37 @@ +import Deployment from '../../models/deployment' +import { Driver } from '../router' + +import config from '../../config/defaults' +import wordPool from '../../config/words.config' +import { generateRandomInt } from '../../utils/generate.utils' + +const generateRandomWord = (exclude: string[]) => wordPool.filter(word => exclude.indexOf(word) === -1)[generateRandomInt(0, wordPool.length - exclude.length)] + +export const registerDeployment = async (provider: Driver) => { + let words = [] + + for(let i = 0; i < config.dynamic_vm_word_count; i++) + words.push(generateRandomWord(words)) + + try { + const name = `cryb_portal-${words.join('-')}`, + deployment = await new Deployment().create(name, provider) + + return deployment + } catch(error) { + console.error(error) + return null + } +} + +export const deregisterDeployment = async (name: string) => { + try { + const deployment = await new Deployment().findByName(name) + await deployment.destroy() + + return name + } catch(error) { + console.error(error) + return null + } +} diff --git a/src/models/deployment/defs.ts b/src/models/deployment/defs.ts new file mode 100644 index 0000000..020ff0b --- /dev/null +++ b/src/models/deployment/defs.ts @@ -0,0 +1,22 @@ +import { Document } from 'mongoose' + +import { Driver } from '../../drivers/router' + +export type DeploymentStatus = 'ready' | 'in-use' | 'creating' + +export default interface IDeployment { + info: { + id: string + deployedAt: number + + status: DeploymentStatus + + server?: string + provider: Driver + } + data: { + name: string + } +} + +export interface IStoredDeployment extends IDeployment, Document {} diff --git a/src/models/deployment/index.ts b/src/models/deployment/index.ts new file mode 100644 index 0000000..cce5dc6 --- /dev/null +++ b/src/models/deployment/index.ts @@ -0,0 +1,148 @@ +import IDeployment, { DeploymentStatus } from './defs' +import StoredDeployment from '../../schemas/deployment.schema' + +import Server, { ServerResolvable } from '../server' + +import { Driver } from '../../drivers/router' +import { generateFlake } from '../../utils/generate.utils' +import { extractServerId } from '../../utils/helpers.utils' + +export type DeploymentResolvable = Deployment | string + +export default class Deployment { + id: string + deployedAt: number + + status: DeploymentStatus + + server?: ServerResolvable + provider: Driver + + name: string + + constructor(json?: IDeployment) { + if(!json) return + + this.setup(json) + } + + load = (id: string) => new Promise(async (resolve, reject) => { + try { + const doc = await StoredDeployment.findOne({ 'info.id': id }) + if(!doc) throw 'DeploymentNotFound' + + this.setup(doc) + + resolve(this) + } catch(error) { + reject(error) + } + }) + + findByName = (name: string) => new Promise(async (resolve, reject) => { + try { + const doc = await StoredDeployment.findOne({ 'data.name': name }) + if(!doc) throw 'DeploymentNotFound' + + this.setup(doc) + + resolve(this) + } catch(error) { + reject(error) + } + }) + + create = (name: string, provider: Driver) => new Promise(async (resolve, reject) => { + try { + const json: IDeployment = { + info: { + id: generateFlake(), + deployedAt: Date.now(), + + status: 'creating', + + provider + }, + data: { + name + } + } + + const stored = new StoredDeployment() + await stored.save() + + this.setup(json) + + resolve(this) + } catch(error) { + reject(error) + } + }) + + destroy = () => new Promise(async (resolve, reject) => { + try { + if(this.server) { + let server: Server + + if(typeof this.server === 'string') + server = await new Server().load(this.server) + else + server = this.server + + await server.destroy() + } + + await StoredDeployment.deleteOne({ + 'info.id': this.id + }) + + resolve() + } catch(error) { + reject(error) + } + }) + + assignServer = (server: ServerResolvable) => new Promise(async (resolve, reject) => { + try { + await StoredDeployment.updateOne({ + 'info.id': this.id + }, { + 'info.server': extractServerId(server) + }) + + this.server = server + + resolve(this) + } catch(error) { + reject(error) + } + }) + + updateStatus = (status: DeploymentStatus) => new Promise(async (resolve, reject) => { + try { + await StoredDeployment.updateOne({ + 'info.id': this.id + }, { + 'info.status': status + }) + + this.status = status + + resolve(this) + } catch(error) { + reject(error) + } + }) + + setup = (json: IDeployment) => { + this.id = json.info.id + this.deployedAt = json.info.deployedAt + + this.status = json.info.status + + if(json.info.server) this.server = json.info.server + this.provider = json.info.provider + + this.name = json.data.name + } +} \ No newline at end of file diff --git a/src/schemas/deployment.schema.ts b/src/schemas/deployment.schema.ts new file mode 100644 index 0000000..08e8d7d --- /dev/null +++ b/src/schemas/deployment.schema.ts @@ -0,0 +1,21 @@ +import { Schema, model } from 'mongoose' + +import { IStoredDeployment } from '../models/deployment/defs' + +const DeploymentSchema = new Schema({ + info: { + id: String, + deployedAt: Number, + + status: String, + + server: String, + provider: String + }, + data: { + name: String + } +}) + +const StoredDeployment = model('Deployment', DeploymentSchema) +export default StoredDeployment diff --git a/src/server/websocket/handlers.ts b/src/server/websocket/handlers.ts index 0ff76f4..73ca859 100644 --- a/src/server/websocket/handlers.ts +++ b/src/server/websocket/handlers.ts @@ -5,10 +5,16 @@ import Server from '../../models/server' import WSEvent, { ClientType } from './defs' import client from '../../config/redis.config' +import Deployment from '../../models/deployment' const ACCEPTABLE_CLIENT_TYPES: ClientType[] = ['server'], isClientWithIdAndType = (id: string, type: ClientType) => (client: WebSocket) => client['id'] === id && client['type'] === type +interface IBeacon { + id?: string + hostname?: string +} + /** * Message incoming from Portal over WS */ @@ -22,7 +28,7 @@ const handleMessage = async (message: WSEvent, socket: WebSocket) => { if(op === 2) { try { const { token, type } = d, - payload = verify(token, process.env.PORTAL_KEY) as { id?: string } + payload = verify(token, process.env.PORTAL_KEY) as IBeacon if(!payload) return socket.close(1013) if(ACCEPTABLE_CLIENT_TYPES.indexOf(type) === -1) return socket.close(1013) @@ -40,6 +46,15 @@ const handleMessage = async (message: WSEvent, socket: WebSocket) => { socket['id'] = server.id socket.send(JSON.stringify({ op: 10, d: { id: server.id } })) + if(payload.hostname) { + const deployment = await new Deployment().findByName(payload.hostname) + + deployment.assignServer(server) + deployment.updateStatus('in-use') + + socket['deployment'] = deployment.id + } + console.log('recieved auth from', type, server.id) } else return socket.close(1013) } catch(error) { diff --git a/src/server/websocket/index.ts b/src/server/websocket/index.ts index f446341..62a44ff 100644 --- a/src/server/websocket/index.ts +++ b/src/server/websocket/index.ts @@ -5,6 +5,7 @@ import Server from '../../models/server' import WSEvent from './defs' import { createPubSubClient } from '../../config/redis.config' import handleMessage, { routeMessage } from './handlers' +import Deployment from '../../models/deployment' const sub = createPubSubClient() @@ -48,6 +49,13 @@ export default (wss: WSS) => { if(type === 'server') { const server = await new Server().load(id) server.destroy() + + if(socket['deployment']) { + const deploymentId = socket['deployment'], + deployment = await new Deployment().load(deploymentId) + + deployment.destroy() + } } }) }) diff --git a/src/utils/generate.utils.ts b/src/utils/generate.utils.ts index 2d006a1..b9b5bfd 100644 --- a/src/utils/generate.utils.ts +++ b/src/utils/generate.utils.ts @@ -6,3 +6,4 @@ const flake = new FlakeId({ }) export const generateFlake = () => intformat(flake.next(), 'dec') +export const generateRandomInt = (min: number, max: number) => Math.floor((Math.random() * max) + min) \ No newline at end of file From c533f7b792894ee5252e5ecf6c50b90854f8030a Mon Sep 17 00:00:00 2001 From: William Gibson Date: Sun, 27 Oct 2019 18:37:18 +0000 Subject: [PATCH 11/11] Add 'EXPERIMENTAL' label to dynamic vm env property --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 8e428b5..5676c97 100644 --- a/.env.example +++ b/.env.example @@ -20,7 +20,7 @@ REDIS_URI= # Make sure to uncomment the below line and uncomment the REDIS_URI line above # REDIS_SENTINELS= -# Enable dynamic spawning of VM instances when none are available +# EXPERIMENTAL: Enable dynamic spawning of VM instances when none are available ENABLE_DYNAMIC_VMS=false # The amount of random words that should be chosen for a dynamic vm name. Defaults to 2 DYNAMIC_VM_NAME_WORD_COUNT=2