From 09731374e8966af23bc579d350baed2fd1c82752 Mon Sep 17 00:00:00 2001 From: Andrew Lloyd Date: Sun, 11 Mar 2018 21:49:46 -0700 Subject: [PATCH 1/6] WIP: Add script to build coverage badge for README (#63) * Add script to build coverage badge for README * Fix linting issues * Use Webtask API to update coverage badge --- .travis.yml | 2 +- README.md | 19 ++++++++----------- config/lcov2badge.js | 32 ++++++++++++++++++++++++++++++++ package.json | 4 +++- 4 files changed, 44 insertions(+), 13 deletions(-) create mode 100644 config/lcov2badge.js diff --git a/.travis.yml b/.travis.yml index 0e88807..60d3426 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,4 +16,4 @@ script: - npm run lint - npm run test:check-coverage after_script: - # - npm run report-coverage + - npm run report-coverage diff --git a/README.md b/README.md index dda85b2..cf283b0 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,21 @@ [![logo](https://i.imgur.com/3OtP3p8.png)](https://softwareengineeringdaily.com/) -# SEDaily Event Stream Processor +# SEDaily Logging, Monitoring, and Analytics -The real time event processing infrastructure gateway server for the Software Engineering Daily [Android](https://github.com/SoftwareEngineeringDaily/SEDaily-Android), [iOS](https://github.com/SoftwareEngineeringDaily/se-daily-iOS), and [web front end](https://github.com/SoftwareEngineeringDaily/sedaily-front-end). The SEDaily event stream is responsible for authenticating connecting clients and validating event payload schemas before putting the event on the SED event bus. Interested clients can subscribe to events on the stream. +The real time event processing infrastructure gateway server for the Software Engineering Daily [Android](https://github.com/SoftwareEngineeringDaily/SEDaily-Android), [iOS](https://github.com/SoftwareEngineeringDaily/se-daily-iOS), and [web front end](https://github.com/SoftwareEngineeringDaily/sedaily-front-end). The SEDaily event stream is responsible for validating event payload schemas before putting the event into InfluxDB. The resulting database is queried using Grafana to get up to the second analytics reporting. - - +
+ + +
## Getting Started ```sh -$ git clone https://github.com/SoftwareEngineeringDaily/sedaily-event-stream.git -$ cd sedaily-event-stream +$ git clone https://github.com/SoftwareEngineeringDaily/sedaily-devops.git +$ cd sedaily-devops ``` ### Set up (local) - - Install and run a local [InfluxDB](https://github.com/influxdata/influxdb) client - `cp .env.local_example .env` - `npm install` or `yarn install` @@ -26,10 +27,6 @@ $ cd sedaily-event-stream - Run `docker-compose up` - If dependencies are updated in package.json, run `docker-compose down` and then `docker-compose up --build`. This will remove the old container and rebuild the API image which installs the new dependencies. -## Current State - -The current state of the SEDaily event stream is analytics focused. The event stream backend relies on Redis Streams, which is only currently available in the [`unstable`](https://github.com/antirez/redis/tree/unstable) branch. Until it is stable, the SEDaily event stream will only directly input events into InfluxDB. Data analytics can be run against queries on the InfluxDB events database. - ## Contributing We use the develop branch to perform work in. Fork the project and clone it, create a branch off of develop and perform your changes. Then submit a pull request to merge your branch into the develop branch here. We have an active Slack community that you can reach out to for more information or just to chat with anyone. Check out the [Slack Channel SED app development](https://softwaredaily.slack.com/app_redirect?channel=sed_app_development) slack channel. Also see the [Open Source Guide](https://softwareengineeringdaily.github.io/). diff --git a/config/lcov2badge.js b/config/lcov2badge.js new file mode 100644 index 0000000..ceddbc3 --- /dev/null +++ b/config/lcov2badge.js @@ -0,0 +1,32 @@ +const request = require('request'); +const lcov2badge = require('lcov2badge'); + +const filePath = process.argv[2]; + +const options = { + filePath: filePath || './coverage/lcov.info', + warnThreshold: 80, // default is 80 + koThreshold: 60, // default is 60 +}; + +function sendCoverageBadge(coverageBadge) { + const requestOptions = { + method: 'post', + body: { coverageBadge, coverageBadgeToken: process.env.COVERAGE_BADGE_TOKEN }, + json: true, + url: process.env.COVERAGE_BADGE_URL + }; + request(requestOptions, (err, res, body) => { + if (err) { + throw err; + } else { + console.log(body.result); // eslint-disable-line no-console + } + }); +} + + +lcov2badge.badge(options, (err, svgBadge) => { + if (err) throw err; + sendCoverageBadge(svgBadge); +}); diff --git a/package.json b/package.json index d732497..b93b9d6 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:watch": "npm run test -- --watch", "test:coverage": "cross-env NODE_ENV=test ./node_modules/.bin/istanbul cover _mocha -- --ui bdd --reporter spec --colors --compilers js:babel-core/register tests --recursive", "test:check-coverage": "npm run test:coverage && istanbul check-coverage", - "report-coverage": "coveralls < ./coverage/lcov.info" + "report-coverage": "node config/lcov2badge.js" }, "repository": { "type": "git", @@ -103,7 +103,9 @@ ] }, "devDependencies": { + "lcov2badge": "^0.1.0", "minimist": "^1.2.0", + "request": "^2.83.0", "swagger-node-express": "^2.1.3" } } From ad280db3bb968b7412289fb77c68ea01ef858cbc Mon Sep 17 00:00:00 2001 From: Andrew Lloyd Date: Sun, 11 Mar 2018 21:57:28 -0700 Subject: [PATCH 2/6] Rename coverageReporter --- config/{lcov2badge.js => coverageReporter.js} | 0 package.json | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename config/{lcov2badge.js => coverageReporter.js} (100%) diff --git a/config/lcov2badge.js b/config/coverageReporter.js similarity index 100% rename from config/lcov2badge.js rename to config/coverageReporter.js diff --git a/package.json b/package.json index b93b9d6..cc29a04 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "test:watch": "npm run test -- --watch", "test:coverage": "cross-env NODE_ENV=test ./node_modules/.bin/istanbul cover _mocha -- --ui bdd --reporter spec --colors --compilers js:babel-core/register tests --recursive", "test:check-coverage": "npm run test:coverage && istanbul check-coverage", - "report-coverage": "node config/lcov2badge.js" + "report-coverage": "node config/coverageReporter.js" }, "repository": { "type": "git", From a45d484ee35ee3fa87bac078096dcfc97ec8c7c7 Mon Sep 17 00:00:00 2001 From: Andrew Lloyd Date: Sun, 11 Mar 2018 22:02:26 -0700 Subject: [PATCH 3/6] Move request and lcov2badge to dependencies so travis ci can build --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index cc29a04..34d7805 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "joi": "10.6.0", "jsonwebtoken": "8.0.0", "kafka-node": "2.2.3", + "lcov2badge": "^0.1.0", "lodash": "4.17.4", "method-override": "2.3.9", "mocha": "3.5.0", @@ -79,6 +80,7 @@ "passport-facebook-token": "3.3.0", "raccoon": "0.2.8", "redis": "^2.8.0", + "request": "^2.83.0", "run-sequence": "2.1.0", "sinon": "^4.1.2", "supertest": "3.0.0", @@ -103,9 +105,7 @@ ] }, "devDependencies": { - "lcov2badge": "^0.1.0", "minimist": "^1.2.0", - "request": "^2.83.0", "swagger-node-express": "^2.1.3" } } From 33987ca37207d1ebd8a23ef243814a0195087792 Mon Sep 17 00:00:00 2001 From: Andrew Lloyd Date: Sun, 11 Mar 2018 22:06:26 -0700 Subject: [PATCH 4/6] Add coverage reporter to test coverage excludes --- .istanbul.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.istanbul.yml b/.istanbul.yml index 350acff..136b186 100644 --- a/.istanbul.yml +++ b/.istanbul.yml @@ -1,6 +1,6 @@ verbose: false instrumentation: - excludes: ['dist/**', 'coverage/**', 'gulpfile.babel.js', 'api/**', 'cli/**'] + excludes: ['dist/**', 'coverage/**', 'gulpfile.babel.js', 'api/**', 'cli/**', 'config/coverageReporter.js'] include-all-sources: true reporting: print: summary From b11524100bc6c2aa9f78cc8759046d91d0bdf3cb Mon Sep 17 00:00:00 2001 From: Andrew Lloyd Date: Sun, 11 Mar 2018 22:16:01 -0700 Subject: [PATCH 5/6] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cf283b0..bcdda56 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The real time event processing infrastructure gateway server for the Software En
- +
## Getting Started From 3ea79e42bd8a466e0f78b38fa281f63700952eb2 Mon Sep 17 00:00:00 2001 From: Andrew Lloyd Date: Sat, 31 Mar 2018 18:59:12 -0700 Subject: [PATCH 6/6] Add a field to identify the environment (Test or Prod) (#67) * Add a field to identify the environment * Allow only types 'production' and 'test' for eventApiEnvs * Add unit tests --- config/param-validation.js | 2 + tests/server/errors/error.test.js | 48 +++++++++++++++++++++++ tests/server/events/event.test.js | 55 ++++++++++++++++++++++++++- tests/server/events/register.event.js | 1 + 4 files changed, 105 insertions(+), 1 deletion(-) diff --git a/config/param-validation.js b/config/param-validation.js index 12d312c..c47a3b9 100644 --- a/config/param-validation.js +++ b/config/param-validation.js @@ -5,6 +5,7 @@ export default { body: { clientId: Joi.string().required(), deviceType: Joi.string().required().valid(['iOS', 'Android', 'Browser', 'API']), + eventApiEnv: Joi.string().required().valid(['production', 'test']), eventTime: Joi.date().timestamp('unix').required(), eventData: Joi.object().required(), eventType: Joi.string().required().valid([ @@ -26,6 +27,7 @@ export default { body: { clientId: Joi.string().required(), deviceType: Joi.string().required().valid(['iOS', 'Android', 'Browser', 'API']), + eventApiEnv: Joi.string().required().valid(['production', 'test']), errorTime: Joi.date().timestamp('unix').required(), errorData: Joi.object().required(), } diff --git a/tests/server/errors/error.test.js b/tests/server/errors/error.test.js index 297f8c8..cc7e846 100644 --- a/tests/server/errors/error.test.js +++ b/tests/server/errors/error.test.js @@ -9,6 +9,7 @@ describe('## Basic Error APIs', () => { it('sending valid error is successful', (done) => { const error = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'API', errorTime: new Date().getTime(), errorData: { @@ -30,6 +31,7 @@ describe('## Basic Error APIs', () => { it('errors when no clientId is sent', (done) => { const error = { deviceType: 'Browser', + eventApiEnv: 'production', errorTime: new Date().getTime(), errorData: {} }; @@ -48,6 +50,7 @@ describe('## Basic Error APIs', () => { it('errors when no deviceType is sent', (done) => { const error = { clientId: '1234567', + eventApiEnv: 'production', errorTime: new Date().getTime(), errorData: { userId: '3462562' @@ -68,6 +71,7 @@ describe('## Basic Error APIs', () => { it('errors when no errorTime is sent', (done) => { const error = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'API', errorData: { userId: '3462562' @@ -88,6 +92,7 @@ describe('## Basic Error APIs', () => { it('errors when no errorData is sent', (done) => { const error = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'API', errorTime: new Date().getTime() }; @@ -102,4 +107,47 @@ describe('## Basic Error APIs', () => { }) .catch(done); }); + + it('errors when no eventApiEnv is sent', (done) => { + const error = { + clientId: '1234567', + errorData: { + userId: '3462562' + }, + deviceType: 'API', + errorTime: new Date().getTime() + }; + request(app) + .post('/api/v1/error') + .send(error) + .expect(httpStatus.BAD_REQUEST) + .then((res) => { + expect(res.body).to.exist; //eslint-disable-line + expect(res.body.message).to.equal('"eventApiEnv" is required') + done(); + }) + .catch(done); + }); + + it('errors when eventApiEnv sent is wrong type', (done) => { + const error = { + clientId: '1234567', + eventApiEnv: 'prod', + errorData: { + userId: '3462562' + }, + deviceType: 'API', + errorTime: new Date().getTime() + }; + request(app) + .post('/api/v1/error') + .send(error) + .expect(httpStatus.BAD_REQUEST) + .then((res) => { + expect(res.body).to.exist; //eslint-disable-line + expect(res.body.message).to.equal('"eventApiEnv" must be one of [production, test]'); //eslint-disable-line + done(); + }) + .catch(done); + }); }); diff --git a/tests/server/events/event.test.js b/tests/server/events/event.test.js index 735edd9..f3bac36 100644 --- a/tests/server/events/event.test.js +++ b/tests/server/events/event.test.js @@ -9,6 +9,7 @@ describe('## Basic Event APIs', () => { it('sending valid event is successful', (done) => { const event = { clientId: '1234567', + eventApiEnv: 'test', deviceType: 'iOS', eventTime: new Date().getTime(), eventType: 'login', @@ -40,6 +41,7 @@ describe('## Basic Event APIs', () => { it('errors with invalid deviceType', (done) => { const event = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'Windows phone', eventTime: new Date().getTime(), eventType: 'login', @@ -62,6 +64,7 @@ describe('## Basic Event APIs', () => { it('errors with invalid timestamp', (done) => { const event = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'Browser', eventTime: new Date(), eventType: 'login', @@ -89,7 +92,7 @@ describe('## Basic Event APIs', () => { .expect(httpStatus.BAD_REQUEST) .then((res) => { expect(res.body).to.exist; //eslint-disable-line - expect(res.body.message).to.equal('"clientId" is required and "deviceType" is required and "eventTime" is required and "eventData" is required and "eventType" is required'); //eslint-disable-line + expect(res.body.message).to.equal('"clientId" is required and "deviceType" is required and "eventApiEnv" is required and "eventTime" is required and "eventData" is required and "eventType" is required'); //eslint-disable-line done(); }) .catch(done); @@ -98,6 +101,7 @@ describe('## Basic Event APIs', () => { it('errors when no clientId sent', (done) => { const event = { deviceType: 'Browser', + eventApiEnv: 'production', eventTime: new Date().getTime(), eventType: 'login', eventData: { @@ -119,6 +123,7 @@ describe('## Basic Event APIs', () => { it('errors when no deviceType sent', (done) => { const event = { clientId: '45426562', + eventApiEnv: 'production', eventTime: new Date().getTime(), eventType: 'login', eventData: { @@ -140,6 +145,7 @@ describe('## Basic Event APIs', () => { it('errors when no eventData sent', (done) => { const event = { clientId: '45426562', + eventApiEnv: 'production', deviceType: 'API', eventTime: new Date().getTime(), eventType: 'login' @@ -159,6 +165,7 @@ describe('## Basic Event APIs', () => { it('errors when no eventType sent', (done) => { const event = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'Android', eventTime: new Date().getTime(), eventData: { @@ -180,6 +187,7 @@ describe('## Basic Event APIs', () => { it('errors when no eventTime sent', (done) => { const event = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'iOS', eventType: 'login', eventData: { @@ -197,4 +205,49 @@ describe('## Basic Event APIs', () => { }) .catch(done); }); + + it('errors when no eventApiEnv sent', (done) => { + const event = { + clientId: '1234567', + deviceType: 'iOS', + eventTime: new Date().getTime(), + eventType: 'login', + eventData: { + userId: '8675309' + } + } + request(app) + .post('/api/v1/event') + .send(event) + .expect(httpStatus.BAD_REQUEST) + .then((res) => { + expect(res.body).to.exist; //eslint-disable-line + expect(res.body.message).to.equal('"eventApiEnv" is required'); //eslint-disable-line + done(); + }) + .catch(done); + }); + + it('errors when eventApiEnv sent is wrong type', (done) => { + const event = { + clientId: '1234567', + deviceType: 'iOS', + eventApiEnv: 'testing', + eventTime: new Date().getTime(), + eventType: 'login', + eventData: { + userId: '8675309' + } + } + request(app) + .post('/api/v1/event') + .send(event) + .expect(httpStatus.BAD_REQUEST) + .then((res) => { + expect(res.body).to.exist; //eslint-disable-line + expect(res.body.message).to.equal('"eventApiEnv" must be one of [production, test]'); //eslint-disable-line + done(); + }) + .catch(done); + }); }); diff --git a/tests/server/events/register.event.js b/tests/server/events/register.event.js index 10487a1..8e2e880 100644 --- a/tests/server/events/register.event.js +++ b/tests/server/events/register.event.js @@ -9,6 +9,7 @@ describe('## Register Events', () => { it('sending valid register event is successful', (done) => { const event = { clientId: '1234567', + eventApiEnv: 'production', deviceType: 'Android', eventTime: new Date().getTime(), eventType: 'register',