From 506ba5824a2ca40a5660d341192ae5a59b7eb44b Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 28 Feb 2021 11:31:05 -0300 Subject: [PATCH 1/4] MongoDB first working version --- README.md | 4 + package.json | 3 +- src/Tag.ts | 8 + src/config/AgentConfig.ts | 6 + src/plugins/MongoDBPlugin.ts | 297 +++++++++++++++++++++++++++++++++++ src/plugins/MySQLPlugin.ts | 8 +- src/plugins/PgPlugin.ts | 4 +- 7 files changed, 323 insertions(+), 7 deletions(-) create mode 100644 src/plugins/MongoDBPlugin.ts diff --git a/README.md b/README.md index f710d45..6e38008 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,10 @@ Environment Variable | Description | Default | `SW_AGENT_LOGGING_LEVEL` | The logging level, could be one of `CRITICAL`, `FATAL`, `ERROR`, `WARN`(`WARNING`), `INFO`, `DEBUG` | `INFO` | | `SW_IGNORE_SUFFIX` | The suffices of endpoints that will be ignored (not traced), comma separated | `.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg` | | `SW_TRACE_IGNORE_PATH` | The paths of endpoints that will be ignored (not traced), comma separated | `` | +| `SW_SQL_TRACE_PARAMETERS` | If set to 'true' then SQL query parameters will be included | `false` | | `SW_SQL_PARAMETERS_MAX_LENGTH` | The maximum string length of SQL parameters to log | `512` | +| `SW_MONGO_TRACE_PARAMETERS` | If set to 'true' then mongodb query parameters will be included | `false` | +| `SW_MONGO_PARAMETERS_MAX_LENGTH` | The maximum string length of mongodb parameters to log | `512` | | `SW_AGENT_MAX_BUFFER_SIZE` | The maximum buffer size before sending the segment data to backend | `'1000'` | ## Supported Libraries @@ -71,6 +74,7 @@ Library | Plugin Name | [`axios`](https://github.com/axios/axios) | `axios` | | [`mysql`](https://github.com/mysqljs/mysql) | `mysql` | | [`pg`](https://github.com/brianc/node-postgres) | `pg` | +| [`mongodb`](https://github.com/mongodb/node-mongodb-native) | `mongodb` | ### Compatible Libraries diff --git a/package.json b/package.json index b037c70..2bfe29c 100644 --- a/package.json +++ b/package.json @@ -46,9 +46,10 @@ "@types/uuid": "^8.0.0", "axios": "^0.21.0", "express": "^4.17.1", - "grpc_tools_node_protoc_ts": "^4.0.0", "grpc-tools": "^1.10.0", + "grpc_tools_node_protoc_ts": "^4.0.0", "jest": "^26.6.3", + "mongodb": "^3.6.4", "mysql": "^2.18.1", "pg": "^8.5.1", "prettier": "^2.0.5", diff --git a/src/Tag.ts b/src/Tag.ts index 305107c..06daf24 100644 --- a/src/Tag.ts +++ b/src/Tag.ts @@ -32,6 +32,7 @@ export default { dbInstanceKey: 'db.instance', dbStatementKey: 'db.statement', dbSqlParametersKey: 'db.sql.parameters', + dbMongoParametersKey: 'db.mongo.parameters', httpStatusCode(val: string | number | undefined): Tag { return { @@ -89,4 +90,11 @@ export default { val: `${val}`, } as Tag; }, + dbMongoParameters(val: string | undefined): Tag { + return { + key: this.dbMongoParametersKey, + overridable: false, + val: `${val}`, + } as Tag; + }, }; diff --git a/src/config/AgentConfig.ts b/src/config/AgentConfig.ts index 94d5f26..349e13a 100644 --- a/src/config/AgentConfig.ts +++ b/src/config/AgentConfig.ts @@ -27,7 +27,10 @@ export type AgentConfig = { maxBufferSize?: number; ignoreSuffix?: string; traceIgnorePath?: string; + sql_trace_parameters?: boolean; sql_parameters_max_length?: number; + mongo_trace_parameters?: boolean; + mongo_parameters_max_length?: number; // the following is internal state computed from config values reIgnoreOperation?: RegExp; }; @@ -60,6 +63,9 @@ export default { Number.parseInt(process.env.SW_AGENT_MAX_BUFFER_SIZE as string, 10) : 1000, ignoreSuffix: process.env.SW_IGNORE_SUFFIX ?? '.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg', traceIgnorePath: process.env.SW_TRACE_IGNORE_PATH || '', + sql_trace_parameters: (process.env.SW_SQL_TRACE_PARAMETERS || '').toLowerCase() === 'true', sql_parameters_max_length: Math.trunc(Math.max(0, Number(process.env.SW_SQL_PARAMETERS_MAX_LENGTH))) || 512, + mongo_trace_parameters: (process.env.SW_MONGO_TRACE_PARAMETERS || '').toLowerCase() === 'true', + mongo_parameters_max_length: Math.trunc(Math.max(0, Number(process.env.SW_MONGO_PARAMETERS_MAX_LENGTH))) || 512, reIgnoreOperation: RegExp(''), // temporary placeholder so Typescript doesn't throw a fit }; diff --git a/src/plugins/MongoDBPlugin.ts b/src/plugins/MongoDBPlugin.ts new file mode 100644 index 0000000..78c6a35 --- /dev/null +++ b/src/plugins/MongoDBPlugin.ts @@ -0,0 +1,297 @@ +/*! + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import SwPlugin from '../core/SwPlugin'; +import ContextManager from '../trace/context/ContextManager'; +import { Component } from '../trace/Component'; +import Tag from '../Tag'; +import { SpanLayer } from '../proto/language-agent/Tracing_pb'; +import PluginInstaller from '../core/PluginInstaller'; +import agentConfig from '../config/AgentConfig'; + +class MongoDBPlugin implements SwPlugin { + readonly module = 'mongodb'; + readonly versions = '*'; + + Cursor: any; + + // Experimental method to determine proper end time of cursor DB operation, we stop the span when the cursor is closed. + // Problematic because other exit spans may be created during processing, for this reason we do not .resync() this + // span to the span list until it is closed. If the cursor is never closed then the span will not be sent. + + maybeHookCursor(span: any, cursor: any): boolean { + if (!(cursor instanceof this.Cursor)) + return false; + + cursor.on('error', (err: any) => { + span.resync(); // this may precede 'close' .resync() but its fine + span.error(err); + span.stop(); + }); + + cursor.on('close', () => { + span.resync(); // cursor does not .resync() until it is closed because maybe other exit spans will be opened during processing + span.stop(); + }); + + return true; + } + + install(installer: PluginInstaller): void { + const plugin = this; + const Collection = installer.require('mongodb/lib/collection'); + this.Cursor = installer.require('mongodb/lib/cursor'); + + const wrapCallback = (span: any, args: any[], idx: number): boolean => { + const callback = args.length > idx && typeof args[idx = args.length - 1] === 'function' ? args[idx] : null; + + if (!callback) + return false; + + args[idx] = function(this: any, error: any, result: any) { + if (error || !plugin.maybeHookCursor(span, result)) { + span.resync(); + + if (error) + span.error(error); + + span.stop(); + } + + return callback.call(this, error, result); + } + + return true; + }; + + const stringify = (params: any) => { + if (params === undefined) + return ''; + + let str = JSON.stringify(params); + + if (str.length > agentConfig.mongo_parameters_max_length) + str = str.slice(0, agentConfig.mongo_parameters_max_length) + ' ...'; + + return str; + } + + const insertFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [doc(s), options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}()`)); + + if (agentConfig.mongo_trace_parameters) + span.tag(Tag.dbMongoParameters(stringify(args[0]))); + + return wrapCallback(span, args, 1); + }; + + const deleteFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [filter, options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}(${stringify(args[0])})`)); + + return wrapCallback(span, args, 1); + }; + + const updateFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [filter, update, options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}(${stringify(args[0])})`)); + + if (agentConfig.mongo_trace_parameters) + span.tag(Tag.dbMongoParameters(stringify(args[1]))); + + return wrapCallback(span, args, 2); + }; + + const findOneFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [query, options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}(${typeof args[0] !== 'function' ? stringify(args[0]) : ''})`)); + + return wrapCallback(span, args, 0); + }; + + const findAndRemoveFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [query, sort, options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}(${stringify(args[0])}${typeof args[1] !== 'function' && args[1] !== undefined ? ', ' + stringify(args[1]) : ''})`)); + + return wrapCallback(span, args, 1); + }; + + const findAndModifyFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [query, sort, doc, options, callback] + let params = stringify(args[0]); + + if (typeof args[1] !== 'function' && args[1] !== undefined) { + params += ', ' + stringify(args[1]); + + if (typeof args[2] !== 'function' && args[2] !== undefined) { + if (agentConfig.mongo_trace_parameters) + span.tag(Tag.dbMongoParameters(stringify(args[2]))); + } + } + + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}(${params})`)); + + return wrapCallback(span, args, 1); + }; + + const mapReduceFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [map, reduce, options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}(${args[0]}, ${args[1]})`)); + + return wrapCallback(span, args, 2); + }; + + const dropFunc = function(this: any, operation: string, span: any, args: any[]): boolean { // args = [options, callback] + span.tag(Tag.dbStatement(`${this.s.namespace.collection}.${operation}()`)); + + return wrapCallback(span, args, 0); + }; + + this.interceptOperation(Collection, 'insert', insertFunc); + this.interceptOperation(Collection, 'insertOne', insertFunc); + this.interceptOperation(Collection, 'insertMany', insertFunc); + this.interceptOperation(Collection, 'save', insertFunc); + this.interceptOperation(Collection, 'deleteOne', deleteFunc); + this.interceptOperation(Collection, 'deleteMany', deleteFunc); + this.interceptOperation(Collection, 'remove', deleteFunc); + this.interceptOperation(Collection, 'removeOne', deleteFunc); + this.interceptOperation(Collection, 'removeMany', deleteFunc); + this.interceptOperation(Collection, 'update', updateFunc); + this.interceptOperation(Collection, 'updateOne', updateFunc); + this.interceptOperation(Collection, 'updateMany', updateFunc); + this.interceptOperation(Collection, 'replaceOne', updateFunc); + this.interceptOperation(Collection, 'find', findOneFunc); // cursor + this.interceptOperation(Collection, 'findOne', findOneFunc); + this.interceptOperation(Collection, 'findOneAndDelete', deleteFunc); + this.interceptOperation(Collection, 'findOneAndReplace', updateFunc); + this.interceptOperation(Collection, 'findOneAndUpdate', updateFunc); + this.interceptOperation(Collection, 'findAndRemove', findAndRemoveFunc); + this.interceptOperation(Collection, 'findAndModify', findAndModifyFunc); + + this.interceptOperation(Collection, 'bulkWrite', insertFunc); + this.interceptOperation(Collection, 'mapReduce', mapReduceFunc); + this.interceptOperation(Collection, 'aggregate', deleteFunc); // cursor + this.interceptOperation(Collection, 'distinct', findAndRemoveFunc); + this.interceptOperation(Collection, 'count', findOneFunc); + this.interceptOperation(Collection, 'estimatedDocumentCount', dropFunc); + this.interceptOperation(Collection, 'countDocuments', findOneFunc); + + this.interceptOperation(Collection, 'rename', deleteFunc); + this.interceptOperation(Collection, 'drop', dropFunc); + + + // TODO? + + // createIndex + // createIndexes + // dropIndex + // dropIndexes + // dropAllIndexes + // ensureIndex + // indexExists + // indexInformation + // indexes + // listIndexes + // reIndex + + // stats + // options + // isCapped + // initializeUnorderedBulkOp + // initializeOrderedBulkOp + // watch + + + // NODO: + + // group + // parallelCollectionScan + // geoHaystackSearch + } + + interceptOperation(Collection: any, operation: string, operationFunc: any): void { + const plugin = this; + const _original = Collection.prototype[operation]; + + if (!_original) + return; + + Collection.prototype[operation] = function(...args: any[]) { + let ret: any; + let host: string; + + try { + host = this.s.db.serverConfig.s.options.servers.map((s: any) => `${s.host}:${s.port}`).join(','); // will this work for non-NativeTopology? + } catch { + host = '???'; + } + + const span = ContextManager.current.newExitSpan('/' + this.s.namespace.db, host).start(); // or this.s.db.databaseName + + try { + span.component = Component.MONGODB; + span.layer = SpanLayer.DATABASE; + span.peer = host; + + span.tag(Tag.dbType('MongoDB')); + span.tag(Tag.dbInstance(`${this.s.namespace.db}`)); + + const hasCB = operationFunc.call(this, operation, span, args); + + ret = _original.apply(this, args); + + if (!hasCB) { + if (plugin.maybeHookCursor(span, ret)) { + // NOOP + + } else if (!ret || typeof ret.then !== 'function') { // generic Promise check + span.stop(); // no callback passed in and no Promise or Cursor returned, play it safe + + return ret; + + } else { + ret = ret.then( + (res: any) => { + span.resync(); + span.stop(); + + return res; + }, + + (err: any) => { + span.resync(); + span.error(err); + span.stop(); + + return Promise.reject(err); + } + ); + } + } + + } catch (e) { + span.error(e); + span.stop(); + + throw e; + } + + span.async(); + + return ret; + }; + } +} + +// noinspection JSUnusedGlobalSymbols +export default new MongoDBPlugin(); diff --git a/src/plugins/MySQLPlugin.ts b/src/plugins/MySQLPlugin.ts index 8e191fc..35ebee0 100644 --- a/src/plugins/MySQLPlugin.ts +++ b/src/plugins/MySQLPlugin.ts @@ -23,7 +23,7 @@ import { Component } from '../trace/Component'; import Tag from '../Tag'; import { SpanLayer } from '../proto/language-agent/Tracing_pb'; import PluginInstaller from '../core/PluginInstaller'; -import config from '../config/AgentConfig'; +import agentConfig from '../config/AgentConfig'; class MySQLPlugin implements SwPlugin { readonly module = 'mysql'; @@ -109,11 +109,11 @@ class MySQLPlugin implements SwPlugin { span.tag(Tag.dbStatement(`${_sql}`)); - if (_values) { + if (agentConfig.sql_trace_parameters && _values) { let vals = _values.map((v: any) => v === undefined ? 'undefined' : JSON.stringify(v)).join(', '); - if (vals.length > config.sql_parameters_max_length) - vals = vals.splice(0, config.sql_parameters_max_length); + if (vals.length > agentConfig.sql_parameters_max_length) + vals = vals.slice(0, agentConfig.sql_parameters_max_length) + ' ...'; span.tag(Tag.dbSqlParameters(`[${vals}]`)); } diff --git a/src/plugins/PgPlugin.ts b/src/plugins/PgPlugin.ts index e8f4435..9164f85 100644 --- a/src/plugins/PgPlugin.ts +++ b/src/plugins/PgPlugin.ts @@ -84,11 +84,11 @@ class MySQLPlugin implements SwPlugin { span.tag(Tag.dbStatement(`${_sql}`)); - if (_values) { + if (agentConfig.sql_trace_parameters && _values) { let vals = _values.map((v: any) => v === undefined ? 'undefined' : JSON.stringify(v)).join(', '); if (vals.length > agentConfig.sql_parameters_max_length) - vals = vals.splice(0, agentConfig.sql_parameters_max_length); + vals = vals.slice(0, agentConfig.sql_parameters_max_length) + ' ...'; span.tag(Tag.dbSqlParameters(`[${vals}]`)); } From 3011cf8d725470c39a60dd04145234944a8c069a Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 28 Feb 2021 11:54:56 -0300 Subject: [PATCH 2/4] Add support for pg-cursor --- README.md | 1 + src/plugins/PgPlugin.ts | 51 +++++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6e38008..e6b4a6e 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Library | Plugin Name | [`axios`](https://github.com/axios/axios) | `axios` | | [`mysql`](https://github.com/mysqljs/mysql) | `mysql` | | [`pg`](https://github.com/brianc/node-postgres) | `pg` | +| [`pg-cursor`](https://github.com/brianc/node-postgres) | `pg-cursor` | | [`mongodb`](https://github.com/mongodb/node-mongodb-native) | `mongodb` | ### Compatible Libraries diff --git a/src/plugins/PgPlugin.ts b/src/plugins/PgPlugin.ts index 9164f85..0d8f4e9 100644 --- a/src/plugins/PgPlugin.ts +++ b/src/plugins/PgPlugin.ts @@ -31,6 +31,13 @@ class MySQLPlugin implements SwPlugin { install(installer: PluginInstaller): void { const Client = installer.require('pg/lib/client'); + + let Cursor: any; + + try { + Cursor = installer.require('pg-cursor'); + } catch { /* Linter food */ } + const _query = Client.prototype.query; Client.prototype.query = function(config: any, values: any, callback: any) { @@ -76,7 +83,7 @@ class MySQLPlugin implements SwPlugin { if (typeof values === 'function') values = wrapCallback(values); - else + else if (_values !== undefined) _values = values; if (typeof callback === 'function') @@ -95,23 +102,37 @@ class MySQLPlugin implements SwPlugin { query = _query.call(this, config, values, callback); - if (query && typeof query.then === 'function') { // generic Promise check - query = query.then( - (res: any) => { - span.resync(); - span.stop(); - - return res; - }, - - (err: any) => { - span.resync(); + if (query) { + if (Cursor && query instanceof Cursor) { + query.on('error', (err: any) => { + span.resync(); // this may precede 'end' .resync() but its fine span.error(err); span.stop(); + }); - return Promise.reject(err); - } - ); + query.on('end', () => { + span.resync(); // cursor does not .resync() until it is closed because maybe other exit spans will be opened during processing + span.stop(); + }); + + } else if (typeof query.then === 'function') { // generic Promise check + query = query.then( + (res: any) => { + span.resync(); + span.stop(); + + return res; + }, + + (err: any) => { + span.resync(); + span.error(err); + span.stop(); + + return Promise.reject(err); + } + ); + } // else we assume there was a callback } } catch (e) { From ed3d8340f6117209fcdf700137bd3ca10a52c1f8 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Sun, 28 Feb 2021 15:42:33 -0300 Subject: [PATCH 3/4] test --- tests/plugins/mongodb/client.ts | 40 ++++++++++ tests/plugins/mongodb/docker-compose.yml | 90 ++++++++++++++++++++++ tests/plugins/mongodb/expected.data.yaml | 98 ++++++++++++++++++++++++ tests/plugins/mongodb/init/init.js | 1 + tests/plugins/mongodb/server.ts | 55 +++++++++++++ tests/plugins/mongodb/test.ts | 57 ++++++++++++++ 6 files changed, 341 insertions(+) create mode 100644 tests/plugins/mongodb/client.ts create mode 100644 tests/plugins/mongodb/docker-compose.yml create mode 100644 tests/plugins/mongodb/expected.data.yaml create mode 100644 tests/plugins/mongodb/init/init.js create mode 100644 tests/plugins/mongodb/server.ts create mode 100644 tests/plugins/mongodb/test.ts diff --git a/tests/plugins/mongodb/client.ts b/tests/plugins/mongodb/client.ts new file mode 100644 index 0000000..25ff2b3 --- /dev/null +++ b/tests/plugins/mongodb/client.ts @@ -0,0 +1,40 @@ +/*! + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import * as http from 'http'; +import agent from '../../../src'; + +process.env.SW_AGENT_LOGGING_LEVEL = 'ERROR'; + +agent.start({ + serviceName: 'client', + maxBufferSize: 1000, +}) + +const server = http.createServer((req, res) => { + http + .request(`http://${process.env.SERVER || 'localhost:5000'}${req.url}`, (r) => { + let data = ''; + r.on('data', (chunk) => (data += chunk)); + r.on('end', () => res.end(data)); + }) + .end(); +}); + +server.listen(5001, () => console.info('Listening on port 5001...')); diff --git a/tests/plugins/mongodb/docker-compose.yml b/tests/plugins/mongodb/docker-compose.yml new file mode 100644 index 0000000..305351b --- /dev/null +++ b/tests/plugins/mongodb/docker-compose.yml @@ -0,0 +1,90 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "2.1" + +services: + collector: + extends: + file: ../common/base-compose.yml + service: collector + networks: + - traveling-light + + mongo: + container_name: mongo + environment: + MONGO_INITDB_ROOT_USERNAME: "root" + MONGO_INITDB_ROOT_PASSWORD: "root" + MONGO_INITDB_DATABASE: "admin" + ports: + - 27017:27017 + volumes: + - ./init:/docker-entrypoint-initdb.d + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/27017"] + interval: 5s + timeout: 60s + retries: 120 + image: "mongo:latest" + networks: + - traveling-light + + server: + extends: + file: ../common/base-compose.yml + service: agent + ports: + - 5000:5000 + environment: + MONGO_HOST: mongo + volumes: + - .:/app/tests/plugins/pg + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5000"] + interval: 5s + timeout: 60s + retries: 120 + entrypoint: + ["bash", "-c", "npx ts-node /app/tests/plugins/pg/server.ts"] + depends_on: + collector: + condition: service_healthy + mongo: + condition: service_healthy + + client: + extends: + file: ../common/base-compose.yml + service: agent + ports: + - 5001:5001 + environment: + SERVER: server:5000 + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5001"] + interval: 5s + timeout: 60s + retries: 120 + entrypoint: + ["bash", "-c", "npx ts-node /app/tests/plugins/pg/client.ts"] + depends_on: + server: + condition: service_healthy + +networks: + traveling-light: diff --git a/tests/plugins/mongodb/expected.data.yaml b/tests/plugins/mongodb/expected.data.yaml new file mode 100644 index 0000000..7cbac82 --- /dev/null +++ b/tests/plugins/mongodb/expected.data.yaml @@ -0,0 +1,98 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +segmentItems: + - serviceName: server + segmentSize: 1 + segments: + - segmentId: not null + spans: + - operationName: /mongo + operationId: 0 + parentSpanId: -1 + spanId: 0 + spanLayer: Http + startTime: gt 0 + endTime: gt 0 + componentId: 49 + spanType: Entry + peer: not null + skipAnalysis: false + tags: + - { key: http.url, value: 'http://server:5000/mongo' } + - { key: http.method, value: GET } + - { key: http.status.code, value: '200' } + refs: + - parentEndpoint: "" + networkAddress: server:5000 + refType: CrossProcess + parentSpanId: 1 + parentTraceSegmentId: not null + parentServiceInstance: not null + parentService: client + traceId: not null + - operationName: /admin + operationId: 0 + parentSpanId: 0 + spanId: 1 + spanLayer: Database + startTime: gt 0 + endTime: gt 0 + componentId: 9 + spanType: Exit + peer: mongo:27017 + skipAnalysis: false + tags: + - { key: db.type, value: MongoDB } + - { key: db.instance, value: admin } + - { key: db.statement, value: docs.findOne() } + - serviceName: client + segmentSize: 1 + segments: + - segmentId: not null + spans: + - operationName: /mongo + operationId: 0 + parentSpanId: -1 + spanId: 0 + spanLayer: Http + startTime: gt 0 + endTime: gt 0 + componentId: 49 + spanType: Entry + peer: not null + skipAnalysis: false + tags: + - { key: http.url, value: 'http://localhost:5001/mongo' } + - { key: http.method, value: GET } + - { key: http.status.code, value: '200' } + - operationName: /mongo + operationId: 0 + parentSpanId: 0 + spanId: 1 + spanLayer: Http + startTime: gt 0 + endTime: gt 0 + componentId: 2 + spanType: Exit + peer: server:5000 + skipAnalysis: false + tags: + - { key: http.url, value: 'server:5000/mongo' } + - { key: http.method, value: GET } + - { key: http.status.code, value: '200' } + - { key: http.status.msg, value: OK } diff --git a/tests/plugins/mongodb/init/init.js b/tests/plugins/mongodb/init/init.js new file mode 100644 index 0000000..48ac0d3 --- /dev/null +++ b/tests/plugins/mongodb/init/init.js @@ -0,0 +1 @@ +db.createCollection('docs'); diff --git a/tests/plugins/mongodb/server.ts b/tests/plugins/mongodb/server.ts new file mode 100644 index 0000000..5870227 --- /dev/null +++ b/tests/plugins/mongodb/server.ts @@ -0,0 +1,55 @@ +/*! + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import * as http from 'http'; +import {MongoClient} from 'mongodb'; +import agent from '../../../src'; + +process.env.SW_AGENT_LOGGING_LEVEL = 'ERROR'; + +agent.start({ + serviceName: 'server', + maxBufferSize: 1000, +}); + +const server = http.createServer(async (req, res) => { + await new Promise((resolve, reject) => { + MongoClient.connect(`mongodb://root:root@${process.env.MONGO_HOST}:27017`, {useUnifiedTopology: true}, function(err: any, client: any) { + if (err) { + res.end(`${err}`); + resolve(null); + } else { + client.db('admin').collection('docs').findOne().then( + (resDB: any) => { + res.end(`${resDB}`); + resolve(null); + client.close(); + }, + (err: any) => { + res.end(`${err}`); + resolve(null); + client.close(); + }, + ); + } + }); + }); +}); + +server.listen(5000, () => console.info('Listening on port 5000...')); diff --git a/tests/plugins/mongodb/test.ts b/tests/plugins/mongodb/test.ts new file mode 100644 index 0000000..787e57d --- /dev/null +++ b/tests/plugins/mongodb/test.ts @@ -0,0 +1,57 @@ +/*! + * + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import * as path from 'path'; +import { DockerComposeEnvironment, StartedDockerComposeEnvironment, Wait } from 'testcontainers'; +import axios from 'axios'; +import waitForExpect from 'wait-for-expect'; +import { promises as fs } from 'fs'; + +const rootDir = path.resolve(__dirname); + +describe('plugin tests', () => { + let compose: StartedDockerComposeEnvironment; + + beforeAll(async () => { + compose = await new DockerComposeEnvironment(rootDir, 'docker-compose.yml') + .withWaitStrategy('client', Wait.forHealthCheck()) + .withWaitStrategy('mongo', Wait.forHealthCheck()) + .up(); + }); + + afterAll(async () => { + await compose.down(); + }); + + it(__filename, async () => { + await waitForExpect(async () => expect((await axios.get('http://localhost:5001/mongo')).status).toBe(200)); + + const expectedData = await fs.readFile(path.join(rootDir, 'expected.data.yaml'), 'utf8'); + + try { + await waitForExpect(async () => + expect((await axios.post('http://localhost:12800/dataValidate', expectedData)).status).toBe(200), + ); + } catch (e) { + const actualData = (await axios.get('http://localhost:12800/receiveData')).data; + console.info({ actualData }); + throw e; + } + }); +}); From b036820fd0d0a3fa77c5c2d3ec4ec2c59d49eae2 Mon Sep 17 00:00:00 2001 From: Tomasz Pytel Date: Mon, 1 Mar 2021 08:51:05 -0300 Subject: [PATCH 4/4] mongodb recursion fixed --- src/plugins/MongoDBPlugin.ts | 9 ++++++- tests/plugins/mongodb/expected.data.yaml | 31 ++++++++++++------------ 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/src/plugins/MongoDBPlugin.ts b/src/plugins/MongoDBPlugin.ts index 78c6a35..60b4534 100644 --- a/src/plugins/MongoDBPlugin.ts +++ b/src/plugins/MongoDBPlugin.ts @@ -20,6 +20,7 @@ import SwPlugin from '../core/SwPlugin'; import ContextManager from '../trace/context/ContextManager'; import { Component } from '../trace/Component'; +import ExitSpan from '../trace/span/ExitSpan'; import Tag from '../Tag'; import { SpanLayer } from '../proto/language-agent/Tracing_pb'; import PluginInstaller from '../core/PluginInstaller'; @@ -227,6 +228,12 @@ class MongoDBPlugin implements SwPlugin { return; Collection.prototype[operation] = function(...args: any[]) { + const spans = ContextManager.spans; + let span = spans[spans.length - 1]; + + if (span && span.component === Component.MONGODB && span instanceof ExitSpan) // mongodb has called into itself internally + return _original.apply(this, args); + let ret: any; let host: string; @@ -236,7 +243,7 @@ class MongoDBPlugin implements SwPlugin { host = '???'; } - const span = ContextManager.current.newExitSpan('/' + this.s.namespace.db, host).start(); // or this.s.db.databaseName + span = ContextManager.current.newExitSpan('/' + this.s.namespace.db, host).start(); // or this.s.db.databaseName try { span.component = Component.MONGODB; diff --git a/tests/plugins/mongodb/expected.data.yaml b/tests/plugins/mongodb/expected.data.yaml index 7cbac82..37061e2 100644 --- a/tests/plugins/mongodb/expected.data.yaml +++ b/tests/plugins/mongodb/expected.data.yaml @@ -21,6 +21,21 @@ segmentItems: segments: - segmentId: not null spans: + - operationName: /admin + operationId: 0 + parentSpanId: 0 + spanId: 1 + spanLayer: Database + startTime: gt 0 + endTime: gt 0 + componentId: 9 + spanType: Exit + peer: mongo:27017 + skipAnalysis: false + tags: + - { key: db.type, value: MongoDB } + - { key: db.instance, value: admin } + - { key: db.statement, value: docs.findOne() } - operationName: /mongo operationId: 0 parentSpanId: -1 @@ -36,6 +51,7 @@ segmentItems: - { key: http.url, value: 'http://server:5000/mongo' } - { key: http.method, value: GET } - { key: http.status.code, value: '200' } + - { key: http.status.msg, value: OK } refs: - parentEndpoint: "" networkAddress: server:5000 @@ -45,21 +61,6 @@ segmentItems: parentServiceInstance: not null parentService: client traceId: not null - - operationName: /admin - operationId: 0 - parentSpanId: 0 - spanId: 1 - spanLayer: Database - startTime: gt 0 - endTime: gt 0 - componentId: 9 - spanType: Exit - peer: mongo:27017 - skipAnalysis: false - tags: - - { key: db.type, value: MongoDB } - - { key: db.instance, value: admin } - - { key: db.statement, value: docs.findOne() } - serviceName: client segmentSize: 1 segments: