From 1990c7b5bf62d7d03df70b79e18f2a076c9174f6 Mon Sep 17 00:00:00 2001 From: Douglas McConnachie Date: Fri, 13 Mar 2020 12:46:21 +0000 Subject: [PATCH] feat(rest): add openapi schema consolidation Add openapi schema enhancer to rest server. Consolidates openapi schema, by creating references to schema to reduce duplication. Can be disabled by setting rest option openApiSpec.consolidate to false. feat(openapi-v3): getEnhancerByName accept generic parameter Signed-off-by: Douglas McConnachie --- packages/openapi-v3/package-lock.json | 6 + packages/openapi-v3/package.json | 1 + .../src/enhancers/spec-enhancer.service.ts | 6 +- packages/rest/package-lock.json | 23 ++ packages/rest/package.json | 2 + .../integration/rest.server.integration.ts | 160 +++++++++- .../consolidate.spec.extension.unit.ts | 281 ++++++++++++++++++ .../rest.server.open-api-spec.unit.ts | 6 + packages/rest/src/rest.component.ts | 2 + packages/rest/src/rest.server.ts | 6 + .../consolidate.spec-enhancer.ts | 182 ++++++++++++ 11 files changed, 672 insertions(+), 3 deletions(-) create mode 100644 packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts create mode 100644 packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts diff --git a/packages/openapi-v3/package-lock.json b/packages/openapi-v3/package-lock.json index a6c4e6ddd852..bd0efbb19c74 100644 --- a/packages/openapi-v3/package-lock.json +++ b/packages/openapi-v3/package-lock.json @@ -19,6 +19,12 @@ "http-status": "*" } }, + "@types/json-merge-patch": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/json-merge-patch/-/json-merge-patch-0.0.4.tgz", + "integrity": "sha1-pSgtqWkKgSpiEoo0cIr0dqMI2UE=", + "dev": true + }, "@types/lodash": { "version": "4.14.150", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.150.tgz", diff --git a/packages/openapi-v3/package.json b/packages/openapi-v3/package.json index d79166c94109..e121d8d1a0aa 100644 --- a/packages/openapi-v3/package.json +++ b/packages/openapi-v3/package.json @@ -23,6 +23,7 @@ "@loopback/testlab": "^3.0.1", "@types/debug": "^4.1.5", "@types/http-status": "^1.1.2", + "@types/json-merge-patch": "0.0.4", "@types/lodash": "^4.14.150", "@types/node": "^10.17.20" }, diff --git a/packages/openapi-v3/src/enhancers/spec-enhancer.service.ts b/packages/openapi-v3/src/enhancers/spec-enhancer.service.ts index c39a9f883a01..9b2b5543d959 100644 --- a/packages/openapi-v3/src/enhancers/spec-enhancer.service.ts +++ b/packages/openapi-v3/src/enhancers/spec-enhancer.service.ts @@ -71,10 +71,12 @@ export class OASEnhancerService { * Find an enhancer by its name * @param name The name of the enhancer you want to find */ - async getEnhancerByName(name: string): Promise { + async getEnhancerByName( + name: string, + ): Promise { // Get the latest list of enhancers const enhancers = await this.getEnhancers(); - return enhancers.find(e => e.name === name); + return enhancers.find(e => e.name === name) as T | undefined; } /** diff --git a/packages/rest/package-lock.json b/packages/rest/package-lock.json index ce5c6df7faed..919c6ef47689 100644 --- a/packages/rest/package-lock.json +++ b/packages/rest/package-lock.json @@ -95,6 +95,21 @@ "integrity": "sha512-otRe77JNNWzoVGLKw8TCspKswRoQToys4tuL6XYVBFxjgeM0RUrx7m3jkaTdxILxeGry3zM8mGYkGXMeQ02guA==", "dev": true }, + "@types/json-schema": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.4.tgz", + "integrity": "sha512-8+KAKzEvSUdeo+kmqnKrqgeE+LcA0tjYWFY7RPProVYwnqDjukzO+3b6dLD56rYX5TdWejnEOLJYOIeh4CXKuA==", + "dev": true + }, + "@types/json-schema-compare": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/@types/json-schema-compare/-/json-schema-compare-0.2.0.tgz", + "integrity": "sha512-TtCXQjsCQi+fcandEbzDJhqyztpVM9c5mtGuk7Hf8yQsdaBpfjEkOicfydAEWB684wGCzUrV5ttvt9hCyDCoxA==", + "dev": true, + "requires": { + "@types/json-schema": "*" + } + }, "@types/lodash": { "version": "4.14.150", "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.150.tgz", @@ -845,6 +860,14 @@ "xmlcreate": "^2.0.3" } }, + "json-schema-compare": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/json-schema-compare/-/json-schema-compare-0.2.2.tgz", + "integrity": "sha512-c4WYmDKyJXhs7WWvAWm3uIYnfyWFoIp+JEoX34rctVvEkMYCPGhXtvmFFXiffBbxfZsvQ0RNnV5H7GvDF5HCqQ==", + "requires": { + "lodash": "^4.17.4" + } + }, "json-schema-traverse": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", diff --git a/packages/rest/package.json b/packages/rest/package.json index 298a91b13fb7..93f3c67da91d 100644 --- a/packages/rest/package.json +++ b/packages/rest/package.json @@ -40,6 +40,7 @@ "debug": "^4.1.1", "express": "^4.17.1", "http-errors": "^1.7.3", + "json-schema-compare": "^0.2.2", "js-yaml": "^3.13.1", "lodash": "^4.17.15", "on-finished": "^2.3.0", @@ -57,6 +58,7 @@ "@loopback/repository": "^2.1.1", "@loopback/testlab": "^3.0.1", "@types/debug": "^4.1.5", + "@types/json-schema-compare": "^0.2.0", "@types/js-yaml": "^3.12.3", "@types/lodash": "^4.14.150", "@types/multer": "^1.4.3", diff --git a/packages/rest/src/__tests__/integration/rest.server.integration.ts b/packages/rest/src/__tests__/integration/rest.server.integration.ts index c6ae6da178ab..4c58b422fb6a 100644 --- a/packages/rest/src/__tests__/integration/rest.server.integration.ts +++ b/packages/rest/src/__tests__/integration/rest.server.integration.ts @@ -4,7 +4,11 @@ // License text available at https://opensource.org/licenses/MIT import {Application} from '@loopback/core'; -import {anOpenApiSpec, anOperationSpec} from '@loopback/openapi-spec-builder'; +import { + aComponentsSpec, + anOpenApiSpec, + anOperationSpec, +} from '@loopback/openapi-spec-builder'; import { createClientForHandler, createRestAppClient, @@ -875,6 +879,160 @@ paths: await server.stop(); }); + it('disables consolidator if openApiSpec.consolidate option is set to false', async () => { + const options = {openApiSpec: {consolidate: false}}; + const server = await givenAServer({rest: options}); + + const EXPECTED_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + server.route('get', '/', EXPECTED_SPEC.paths['/'].get, () => {}); + + await server.start(); + const spec = await server.getApiSpec(); + expect(spec).to.eql(EXPECTED_SPEC); + await server.stop(); + }); + + it('runs consolidator if openApiSpec.consolidate option is set to true', async () => { + const options = {openApiSpec: {consolidate: true}}; + const server = await givenAServer({rest: options}); + + const EXPECTED_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withComponents( + aComponentsSpec().withSchema('loopback.example', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + server.route( + 'get', + '/', + anOperationSpec() + .withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }) + .build(), + () => {}, + ); + + await server.start(); + const spec = await server.getApiSpec(); + expect(spec).to.eql(EXPECTED_SPEC); + await server.stop(); + }); + + it('runs consolidator if openApiSpec.consolidate option is undefined', async () => { + const options = {openApiSpec: {consolidate: undefined}}; + const server = await givenAServer({rest: options}); + + const EXPECTED_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withComponents( + aComponentsSpec().withSchema('loopback.example', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + server.route( + 'get', + '/', + anOperationSpec() + .withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }) + .build(), + () => {}, + ); + + await server.start(); + const spec = await server.getApiSpec(); + expect(spec).to.eql(EXPECTED_SPEC); + await server.stop(); + }); + it('registers controller routes under routes.*', async () => { const server = await givenAServer(); server.controller(DummyController); diff --git a/packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts b/packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts new file mode 100644 index 000000000000..3cba032c8437 --- /dev/null +++ b/packages/rest/src/__tests__/unit/rest.server/consolidate.spec.extension.unit.ts @@ -0,0 +1,281 @@ +// Copyright IBM Corp. 2020. All Rights Reserved. +// Node module: @loopback/openapi-v3 +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +import { + aComponentsSpec, + anOpenApiSpec, + anOperationSpec, +} from '@loopback/openapi-spec-builder'; +import {expect} from '@loopback/testlab'; +import {ConsolidationEnhancer} from '../../../spec-enhancers/consolidate.spec-enhancer'; + +const consolidationEnhancer = new ConsolidationEnhancer(); + +describe('consolidateSchemaObjects', () => { + it('moves schema with title to component.schemas, replaces with reference', () => { + const INPUT_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + const EXPECTED_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withComponents( + aComponentsSpec().withSchema('loopback.example', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(EXPECTED_SPEC); + }); + + it('ignores schema without title property', () => { + const INPUT_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(INPUT_SPEC); + }); + + it('avoids naming collision', () => { + const INPUT_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .withComponents( + aComponentsSpec().withSchema('loopback.example', { + title: 'Different loopback.example exists', + properties: { + testDiff: { + type: 'string', + }, + }, + }), + ) + .build(); + + const EXPECTED_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example1', + }, + }, + }, + }), + ) + .withComponents( + aComponentsSpec() + .withSchema('loopback.example', { + title: 'Different loopback.example exists', + properties: { + testDiff: { + type: 'string', + }, + }, + }) + .withSchema('loopback.example1', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(EXPECTED_SPEC); + }); + + it('consolidates same schema in multiple locations', () => { + const INPUT_SPEC = anOpenApiSpec() + .withOperation( + 'get', + // first time has 'loopback.example' + '/path1', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .withOperation( + 'get', + // second time has 'loopback.example' + '/path2', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + const EXPECTED_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/path1', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withOperation( + 'get', + '/path2', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + $ref: '#/components/schemas/loopback.example', + }, + }, + }, + }), + ) + .withComponents( + aComponentsSpec().withSchema('loopback.example', { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(EXPECTED_SPEC); + }); + + it('obeys disabled option when set to true', () => { + consolidationEnhancer.disabled = true; + const INPUT_SPEC = anOpenApiSpec() + .withOperation( + 'get', + '/', + anOperationSpec().withResponse(200, { + description: 'Example', + content: { + 'application/json': { + schema: { + title: 'loopback.example', + properties: { + test: { + type: 'string', + }, + }, + }, + }, + }, + }), + ) + .build(); + + expect(consolidationEnhancer.modifySpec(INPUT_SPEC)).to.eql(INPUT_SPEC); + }); +}); diff --git a/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts b/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts index ad7930ef4798..3266d941d807 100644 --- a/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts +++ b/packages/rest/src/__tests__/unit/rest.server/rest.server.open-api-spec.unit.ts @@ -14,6 +14,7 @@ import { RestServer, } from '../../..'; import {RestTags} from '../../../keys'; +import {ConsolidationEnhancer} from '../../../spec-enhancers/consolidate.spec-enhancer'; import {TestInfoSpecEnhancer} from './fixtures/info.spec.extension'; describe('RestServer.getApiSpec()', () => { @@ -321,6 +322,11 @@ describe('RestServer.getApiSpec()', () => { }); }); + it('registers consolidate enhancer', async () => { + const enhancer = await server.OASEnhancer.getEnhancerByName('consolidate'); + expect(enhancer).to.be.instanceOf(ConsolidationEnhancer); + }); + it('invokes registered oas enhancers', async () => { const EXPECTED_SPEC_INFO = { title: 'LoopBack Test Application', diff --git a/packages/rest/src/rest.component.ts b/packages/rest/src/rest.component.ts index d2a1b3f2142e..53e4fde8251c 100644 --- a/packages/rest/src/rest.component.ts +++ b/packages/rest/src/rest.component.ts @@ -42,6 +42,7 @@ import { RestServerConfig, } from './rest.server'; import {DefaultSequence} from './sequence'; +import {ConsolidationEnhancer} from './spec-enhancers/consolidate.spec-enhancer'; import {InfoSpecEnhancer} from './spec-enhancers/info.spec-enhancer'; import {AjvFactoryProvider} from './validation/ajv-factory.provider'; @@ -85,6 +86,7 @@ export class RestComponent implements Component { RestBindings.REQUEST_BODY_PARSER_STREAM, ), createBindingFromClass(InfoSpecEnhancer), + createBindingFromClass(ConsolidationEnhancer), ]; servers: { [name: string]: Constructor; diff --git a/packages/rest/src/rest.server.ts b/packages/rest/src/rest.server.ts index 88b524a27baa..3e5576f07a16 100644 --- a/packages/rest/src/rest.server.ts +++ b/packages/rest/src/rest.server.ts @@ -1061,6 +1061,12 @@ export interface OpenApiSpecOptions { * Set this flag to disable the endpoint for OpenAPI spec */ disabled?: true; + + /** + * Set this flag to `false` to disable OAS schema consolidation. If not set, + * the value defaults to `true`. + */ + consolidate?: boolean; } export interface ApiExplorerOptions { diff --git a/packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts b/packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts new file mode 100644 index 000000000000..62650ad0efea --- /dev/null +++ b/packages/rest/src/spec-enhancers/consolidate.spec-enhancer.ts @@ -0,0 +1,182 @@ +// Copyright IBM Corp. 2020. All Rights Reserved. +// Node module: @loopback/rest +// This file is licensed under the MIT License. +// License text available at https://opensource.org/licenses/MIT + +import { + ApplicationConfig, + bind, + BindingScope, + CoreBindings, + inject, +} from '@loopback/core'; +import { + asSpecEnhancer, + ISpecificationExtension, + isSchemaObject, + OASEnhancer, + OpenApiSpec, + ReferenceObject, + SchemaObject, +} from '@loopback/openapi-v3'; +import debugFactory from 'debug'; +import compare from 'json-schema-compare'; +import _ from 'lodash'; + +const debug = debugFactory('loopback:openapi:spec-enhancer:consolidate'); + +/** + * This enhancer consolidates schemas into `/components/schemas` and replaces + * instances of said schema with a $ref pointer. + * + * Please note that the title property must be set on a schema in order to be + * considered for consolidation. + * + * For example, with the following schema instance: + * + * ```json + * schema: { + * title: 'loopback.example', + * properties: { + * test: { + * type: 'string', + * }, + * }, + * } + * ``` + * + * The consolidator will copy the schema body to + * `/components/schemas/loopback.example` and replace any instance of the schema + * with a reference to the component schema as follows: + * + * ```json + * schema: { + * $ref: '#/components/schemas/loopback.example', + * } + * ``` + * + * When comparing schemas to avoid naming collisions, the description field + * is ignored. + */ +@bind(asSpecEnhancer, {scope: BindingScope.SINGLETON}) +export class ConsolidationEnhancer implements OASEnhancer { + name = 'consolidate'; + disabled: boolean; + + constructor( + @inject(CoreBindings.APPLICATION_CONFIG, {optional: true}) + readonly config?: ApplicationConfig, + ) { + this.disabled = this.config?.rest?.openApiSpec?.consolidate === false; + } + + modifySpec(spec: OpenApiSpec): OpenApiSpec { + return !this.disabled ? this.consolidateSchemaObjects(spec) : spec; + } + + /** + * Recursively search OpenApiSpec PathsObject for SchemaObjects with title + * property. Moves reusable schema bodies to #/components/schemas and replace + * with json pointer. It handles title collisions with schema body comparision. + */ + private consolidateSchemaObjects(spec: OpenApiSpec): OpenApiSpec { + // use 'paths' as crawl root + this.recursiveWalk(spec.paths, ['paths'], spec); + + return spec; + } + + private recursiveWalk( + rootSchema: ISpecificationExtension, + parentPath: Array, + spec: OpenApiSpec, + ) { + if (this.isTraversable(rootSchema)) { + Object.entries(rootSchema).forEach(([key, subSchema]) => { + if (subSchema) { + this.recursiveWalk(subSchema, parentPath.concat(key), spec); + this.processSchema(subSchema, parentPath.concat(key), spec); + } + }); + } + } + + /** + * Carry out schema consolidation after tree traversal. If 'title' property + * set then we consider current schema for consolidation. SchemaObjects with + * properties (and title set) are moved to #/components/schemas/ and + * replaced with ReferenceObject. + * + * Features: + * - name collision protection + * + * @param schema - current schema element to process + * @param parentPath - path object to parent + * @param spec - subject OpenApi specification + */ + private processSchema( + schema: SchemaObject | ReferenceObject, + parentPath: Array<string>, + spec: OpenApiSpec, + ) { + const schemaObj = this.ifConsolidationCandidate(schema); + if (schemaObj) { + // name collison protection + let instanceNo = 1; + let title = schemaObj.title!; + let refSchema = this.getRefSchema(title, spec); + while ( + refSchema && + !compare(schemaObj as ISpecificationExtension, refSchema, { + ignore: ['description'], + }) + ) { + title = `${schemaObj.title}${instanceNo++}`; + refSchema = this.getRefSchema(title, spec); + } + if (!refSchema) { + debug('Creating new component $ref with schema %j', schema); + this.patchRef(title, schema, spec); + } + debug('Creating link to $ref %j', title); + this.patchPath(title, parentPath, spec); + } + } + + private getRefSchema( + name: string, + spec: OpenApiSpec, + ): ISpecificationExtension | undefined { + const schema = _.get(spec, ['components', 'schemas', name]); + + return schema; + } + + private patchRef( + name: string, + value: ISpecificationExtension, + spec: OpenApiSpec, + ) { + _.set(spec, ['components', 'schemas', name], value); + } + + private patchPath(name: string, path: Array<string>, spec: OpenApiSpec) { + const patch = { + $ref: `#/components/schemas/${name}`, + }; + _.set(spec, path, patch); + } + + private ifConsolidationCandidate( + schema: SchemaObject | ReferenceObject, + ): SchemaObject | undefined { + // use title to discriminate references + return isSchemaObject(schema) && schema.properties && schema.title + ? schema + : undefined; + } + + private isTraversable(schema: ISpecificationExtension): boolean { + return schema && typeof schema === 'object' ? true : false; + } +}