Model Discovery

docs/site/Discovering-models.md

@@ -0,0 +1,45 @@
+---
+lang: en
+title: 'Discovering models from relational databases'
+keywords: LoopBack 4.0, LoopBack-Next
+sidebar: lb4_sidebar
+permalink: /doc/en/lb4/Discovering-models.html
+---
+
+## Synopsis
+
+LoopBack makes it simple to create models from an existing relational database.
+This process is called _discovery_ and is supported by the following connectors:
+
+- Cassandra
+- MySQL
+- Oracle
+- PostgreSQL
+- SQL Server
+- IBM DB2
+- IBM DashDB
+- IBM DB2 for z/OS
+- [SAP HANA](https://www.npmjs.org/package/loopback-connector-saphana) - Not
+  officially supported;
+
+## Overview
+
+Models can be discovered from a supported datasource by running the
+`lb4 discover` command.
+
+**The LoopBack project must be built and contain the built datasource files in
+`PROJECT_DIR/dist/datasources/*.js`**
+
+### Options
+
+`--dataSource`: Put a valid datasource name here to skip the datasource prompt
+
+`--views`: Choose whether to discover views. Default is true
+
+`--all`: Skips the model prompt and discovers all of them
+
+`--outDir`: Specify the directory into which the `model.model.ts` files will be
+placed. Default is `src/models`
+
+`--schema`: Specify the schema which the datasource will find the models to
+discover

docs/site/Model.md

@@ -97,6 +97,12 @@ export class Customer {
 }
 ```
 
+## Model Discovery
+
+LoopBack can automatically create model definitions by discovering the schema of
+your database. See [Discovering models](Discovering-models.md) for more details
+and a list of connectors supporting model discovery.
+
 ## Using the Juggler Bridge
 
 To define a model for use with the juggler bridge, extend your classes from

docs/site/sidebars/lb4_sidebar.yml

@@ -287,6 +287,10 @@ children:
     url: Model-generator.html
     output: 'web, pdf'
 
+  - title: 'Model discovery'
+    url: Discovering-models.html
+    output: 'web, pdf'
+
   - title: 'Repository generator'
     url: Repository-generator.html
     output: 'web, pdf'

docs/site/tables/lb4-artifact-commands.html

@@ -47,11 +47,18 @@
       <td><a href="OpenAPI-generator.html">OpenAPI generator</a></td>
     </tr>
 
+    <tr>
+      <td><code>lb4 discover</code></td>
+      <td>Discover models from relational databases</td>
+      <td><a href="Discovering-models.html">Model Discovery</a></td>
+    </tr>
+
     <tr>
       <td><code>lb4 observer</code></td>
       <td>Generate life cycle observers for application start/stop</td>
       <td><a href="Life-cycle-observer-generator.html">Life cycle observer generator</a></td>
     </tr>
 
+
   </tbody>
 </table>

packages/cli/README.md

@@ -275,9 +275,35 @@ Run the following command to install the CLI.
 
     Arguments:
       name  # Name for the observer  Type: String  Required: false
+
+    ```
+
+11. To discover a model from a supported datasource
+
+    ```sh
+    cd <your-project-directory>
+    lb4 discover
+      lb4 discover [<name>] [options]
+
+    Options:
+      -h,    --help           # Print the generator's options and usage
+             --skip-cache     # Do not remember prompt answers                                              Default: false
+             --skip-install   # Do not automatically install dependencies                                   Default: false
+             --force-install  # Fail on install dependencies error                                          Default: false
+      -c,    --config         # JSON file name or value to configure options
+      -y,    --yes            # Skip all confirmation prompts with default or provided value
+             --format         # Format generated code using npm run lint:fix
+      -ds,   --dataSource     # The name of the datasource to discover
+             --views          # Boolean to discover views                                                   Default: true
+             --schema         # Schema to discover
+             --all            # Discover all models without prompting users to select                       Default: false
+             --outDir         # Specify the directory into which the `model.model.ts` files will be placed
+
+    Arguments:
+      name  # Name for the discover  Type: String  Required: false
     ```
 
-11. To list available commands
+12. To list available commands
 
     `lb4 --commands` (or `lb4 -l`)
 
@@ -296,7 +322,7 @@ Run the following command to install the CLI.
 
     Please note `lb4 --help` also prints out available commands.
 
-12. To print out version information
+13. To print out version information
 
     `lb4 --version` (or `lb4 -v`)
 

packages/cli/generators/discover/index.js

@@ -0,0 +1,255 @@
+path = require('path');
+const fs = require('fs');
+const ArtifactGenerator = require('../../lib/artifact-generator');
+const modelMaker = require('../../lib/model-discoverer');
+const debug = require('../../lib/debug')('discover-generator');
+const utils = require('../../lib/utils');
+const modelDiscoverer = require('../../lib/model-discoverer');
+const rootDir = 'src';
+
+module.exports = class DiscoveryGenerator extends ArtifactGenerator {
+  constructor(args, opts) {
+    super(args, opts);
+
+    this.option('dataSource', {
+      type: String,
+      alias: 'ds',
+      description: 'The name of the datasource to discover',
+    });
+
+    this.option('views', {
+      type: Boolean,
+      description: 'Boolean to discover views',
+      default: true,
+    });
+
+    this.option('schema', {
+      type: String,
+      description: 'Schema to discover',
+      default: '',
+    });
+
+    this.option('all', {
+      type: Boolean,
+      description: 'Discover all models without prompting users to select',
+      default: false,
+    });
+
+    this.option('outDir', {
+      type: String,
+      description:
+        'Specify the directory into which the `model.model.ts` files will be placed',
+      default: undefined,
+    });
+  }
+
+  _setupGenerator() {
+    this.artifactInfo = {
+      type: 'discover',
+      rootDir,
+      outDir: path.resolve(rootDir, 'models'),
+    };
+
+    return super._setupGenerator();
+  }
+
+  /**
+   * If we have a dataSource, attempt to load it
+   * @returns {*}
+   */
+  setOptions() {
+    if (this.options.dataSource) {
+      debug(`Data source specified: ${this.options.dataSource}`);
+      this.artifactInfo.dataSource = modelMaker.loadDataSourceByName(
+        this.options.dataSource,
+      );
+    }
+
+    return super.setOptions();
+  }
+
+  /**
+   * Ensure CLI is being run in a LoopBack 4 project.
+   */
+  checkLoopBackProject() {
+    if (this.shouldExit()) return;
+    return super.checkLoopBackProject();
+  }
+
+  /**
+   * Loads all datasources to choose if the dataSource option isn't set
+   */
+  async loadAllDatasources() {
+    // If we have a dataSourcePath then it is already loaded for us, we don't need load any
+    if (this.artifactInfo.dataSource) {
+      return;
+    }
+    const dsDir = modelMaker.DEFAULT_DATASOURCE_DIRECTORY;
+    const datasourcesList = await utils.getArtifactList(
+      dsDir,
+      'datasource',
+      false,
+    );
+    debug(datasourcesList);
+
+    this.dataSourceChoices = datasourcesList.map(s =>
+      modelDiscoverer.loadDataSource(
+        path.resolve(dsDir, `${utils.kebabCase(s)}.datasource.js`),
+      ),
+    );
+    debug(`Done importing datasources`);
+  }
+
+  /**
+   * Ask the user to select the data source from which to discover
+   */
+  promptDataSource() {
+    if (this.shouldExit()) return;
+    const prompts = [
+      {
+        name: 'dataSource',
+        message: `Select the connector to discover`,
+        type: 'list',
+        choices: this.dataSourceChoices,
+        when:
+          this.artifactInfo.dataSource === undefined &&
+          !this.artifactInfo.modelDefinitions,
+      },
+    ];
+
+    return this.prompt(prompts).then(answer => {
+      if (!answer.dataSource) return;
+      debug(`Datasource answer: ${JSON.stringify(answer)}`);
+
+      this.artifactInfo.dataSource = this.dataSourceChoices.find(
+        d => d.name === answer.dataSource,
+      );
+    });
+  }
+
+  /**
+   * Puts all discoverable models in this.modelChoices
+   */
+  async discoverModelInfos() {
+    if (this.artifactInfo.modelDefinitions) return;
+    debug(`Getting all models from ${this.artifactInfo.dataSource.name}`);
+
+    this.modelChoices = await modelMaker.discoverModelNames(
+      this.artifactInfo.dataSource,
+      {views: this.options.views, schema: this.options.schema},
+    );
+    debug(
+      `Got ${this.modelChoices.length} models from ${
+        this.artifactInfo.dataSource.name
+      }`,
+    );
+  }
+
+  /**
+   * Now that we have a list of all models for a datasource,
+   * ask which models to discover
+   */
+  promptModelChoices() {
+    // If we are discovering all we don't need to prompt
+    if (this.options.all) {
+      this.discoveringModels = this.modelChoices;
+    }
+
+    const prompts = [
+      {
+        name: 'discoveringModels',
+        message: `Select the models which to discover`,
+        type: 'checkbox',
+        choices: this.modelChoices,
+        when:
+          this.discoveringModels === undefined &&
+          !this.artifactInfo.modelDefinitions,
+      },
+    ];
+
+    return this.prompt(prompts).then(answers => {
+      if (!answers.discoveringModels) return;
+      debug(`Models chosen: ${JSON.stringify(answers)}`);
+      this.discoveringModels = [];
+      answers.discoveringModels.forEach(m => {
+        this.discoveringModels.push(this.modelChoices.find(c => c.name === m));
+      });
+    });
+  }
+
+  /**
+   * Using artifactInfo.dataSource,
+   * artifactInfo.modelNameOptions
+   *
+   * this will discover every model
+   * and put it in artifactInfo.modelDefinitions
+   * @return {Promise<void>}
+   */
+  async getAllModelDefs() {
+    this.artifactInfo.modelDefinitions = [];
+    for (let i = 0; i < this.discoveringModels.length; i++) {
+      const modelInfo = this.discoveringModels[i];
+      debug(`Discovering: ${modelInfo.name}...`);
+      this.artifactInfo.modelDefinitions.push(
+        await modelMaker.discoverSingleModel(
+          this.artifactInfo.dataSource,
+          modelInfo.name,
+          {schema: modelInfo.schema},
+        ),
+      );
+      debug(`Discovered: ${modelInfo.name}`);
+    }
+  }
+
+  /**
+   * Iterate through all the models we have discovered and scaffold
+   */
+  async scaffold() {
+    this.artifactInfo.indexesToBeUpdated =
+      this.artifactInfo.indexesToBeUpdated || [];
+
+    // Exit if needed
+    if (this.shouldExit()) return false;
+
+    for (let i = 0; i < this.artifactInfo.modelDefinitions.length; i++) {
+      const modelDefinition = this.artifactInfo.modelDefinitions[i];
+      Object.entries(modelDefinition.properties).forEach(([k, v]) =>
+        modelDiscoverer.sanitizeProperty(v),
+      );
+      modelDefinition.isModelBaseBuiltin = true;
+      modelDefinition.modelBaseClass = 'Entity';
+      modelDefinition.className = utils.pascalCase(modelDefinition.name);
+      // These last two are so that the templat doesn't error out of they aren't there
+      modelDefinition.allowAdditionalProperties = true;
+      modelDefinition.modelSettings = modelDefinition.settings || {};
+      debug(`Generating: ${modelDefinition.name}`);
+
+      const fullPath = path.resolve(
+        this.options.outDir || this.artifactInfo.outDir,
+        utils.getModelFileName(modelDefinition.name),
+      );
+      debug(`Writing: ${fullPath}`);
+
+      this.copyTemplatedFiles(
+        modelDiscoverer.MODEL_TEMPLATE_PATH,
+        fullPath,
+        modelDefinition,
+      );
+
+      this.artifactInfo.indexesToBeUpdated.push({
+        dir: this.options.outDir || this.artifactInfo.outDir,
+        file: utils.getModelFileName(modelDefinition.name),
+      });
+    }
+
+    // This part at the end is just for the ArtifactGenerator
+    // end message to output something nice, before it was "Discover undefined was created in src/models/"
+    this.artifactInfo.name = this.artifactInfo.modelDefinitions
+      .map(d => utils.getModelFileName(d.name))
+      .join(',');
+  }
+
+  async end() {
+    await super.end();
+  }
+};