feat(content-item): detect and rewrite media links when importing with --media flag

jest.config.js

@@ -4,5 +4,8 @@ module.exports = {
   transform: {
     '^.+\\.ts?$': 'ts-jest'
   },
-  preset: 'ts-jest'
+  preset: 'ts-jest',
+  coveragePathIgnorePatterns: [
+    '^.+\\.mocks\.ts?$'
+  ]
 };

package.json

@@ -88,11 +88,14 @@
     "@types/jest": "^24.0.18",
     "@types/lodash": "^4.14.144",
     "@types/node-fetch": "^2.5.7",
+    "@types/promise-retry": "^1.1.3",
     "@types/rimraf": "^3.0.0",
     "@types/table": "^4.0.7",
+    "@types/url-template": "^2.0.28",
     "@typescript-eslint/eslint-plugin": "^2.3.0",
     "@typescript-eslint/parser": "^2.3.0",
     "adm-zip": "^0.4.13",
+    "axios-mock-adapter": "^1.19.0",
     "commitizen": "^4.0.3",
     "cz-conventional-changelog": "^3.0.2",
     "eslint": "^6.4.0",
@@ -115,6 +118,7 @@
     "dc-management-sdk-js": "^1.9.0",
     "lodash": "^4.17.15",
     "node-fetch": "^2.6.0",
+    "promise-retry": "^2.0.1",
     "rimraf": "^3.0.0",
     "sanitize-filename": "^1.6.3",
     "table": "^5.4.6",

src/commands/content-item/archive.spec.ts

@@ -687,7 +687,7 @@ describe('content-item archive command', () => {
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
       (readline as any).setResponses(['y']);
 
-      const logFileName = 'temp/content-item-unrachive.log';
+      const logFileName = 'temp/content-item-unarchive.log';
       const log = '// Type log test file\n' + 'UNARCHIVE 1\n' + 'UNARCHIVE 2\n' + 'UNARCHIVE idMissing';
 
       const dir = dirname(logFileName);
@@ -740,7 +740,7 @@ describe('content-item archive command', () => {
         await promisify(unlink)('temp/content-item-unarchive.log');
       }
 
-      const logFileName = 'temp/content-item-unrachive.log';
+      const logFileName = 'temp/content-item-unarchive.log';
       const log = '// Type log test file\n' + 'UNARCHIVE 1\n' + 'UNARCHIVE 2\n' + 'UNARCHIVE idMissing';
 
       const dir = dirname(logFileName);

src/commands/content-item/copy.spec.ts

@@ -147,6 +147,13 @@ describe('content-item copy command', () => {
         describe: 'Exclude delivery keys when importing content items.'
       });
 
+      expect(spyOption).toHaveBeenCalledWith('media', {
+        type: 'boolean',
+        boolean: true,
+        describe:
+          "Detect and rewrite media links to match assets in the target account's DAM. Your client must have DAM permissions configured."
+      });
+
       expect(spyOption).toHaveBeenCalledWith('logFile', {
         type: 'string',
         default: LOG_FILENAME,
@@ -227,7 +234,8 @@ describe('content-item copy command', () => {
         publish: true,
         republish: true,
 
-        excludeKeys: true
+        excludeKeys: true,
+        media: true
       };
       await handler(argv);
 
@@ -256,6 +264,7 @@ describe('content-item copy command', () => {
       expect(importCalls[0].republish).toEqual(argv.republish);
 
       expect(importCalls[0].excludeKeys).toEqual(argv.excludeKeys);
+      expect(importCalls[0].media).toEqual(argv.media);
     });
 
     it('should forward to import-revert when revertLog is present.', async () => {

src/commands/content-item/copy.ts

@@ -127,6 +127,13 @@ export const builder = (yargs: Argv): void => {
       describe: 'Exclude delivery keys when importing content items.'
     })
 
+    .option('media', {
+      type: 'boolean',
+      boolean: true,
+      describe:
+        "Detect and rewrite media links to match assets in the target account's DAM. Your client must have DAM permissions configured."
+    })
+
     .option('logFile', {
       type: 'string',
       default: LOG_FILENAME,
@@ -213,12 +220,14 @@ export const handler = async (argv: Arguments<CopyItemBuilderOptions & Configura
         force: argv.force,
         validate: argv.validate,
         skipIncomplete: argv.skipIncomplete,
-        logFile: log,
 
         republish: argv.republish,
         publish: argv.publish,
 
-        excludeKeys: argv.excludeKeys
+        excludeKeys: argv.excludeKeys,
+
+        media: argv.media,
+        logFile: log
       });
 
       if (importResult) {

src/commands/content-item/import.spec.ts

@@ -15,11 +15,13 @@ import rmdir from 'rimraf';
 import { ensureDirectoryExists } from '../../common/import/directory-utils';
 import { MockContent, ItemTemplate } from '../../common/dc-management-sdk-js/mock-content';
 import { FileLog } from '../../common/file-log';
+import { MediaRewriter } from '../../common/media/media-rewriter';
 
 jest.mock('readline');
 jest.mock('./import-revert');
 jest.mock('../../services/dynamic-content-client-factory');
 jest.mock('../../common/import/publish-queue');
+jest.mock('../../common/media/media-rewriter');
 jest.mock('../../common/log-helpers');
 
 function rimraf(dir: string): Promise<Error> {
@@ -118,6 +120,13 @@ describe('content-item import command', () => {
         describe: 'Exclude delivery keys when importing content items.'
       });
 
+      expect(spyOption).toHaveBeenCalledWith('media', {
+        type: 'boolean',
+        boolean: true,
+        describe:
+          "Detect and rewrite media links to match assets in the target account's DAM. Your client must have DAM permissions configured."
+      });
+
       expect(spyOption).toHaveBeenCalledWith('logFile', {
         type: 'string',
         default: LOG_FILENAME,
@@ -1304,5 +1313,33 @@ describe('content-item import command', () => {
 
       await rimraf('temp/import/abort2/');
     });
+
+    it('should call the media rewriter when --media is passed', async () => {
+      const templates: ItemTemplate[] = [{ label: 'item1', repoId: 'repo', typeSchemaUri: 'http://type' }];
+
+      await createContent('temp/import/media1/', templates, false);
+
+      const mockContent = new MockContent(dynamicContentClientFactory as jest.Mock);
+      mockContent.createMockRepository('targetRepo');
+      mockContent.registerContentType('http://type', 'type', 'targetRepo');
+
+      const argv = {
+        ...yargArgs,
+        ...config,
+        dir: 'temp/import/media1/',
+        mapFile: 'temp/import/media1.json',
+        baseRepo: 'targetRepo',
+        media: true
+      };
+      await handler(argv);
+
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      expect((MediaRewriter as any).rewrites).toEqual(1);
+
+      expect(mockContent.metrics.itemsCreated).toEqual(1);
+      expect(mockContent.metrics.itemsUpdated).toEqual(0);
+
+      await rimraf('temp/import/media1/');
+    });
   });
 });

src/commands/content-item/import.ts

@@ -31,6 +31,7 @@ import { asyncQuestion } from '../../common/archive/archive-helpers';
 import { AmplienceSchemaValidator, defaultSchemaLookup } from '../../common/content-item/amplience-schema-validator';
 import { getDefaultLogPath } from '../../common/log-helpers';
 import { PublishQueue } from '../../common/import/publish-queue';
+import { MediaRewriter } from '../../common/media/media-rewriter';
 
 export function getDefaultMappingPath(name: string, platform: string = process.platform): string {
   return join(
@@ -114,6 +115,13 @@ export const builder = (yargs: Argv): void => {
       describe: 'Exclude delivery keys when importing content items.'
     })
 
+    .option('media', {
+      type: 'boolean',
+      boolean: true,
+      describe:
+        "Detect and rewrite media links to match assets in the target account's DAM. Your client must have DAM permissions configured."
+    })
+
     .option('logFile', {
       type: 'string',
       default: LOG_FILENAME,
@@ -272,7 +280,7 @@ const prepareContentForImport = async (
   folder: Folder | null,
   mapping: ContentMapping,
   log: FileLog,
-  argv: ImportItemBuilderOptions
+  argv: Arguments<ImportItemBuilderOptions & ConfigurationParameters>
 ): Promise<ContentDependancyTree | null> => {
   // traverse folder structure and find content items
   // replicate relative path string in target repo/folder (create if does not exist)
@@ -509,6 +517,29 @@ const prepareContentForImport = async (
     }
   }
 
+  // Step 2.5: Detect media links and rewrite their IDs, endpoints etc.
+  if (argv.media) {
+    log.appendLine(`Detecting and rewriting media links...`);
+    const rewriter = new MediaRewriter(argv, contentItems);
+
+    let missing: Set<string>;
+    try {
+      missing = await rewriter.rewrite();
+    } catch (e) {
+      log.error(
+        `Failed to rewrite media links. Make sure your client is properly configured, or remove the --media flag.`,
+        e
+      );
+      return null;
+    }
+
+    log.appendLine(`Finished rewriting media links.`);
+
+    if (missing.size > 0) {
+      log.warn(`${missing.size} media items could not be found in the target asset store. Ignoring.`);
+    }
+  }
+
   // Step 3: Track dependancies between content items and update them to match the new content ids.
   //         To do this, we must insert content that is depended on before inserting the replacement.
   //         Circular references cannot be resolved, so they should be handled by an insert with invalid id, then subsequent update.

src/commands/content-item/move.spec.ts

@@ -128,6 +128,13 @@ describe('content-item move command', () => {
         describe: 'Exclude delivery keys when importing content items.'
       });
 
+      expect(spyOption).toHaveBeenCalledWith('media', {
+        type: 'boolean',
+        boolean: true,
+        describe:
+          "Detect and rewrite media links to match assets in the target account's DAM. Your client must have DAM permissions configured."
+      });
+
       expect(spyOption).toHaveBeenCalledWith('logFile', {
         type: 'string',
         default: LOG_FILENAME,
@@ -210,7 +217,8 @@ describe('content-item move command', () => {
         mapFile: 'map.json',
         force: false,
         validate: false,
-        skipIncomplete: false
+        skipIncomplete: false,
+        media: true
       };
       await handler(argv);
 
@@ -229,6 +237,7 @@ describe('content-item move command', () => {
       expect(copyCalls[0].force).toEqual(argv.force);
       expect(copyCalls[0].validate).toEqual(argv.validate);
       expect(copyCalls[0].skipIncomplete).toEqual(argv.skipIncomplete);
+      expect(copyCalls[0].media).toEqual(argv.media);
 
       expect(argv.exportedIds).toEqual(exportIds);
 

src/commands/content-item/move.ts

@@ -122,6 +122,13 @@ export const builder = (yargs: Argv): void => {
       describe: 'Exclude delivery keys when importing content items.'
     })
 
+    .option('media', {
+      type: 'boolean',
+      boolean: true,
+      describe:
+        "Detect and rewrite media links to match assets in the target account's DAM. Your client must have DAM permissions configured."
+    })
+
     .option('logFile', {
       type: 'string',
       default: LOG_FILENAME,