feat: validate input on run

package.json

@@ -83,6 +83,7 @@
         "istextorbinary": "~9.5.0",
         "jju": "~1.4.0",
         "load-json-file": "~7.0.1",
+        "lodash.clonedeep": "^4.5.0",
         "mime": "~4.0.1",
         "mixpanel": "~0.18.0",
         "open": "~10.1.0",
@@ -107,6 +108,7 @@
         "@types/inquirer": "^9.0.7",
         "@types/is-ci": "^3.0.4",
         "@types/jju": "^1.4.5",
+        "@types/lodash.clonedeep": "^4",
         "@types/mime": "^4.0.0",
         "@types/node": "^20.11.20",
         "@types/semver": "^7.5.8",

src/lib/consts.ts

@@ -96,4 +96,6 @@ export enum CommandExitCodes {
     RunAborted = 3,
 
     NoFilesToPush = 4,
+
+    InvalidInput = 5,
 }

src/lib/input_schema.ts

@@ -1,9 +1,11 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
 import process from 'node:process';
 
 import { KEY_VALUE_STORE_KEYS } from '@apify/consts';
 import { validateInputSchema } from '@apify/input_schema';
+import deepClone from 'lodash.clonedeep';
 import _ from 'underscore';
 import { writeJsonFile } from 'write-json-file';
 
@@ -88,7 +90,6 @@ export const createPrefilledInputFileFromInputSchema = async (actorFolderDir: st
             const validator = new Ajv({ strict: false });
             validateInputSchema(validator, inputSchema);
 
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
             inputFile = _.mapObject(inputSchema.properties as any, (fieldSchema) => ((fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden')
                 ? fieldSchema.default
                 : fieldSchema.prefill
@@ -102,3 +103,42 @@ export const createPrefilledInputFileFromInputSchema = async (actorFolderDir: st
         await writeJsonFile(inputJsonPath, inputFile);
     }
 };
+
+export const getDefaultsAndPrefillsFromInputSchema = (inputSchema: any) => {
+    const defaults: Record<string, unknown> = {};
+
+    for (const [key, fieldSchema] of Object.entries<any>(inputSchema.properties)) {
+        if (fieldSchema.default !== undefined) {
+            defaults[key] = fieldSchema.default;
+        } else if (fieldSchema.prefill !== undefined) {
+            defaults[key] = fieldSchema.prefill;
+        }
+    }
+
+    return defaults;
+};
+
+// Lots of code copied from @apify-packages/actor, this really should be moved to the shared input_schema package
+export const getAjvValidator = (inputSchema: any, ajvInstance: import('ajv').Ajv) => {
+    const copyOfSchema = deepClone(inputSchema);
+    copyOfSchema.required = [];
+
+    for (const [inputSchemaFieldKey, inputSchemaField] of Object.entries<any>(inputSchema.properties)) {
+        // `required` field doesn't need to be present in input schema
+        const isRequired = inputSchema.required?.includes(inputSchemaFieldKey);
+        const hasDefault = inputSchemaField.default !== undefined;
+
+        if (isRequired && !hasDefault) {
+            // If field is required but has default, we act like it's optional because we always have value to use
+            copyOfSchema.required.push(inputSchemaFieldKey);
+            if (inputSchemaField.type === 'array') {
+                // If array is required, it has to have at least 1 item.
+                inputSchemaField.minItems = Math.max(1, inputSchemaField.minItems || 0);
+            }
+        }
+    }
+
+    delete copyOfSchema.$schema;
+
+    return ajvInstance.compile(copyOfSchema);
+};

src/lib/utils.ts

@@ -532,6 +532,8 @@ export const checkIfStorageIsEmpty = async () => {
         `${getLocalStorageDir()}/**`,
         // Omit INPUT.* file
         `!${getLocalKeyValueStorePath()}/${KEY_VALUE_STORE_KEYS.INPUT}.*`,
+        // Omit INPUT_CLI-* files
+        `!${getLocalKeyValueStorePath()}/${KEY_VALUE_STORE_KEYS.INPUT}_CLI-*`,
     ]);
 
     return filesWithoutInput.length === 0;

test/__setup__/hooks/useProcessCwdMock.ts

@@ -12,6 +12,19 @@ export function useProcessCwdMock(cwdMock: () => string) {
         };
     });
 
+    vitest.doMock('process', async (importActual) => {
+        const actual = await importActual<typeof import('process')>();
+
+        return {
+            ...actual,
+            cwd: cwdMock,
+            default: {
+                ...actual,
+                cwd: cwdMock,
+            },
+        };
+    });
+
     const processCwdSpy = vitest.spyOn(process, 'cwd');
     processCwdSpy.mockImplementation(cwdMock);
 }

test/__setup__/input-schemas/defaults.json

@@ -0,0 +1,24 @@
+{
+  "title": "Defaults",
+  "description": "Ensures defaults also get filled into the input",
+  "type": "object",
+  "schemaVersion": 1,
+  "properties": {
+    "awesome": {
+      "title": "Are you awesome",
+      "type": "boolean",
+      "description": "yesnt",
+      "editor": "checkbox"
+    },
+    "help": {
+      "title": "optional",
+      "type": "string",
+      "description": "A message, stop looking in these files",
+      "default": "this_maze_is_not_meant_for_you",
+      "editor": "textfield"
+    }
+  },
+  "required": [
+    "awesome"
+  ]
+}

test/__setup__/input-schemas/missing-required-property.json

@@ -0,0 +1,17 @@
+{
+  "title": "required",
+  "description": "Ensures cli throws when required fields are missing",
+  "type": "object",
+  "schemaVersion": 1,
+  "properties": {
+    "awesome": {
+      "title": "Are you awesome",
+      "type": "boolean",
+      "description": "yesnt",
+      "editor": "checkbox"
+    }
+  },
+  "required": [
+    "awesome"
+  ]
+}

test/__setup__/input-schemas/prefills.json

@@ -0,0 +1,24 @@
+{
+  "title": "Defaults",
+  "description": "Ensures defaults also get filled into the input",
+  "type": "object",
+  "schemaVersion": 1,
+  "properties": {
+    "awesome": {
+      "title": "Are you awesome",
+      "type": "boolean",
+      "description": "yesnt",
+      "editor": "checkbox"
+    },
+    "help": {
+      "title": "optional",
+      "type": "string",
+      "description": "A message, stop looking in these files",
+      "prefill": "this_maze_is_not_meant_for_you",
+      "editor": "textfield"
+    }
+  },
+  "required": [
+    "awesome"
+  ]
+}

test/commands/run.test.ts

@@ -1,6 +1,8 @@
-import { existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { copyFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
 
 import { APIFY_ENV_VARS } from '@apify/consts';
+import { captureOutput } from '@oclif/test';
 import { loadJsonFileSync } from 'load-json-file';
 import { writeJsonFileSync } from 'write-json-file';
 
@@ -12,6 +14,21 @@ import { useAuthSetup } from '../__setup__/hooks/useAuthSetup.js';
 import { useTempPath } from '../__setup__/hooks/useTempPath.js';
 
 const actName = 'run-my-actor';
+const pathToDefaultsInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/defaults.json', import.meta.url));
+const pathToMissingRequiredInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/missing-required-property.json', import.meta.url));
+const pathToPrefillsInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/prefills.json', import.meta.url));
+
+const INPUT_SCHEMA_ACTOR_SRC = `
+import { Actor } from 'apify';
+
+Actor.main(async () => {
+    const input = await Actor.getInput();
+
+    await Actor.setValue('OUTPUT', input);
+
+    console.log('Done.');
+});
+`;
 
 useAuthSetup({ perTest: true });
 
@@ -233,4 +250,86 @@ describe('apify run', () => {
             throw new Error('Can not run Actor without storage folder!');
         }
     });
+
+    describe('input tests', () => {
+        const actPath = joinPath('src/main.js');
+        const inputSchemaPath = joinPath('INPUT_SCHEMA.json');
+        const inputPath = joinPath(getLocalKeyValueStorePath(), 'INPUT.json');
+        const outputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json');
+        const handPassedInput = JSON.stringify({ awesome: null });
+
+        beforeAll(() => {
+            writeFileSync(actPath, INPUT_SCHEMA_ACTOR_SRC, { flag: 'w' });
+        });
+
+        it('throws when required field is not provided', async () => {
+            writeFileSync(inputPath, '{}', { flag: 'w' });
+            copyFileSync(pathToMissingRequiredInputSchema, inputSchemaPath);
+
+            const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url));
+
+            expect(error).toBeDefined();
+            expect(error!.message).toMatch(/Field awesome is required/i);
+        });
+
+        it('throws when required field has wrong type', async () => {
+            writeFileSync(inputPath, '{"awesome": 42}', { flag: 'w' });
+            copyFileSync(pathToDefaultsInputSchema, inputSchemaPath);
+
+            const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url));
+
+            expect(error).toBeDefined();
+            expect(error!.message).toMatch(/Field awesome must be boolean/i);
+        });
+
+        it('throws when passing manual input, but local file has correct input', async () => {
+            writeFileSync(inputPath, '{"awesome": true}', { flag: 'w' });
+            copyFileSync(pathToDefaultsInputSchema, inputSchemaPath);
+
+            const { error } = await captureOutput(async () => RunCommand.run(['--input', handPassedInput], import.meta.url));
+
+            expect(error).toBeDefined();
+            expect(error!.message).toMatch(/Field awesome must be boolean/i);
+        });
+
+        it('throws when input has default field of wrong type', async () => {
+            writeFileSync(inputPath, '{"awesome": true, "help": 123}', { flag: 'w' });
+            copyFileSync(pathToDefaultsInputSchema, inputSchemaPath);
+
+            const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url));
+
+            expect(error).toBeDefined();
+            expect(error!.message).toMatch(/Field help must be string/i);
+        });
+
+        it('throws when input has prefilled field of wrong type', async () => {
+            writeFileSync(inputPath, '{"awesome": true, "help": 123}', { flag: 'w' });
+            copyFileSync(pathToPrefillsInputSchema, inputSchemaPath);
+
+            const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url));
+
+            expect(error).toBeDefined();
+            expect(error!.message).toMatch(/Field help must be string/i);
+        });
+
+        it('automatically inserts missing defaulted fields', async () => {
+            writeFileSync(inputPath, '{"awesome": true}', { flag: 'w' });
+            copyFileSync(pathToDefaultsInputSchema, inputSchemaPath);
+
+            await RunCommand.run([], import.meta.url);
+
+            const output = loadJsonFileSync(outputPath);
+            expect(output).toStrictEqual({ awesome: true, help: 'this_maze_is_not_meant_for_you' });
+        });
+
+        it('automatically inserts missing prefilled fields', async () => {
+            writeFileSync(inputPath, '{"awesome": true}', { flag: 'w' });
+            copyFileSync(pathToPrefillsInputSchema, inputSchemaPath);
+
+            await RunCommand.run([], import.meta.url);
+
+            const output = loadJsonFileSync(outputPath);
+            expect(output).toStrictEqual({ awesome: true, help: 'this_maze_is_not_meant_for_you' });
+        });
+    });
 });

yarn.lock

@@ -2277,6 +2277,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/lodash.clonedeep@npm:^4":
+  version: 4.5.9
+  resolution: "@types/lodash.clonedeep@npm:4.5.9"
+  dependencies:
+    "@types/lodash": "npm:*"
+  checksum: 10c0/2f224ce9578046bccd1cd9594fb73540600ebd3d59a45695166a6123e2c376b84ab106b005a00453f357907f25bc8bfd2271b822be76e8f5527eadb4690b5e96
+  languageName: node
+  linkType: hard
+
+"@types/lodash@npm:*":
+  version: 4.17.5
+  resolution: "@types/lodash@npm:4.17.5"
+  checksum: 10c0/55924803ed853e72261512bd3eaf2c5b16558c3817feb0a3125ef757afe46e54b86f33d1960e40b7606c0ddab91a96f47966bf5e6006b7abfd8994c13b04b19b
+  languageName: node
+  linkType: hard
+
 "@types/mime@npm:^1":
   version: 1.3.5
   resolution: "@types/mime@npm:1.3.5"
@@ -2889,6 +2905,7 @@ __metadata:
     "@types/inquirer": "npm:^9.0.7"
     "@types/is-ci": "npm:^3.0.4"
     "@types/jju": "npm:^1.4.5"
+    "@types/lodash.clonedeep": "npm:^4"
     "@types/mime": "npm:^4.0.0"
     "@types/node": "npm:^20.11.20"
     "@types/semver": "npm:^7.5.8"
@@ -2918,6 +2935,7 @@ __metadata:
     istextorbinary: "npm:~9.5.0"
     jju: "npm:~1.4.0"
     load-json-file: "npm:~7.0.1"
+    lodash.clonedeep: "npm:^4.5.0"
     mime: "npm:~4.0.1"
     mixpanel: "npm:~0.18.0"
     oclif: "npm:^4.4.18"
@@ -6598,6 +6616,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"lodash.clonedeep@npm:^4.5.0":
+  version: 4.5.0
+  resolution: "lodash.clonedeep@npm:4.5.0"
+  checksum: 10c0/2caf0e4808f319d761d2939ee0642fa6867a4bbf2cfce43276698828380756b99d4c4fa226d881655e6ac298dd453fe12a5ec8ba49861777759494c534936985
+  languageName: node
+  linkType: hard
+
 "lodash.isequal@npm:^4.5.0":
   version: 4.5.0
   resolution: "lodash.isequal@npm:4.5.0"