From cb098efee4af2b36f13fd5c2068a5ab80fcf5c6b Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Tue, 30 Apr 2024 16:51:42 +0300 Subject: [PATCH 1/5] feat: validate input on run --- src/commands/run.ts | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/commands/run.ts b/src/commands/run.ts index 5380016e6..21eccfe0c 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -4,21 +4,26 @@ import { join } from 'node:path'; import process from 'node:process'; import { APIFY_ENV_VARS } from '@apify/consts'; +import { validateInputSchema } from '@apify/input_schema'; import { Flags } from '@oclif/core'; import { loadJsonFile } from 'load-json-file'; +import mime from 'mime'; import { minVersion } from 'semver'; import { ApifyCommand } from '../lib/apify_command.js'; import { DEFAULT_LOCAL_STORAGE_DIR, LANGUAGE, LEGACY_LOCAL_STORAGE_DIR, PROJECT_TYPES, SUPPORTED_NODEJS_VERSION } from '../lib/consts.js'; import { execWithLog } from '../lib/exec.js'; +import { readInputSchema } from '../lib/input_schema.js'; import { error, info, warning } from '../lib/outputs.js'; import { ProjectAnalyzer } from '../lib/project_analyzer.js'; import { ScrapyProjectAnalyzer } from '../lib/projects/scrapy/ScrapyProjectAnalyzer.js'; import { replaceSecretsValue } from '../lib/secrets.js'; import { + Ajv, checkIfStorageIsEmpty, detectLocalActorLanguage, getLocalConfigOrThrow, + getLocalInput, getLocalStorageDir, getLocalUserInfo, getNpmCmd, @@ -174,6 +179,9 @@ export class RunCommand extends ApifyCommand { } } + // Validate input + await this.validateInput(); + // Attach env vars from local config files const localEnvVars: Record = { [APIFY_ENV_VARS.LOCAL_STORAGE_DIR]: actualStoragePath, @@ -272,6 +280,33 @@ export class RunCommand extends ApifyCommand { } } } + + private async validateInput() { + const { inputSchema } = await readInputSchema({ forcePath: this.args.path, cwd: process.cwd() }); + + if (!inputSchema) { + // We cannot validate input schema if it is not found. + return; + } + + // Step 1: validate the input schema + const validator = new Ajv({ strict: false }); + validateInputSchema(validator, inputSchema); // This one throws an error in a case of invalid schema. + + // Step 2: get the input from local store + const input = getLocalInput(process.cwd()); + + if (!input) { + // No input -> nothing to do + // TODO: but maybe prefill? + return; + } + + if (mime.getExtension(input.contentType!) === 'json') { + // Step 3: validate the input + const _inputJson = JSON.parse(input.body.toString('utf-8')); + } + } } enum RunType { From b2919845ff94f6c07fb2054c02a63e5be10af7a4 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Tue, 30 Apr 2024 21:05:22 +0300 Subject: [PATCH 2/5] feat: validate input schema/input on run --- src/commands/run.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/commands/run.ts b/src/commands/run.ts index 21eccfe0c..06c7fbc02 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import process from 'node:process'; import { APIFY_ENV_VARS } from '@apify/consts'; -import { validateInputSchema } from '@apify/input_schema'; +import { validateInputSchema, validateInputUsingValidator } from '@apify/input_schema'; import { Flags } from '@oclif/core'; import { loadJsonFile } from 'load-json-file'; import mime from 'mime'; @@ -290,7 +290,7 @@ export class RunCommand extends ApifyCommand { } // Step 1: validate the input schema - const validator = new Ajv({ strict: false }); + const validator = new Ajv({ strict: false, unicodeRegExp: false }); validateInputSchema(validator, inputSchema); // This one throws an error in a case of invalid schema. // Step 2: get the input from local store @@ -304,7 +304,18 @@ export class RunCommand extends ApifyCommand { if (mime.getExtension(input.contentType!) === 'json') { // Step 3: validate the input - const _inputJson = JSON.parse(input.body.toString('utf-8')); + const inputJson = JSON.parse(input.body.toString('utf-8')); + + const clonedInputSchema = JSON.parse(JSON.stringify(inputSchema)); + Reflect.deleteProperty(clonedInputSchema, '$schema'); + + const compiledInputSchema = validator.compile(clonedInputSchema); + + const errors = validateInputUsingValidator(compiledInputSchema, inputSchema, inputJson); + + if (errors.length > 0) { + throw new Error(`The input in your storage is invalid. Please fix the following errors:\n${errors.map((e) => ` - ${e.message}`).join('\n')}`); + } } } } From 2a98649761d16f4eec1914d7611bf019cd0fb92d Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 21 Jun 2024 18:58:25 +0300 Subject: [PATCH 3/5] chore: make it work and test --- package.json | 2 + src/commands/run.ts | 198 ++++++++++++++++++++++++++++++---------- src/lib/input_schema.ts | 42 ++++++++- src/lib/utils.ts | 2 + yarn.lock | 25 +++++ 5 files changed, 221 insertions(+), 48 deletions(-) diff --git a/package.json b/package.json index 94b7ddc7c..de8be2360 100644 --- a/package.json +++ b/package.json @@ -83,6 +83,7 @@ "istextorbinary": "~9.5.0", "jju": "~1.4.0", "load-json-file": "~7.0.1", + "lodash.clonedeep": "^4.5.0", "mime": "~4.0.1", "mixpanel": "~0.18.0", "open": "~10.1.0", @@ -107,6 +108,7 @@ "@types/inquirer": "^9.0.7", "@types/is-ci": "^3.0.4", "@types/jju": "^1.4.5", + "@types/lodash.clonedeep": "^4", "@types/mime": "^4.0.0", "@types/node": "^20.11.20", "@types/semver": "^7.5.8", diff --git a/src/commands/run.ts b/src/commands/run.ts index 06c7fbc02..79cadd432 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -1,6 +1,6 @@ import { existsSync, renameSync } from 'node:fs'; -import { stat } from 'node:fs/promises'; -import { join } from 'node:path'; +import { readFile, stat, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; import process from 'node:process'; import { APIFY_ENV_VARS } from '@apify/consts'; @@ -11,9 +11,10 @@ import mime from 'mime'; import { minVersion } from 'semver'; import { ApifyCommand } from '../lib/apify_command.js'; -import { DEFAULT_LOCAL_STORAGE_DIR, LANGUAGE, LEGACY_LOCAL_STORAGE_DIR, PROJECT_TYPES, SUPPORTED_NODEJS_VERSION } from '../lib/consts.js'; +import { CommandExitCodes, DEFAULT_LOCAL_STORAGE_DIR, LANGUAGE, LEGACY_LOCAL_STORAGE_DIR, PROJECT_TYPES, SUPPORTED_NODEJS_VERSION } from '../lib/consts.js'; import { execWithLog } from '../lib/exec.js'; -import { readInputSchema } from '../lib/input_schema.js'; +import { deleteFile } from '../lib/files.js'; +import { getAjvValidator, getDefaultsAndPrefillsFromInputSchema, readInputSchema } from '../lib/input_schema.js'; import { error, info, warning } from '../lib/outputs.js'; import { ProjectAnalyzer } from '../lib/project_analyzer.js'; import { ScrapyProjectAnalyzer } from '../lib/projects/scrapy/ScrapyProjectAnalyzer.js'; @@ -24,6 +25,7 @@ import { detectLocalActorLanguage, getLocalConfigOrThrow, getLocalInput, + getLocalKeyValueStorePath, getLocalStorageDir, getLocalUserInfo, getNpmCmd, @@ -74,6 +76,20 @@ export class RunCommand extends ApifyCommand { ].join(' '), required: false, }), + input: Flags.string({ + char: 'i', + // eslint-disable-next-line max-len + description: 'Optional JSON input to be given to the Actor. You can either provide the JSON string as a value to this, or `-` to read from standard input.', + required: false, + allowStdin: true, + exclusive: ['input-file'], + }), + 'input-file': Flags.string({ + aliases: ['if'], + description: 'Optional path to a file with JSON input to be given to the Actor. The file must be a valid JSON file.', + required: false, + exclusive: ['input'], + }), }; async run() { @@ -179,8 +195,52 @@ export class RunCommand extends ApifyCommand { } } - // Validate input - await this.validateInput(); + // Select correct input and validate it + let input: Record | undefined; + + if (this.flags.input) { + switch (this.flags.input[0]) { + case '-': { + error({ message: 'You need to pipe something into standard input when you specify the `-` value to `--input`.' }); + process.exitCode = CommandExitCodes.InvalidInput; + return; + } + default: { + try { + input = JSON.parse(this.flags.input); + + if (Array.isArray(input)) { + error({ message: 'The input must be an object, not an array.' }); + process.exitCode = CommandExitCodes.InvalidInput; + return; + } + } catch (err) { + error({ message: `Cannot parse JSON input.\n ${(err as Error).message}` }); + process.exitCode = CommandExitCodes.InvalidInput; + return; + } + } + } + } else if (this.flags.inputFile) { + const fullPath = resolve(cwd, this.flags.inputFile); + + try { + const fileContent = await readFile(fullPath, 'utf8'); + input = JSON.parse(fileContent); + + if (Array.isArray(input)) { + error({ message: 'The input file must contain an object, not an array.' }); + process.exitCode = CommandExitCodes.InvalidInput; + return; + } + } catch (err) { + error({ message: `Cannot read input file at path "${fullPath}".\n ${(err as Error).message}` }); + process.exitCode = CommandExitCodes.InvalidInput; + return; + } + } + + const envVariableOverride = await this.validateAndStoreInput(input); // Attach env vars from local config files const localEnvVars: Record = { @@ -188,6 +248,7 @@ export class RunCommand extends ApifyCommand { CRAWLEE_STORAGE_DIR: actualStoragePath, CRAWLEE_PURGE_ON_START, }; + if (proxy && proxy.password) localEnvVars[APIFY_ENV_VARS.PROXY_PASSWORD] = proxy.password; if (userId) localEnvVars[APIFY_ENV_VARS.USER_ID] = userId; if (token) localEnvVars[APIFY_ENV_VARS.TOKEN] = token; @@ -195,19 +256,25 @@ export class RunCommand extends ApifyCommand { const updatedEnv = replaceSecretsValue(localConfig!.environmentVariables as Record); Object.assign(localEnvVars, updatedEnv); } + // NOTE: User can overwrite env vars const env = Object.assign(localEnvVars, process.env); + if (envVariableOverride) { + env[APIFY_ENV_VARS.INPUT_KEY] = envVariableOverride; + } + if (!userId) { warning({ message: 'You are not logged in with your Apify Account. Some features like Apify Proxy will not work. Call "apify login" to fix that.', }); } - if (language === LANGUAGE.NODEJS) { // Actor is written in Node.js - const currentNodeVersion = languageVersion; - const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION); - if (currentNodeVersion) { + try { + if (language === LANGUAGE.NODEJS) { // Actor is written in Node.js + const currentNodeVersion = languageVersion; + const minimumSupportedNodeVersion = minVersion(SUPPORTED_NODEJS_VERSION); + if (currentNodeVersion) { // --max-http-header-size=80000 // Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb. // However they did not think about all the sites there with large headers, @@ -221,34 +288,32 @@ export class RunCommand extends ApifyCommand { }); } - this.telemetryData.actorNodejsVersion = currentNodeVersion; - this.telemetryData.actorLanguage = LANGUAGE.NODEJS; + this.telemetryData.actorNodejsVersion = currentNodeVersion; + this.telemetryData.actorLanguage = LANGUAGE.NODEJS; - // We allow "module" type directly in node too (it will work for a folder that has an `index.js` file) - if (runType === RunType.DirectFile || runType === RunType.Module) { - await execWithLog('node', [entrypoint], { env, cwd }); - } else { + // We allow "module" type directly in node too (it will work for a folder that has an `index.js` file) + if (runType === RunType.DirectFile || runType === RunType.Module) { + await execWithLog('node', [entrypoint], { env, cwd }); + } else { // TODO(vladfrangu): what is this for? Some old template maybe? // && !existsSync(serverJsFile) // const serverJsFile = join(cwd, 'server.js'); - const packageJson = await loadJsonFile<{ scripts: Record }>(packageJsonPath); + const packageJson = await loadJsonFile<{ scripts: Record }>(packageJsonPath); - if (!packageJson.scripts) { - throw new Error('No scripts were found in package.json. Please set it up for your project. ' + if (!packageJson.scripts) { + throw new Error('No scripts were found in package.json. Please set it up for your project. ' + 'For more information about that call "apify help run".'); - } + } - if (!packageJson.scripts[entrypoint]) { - throw new Error(`The script "${entrypoint}" was not found in package.json. Please set it up for your project. ` + if (!packageJson.scripts[entrypoint]) { + throw new Error(`The script "${entrypoint}" was not found in package.json. Please set it up for your project. ` + 'For more information about that call "apify help run".'); - } + } await execWithLog(getNpmCmd(), ['run', entrypoint], { env, cwd }); } } else { - error({ - message: `No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js Actors locally.`, - }); + error(`No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js Actors locally.`); } } else if (language === LANGUAGE.PYTHON) { const pythonVersion = languageVersion; @@ -258,13 +323,13 @@ export class RunCommand extends ApifyCommand { if (isPythonVersionSupported(pythonVersion)) { const pythonCommand = getPythonCommand(cwd); - if (isScrapyProject && !this.flags.entrypoint) { - const project = new ScrapyProjectAnalyzer(cwd); - project.loadScrapyCfg(); - if (project.configuration.hasKey('apify', 'mainpy_location')) { - entrypoint = project.configuration.get('apify', 'mainpy_location')!; + if (isScrapyProject && !this.flags.entrypoint) { + const project = new ScrapyProjectAnalyzer(cwd); + project.loadScrapyCfg(); + if (project.configuration.hasKey('apify', 'mainpy_location')) { + entrypoint = project.configuration.get('apify', 'mainpy_location')!; + } } - } if (runType === RunType.Module) { await execWithLog(pythonCommand, ['-m', entrypoint], { env, cwd }); @@ -272,51 +337,90 @@ export class RunCommand extends ApifyCommand { await execWithLog(pythonCommand, [entrypoint], { env, cwd }); } } else { - error({ message: `Python Actors require Python 3.8 or higher, but you have Python ${pythonVersion}!` }); - error({ message: 'Please install Python 3.8 or higher to be able to run Python Actors locally.' }); + error(`Python Actors require Python 3.8 or higher, but you have Python ${pythonVersion}!`); + error('Please install Python 3.8 or higher to be able to run Python Actors locally.'); } } else { - error({ message: 'No Python detected! Please install Python 3.8 or higher to be able to run Python Actors locally.' }); + error('No Python detected! Please install Python 3.8 or higher to be able to run Python Actors locally.'); } } } - private async validateInput() { + private async validateAndStoreInput(inputOverride?: Record) { const { inputSchema } = await readInputSchema({ forcePath: this.args.path, cwd: process.cwd() }); if (!inputSchema) { // We cannot validate input schema if it is not found. - return; + return null; } // Step 1: validate the input schema const validator = new Ajv({ strict: false, unicodeRegExp: false }); validateInputSchema(validator, inputSchema); // This one throws an error in a case of invalid schema. - // Step 2: get the input from local store + const defaults = getDefaultsAndPrefillsFromInputSchema(inputSchema); + const compiledInputSchema = getAjvValidator(inputSchema, validator); + + const inputKey = `INPUT_CLI-${Date.now()}`; + const inputFilePath = join(process.cwd(), getLocalKeyValueStorePath(), `${inputKey}.json`); + + // Step 2. If there is an input override, we validate it and store it + if (inputOverride) { + const fullInputOverride = { + ...defaults, + ...inputOverride, + }; + + const errors = validateInputUsingValidator(compiledInputSchema, inputSchema, fullInputOverride); + + if (errors.length > 0) { + throw new Error(`The input provided is invalid. Please fix the following errors:\n${errors.map((e) => ` - ${e.message}`).join('\n')}`); + } + + await writeFile(inputFilePath, JSON.stringify(fullInputOverride, null, 2)); + + return inputKey; + } + + // Step 3: get the input from local store const input = getLocalInput(process.cwd()); if (!input) { - // No input -> nothing to do - // TODO: but maybe prefill? - return; + // No input -> use defaults for this run + await writeFile(inputFilePath, JSON.stringify(defaults, null, 2)); + + return inputKey; } if (mime.getExtension(input.contentType!) === 'json') { - // Step 3: validate the input + // Step 4: validate the input const inputJson = JSON.parse(input.body.toString('utf-8')); - const clonedInputSchema = JSON.parse(JSON.stringify(inputSchema)); - Reflect.deleteProperty(clonedInputSchema, '$schema'); + if (Array.isArray(inputJson)) { + throw new Error('The input in your storage is invalid. It should be an object, not an array.'); + } - const compiledInputSchema = validator.compile(clonedInputSchema); + const fullInput = { + ...defaults, + ...inputJson, + }; - const errors = validateInputUsingValidator(compiledInputSchema, inputSchema, inputJson); + const errors = validateInputUsingValidator(compiledInputSchema, inputSchema, fullInput); if (errors.length > 0) { - throw new Error(`The input in your storage is invalid. Please fix the following errors:\n${errors.map((e) => ` - ${e.message}`).join('\n')}`); + throw new Error(`The input in your storage is invalid. Please fix the following errors:\n${ + errors.map((e) => ` - ${ + e.message.replace('Field input.', 'Field ') + }`).join('\n')}`); } + + // Step 4: store the input + await writeFile(inputFilePath, JSON.stringify(fullInput, null, 2)); + + return inputKey; } + + return null; } } diff --git a/src/lib/input_schema.ts b/src/lib/input_schema.ts index 59dfb04ae..ccde692e4 100644 --- a/src/lib/input_schema.ts +++ b/src/lib/input_schema.ts @@ -1,9 +1,11 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ import { existsSync } from 'node:fs'; import { join } from 'node:path'; import process from 'node:process'; import { KEY_VALUE_STORE_KEYS } from '@apify/consts'; import { validateInputSchema } from '@apify/input_schema'; +import deepClone from 'lodash.clonedeep'; import _ from 'underscore'; import { writeJsonFile } from 'write-json-file'; @@ -88,7 +90,6 @@ export const createPrefilledInputFileFromInputSchema = async (actorFolderDir: st const validator = new Ajv({ strict: false }); validateInputSchema(validator, inputSchema); - // eslint-disable-next-line @typescript-eslint/no-explicit-any inputFile = _.mapObject(inputSchema.properties as any, (fieldSchema) => ((fieldSchema.type === 'boolean' || fieldSchema.editor === 'hidden') ? fieldSchema.default : fieldSchema.prefill @@ -102,3 +103,42 @@ export const createPrefilledInputFileFromInputSchema = async (actorFolderDir: st await writeJsonFile(inputJsonPath, inputFile); } }; + +export const getDefaultsAndPrefillsFromInputSchema = (inputSchema: any) => { + const defaults: Record = {}; + + for (const [key, fieldSchema] of Object.entries(inputSchema.properties)) { + if (fieldSchema.default !== undefined) { + defaults[key] = fieldSchema.default; + } else if (fieldSchema.prefill !== undefined) { + defaults[key] = fieldSchema.prefill; + } + } + + return defaults; +}; + +// Lots of code copied from @apify-packages/actor, this really should be moved to the shared input_schema package +export const getAjvValidator = (inputSchema: any, ajvInstance: import('ajv').Ajv) => { + const copyOfSchema = deepClone(inputSchema); + copyOfSchema.required = []; + + for (const [inputSchemaFieldKey, inputSchemaField] of Object.entries(inputSchema.properties)) { + // `required` field doesn't need to be present in input schema + const isRequired = inputSchema.required?.includes(inputSchemaFieldKey); + const hasDefault = inputSchemaField.default !== undefined; + + if (isRequired && !hasDefault) { + // If field is required but has default, we act like it's optional because we always have value to use + copyOfSchema.required.push(inputSchemaFieldKey); + if (inputSchemaField.type === 'array') { + // If array is required, it has to have at least 1 item. + inputSchemaField.minItems = Math.max(1, inputSchemaField.minItems || 0); + } + } + } + + delete copyOfSchema.$schema; + + return ajvInstance.compile(copyOfSchema); +}; diff --git a/src/lib/utils.ts b/src/lib/utils.ts index e4b2e9f86..7ed9d006f 100644 --- a/src/lib/utils.ts +++ b/src/lib/utils.ts @@ -532,6 +532,8 @@ export const checkIfStorageIsEmpty = async () => { `${getLocalStorageDir()}/**`, // Omit INPUT.* file `!${getLocalKeyValueStorePath()}/${KEY_VALUE_STORE_KEYS.INPUT}.*`, + // Omit INPUT_CLI-* files + `!${getLocalKeyValueStorePath()}/${KEY_VALUE_STORE_KEYS.INPUT}_CLI-*`, ]); return filesWithoutInput.length === 0; diff --git a/yarn.lock b/yarn.lock index 2e3d3239e..ac540a4f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2322,6 +2322,22 @@ __metadata: languageName: node linkType: hard +"@types/lodash.clonedeep@npm:^4": + version: 4.5.9 + resolution: "@types/lodash.clonedeep@npm:4.5.9" + dependencies: + "@types/lodash": "npm:*" + checksum: 10c0/2f224ce9578046bccd1cd9594fb73540600ebd3d59a45695166a6123e2c376b84ab106b005a00453f357907f25bc8bfd2271b822be76e8f5527eadb4690b5e96 + languageName: node + linkType: hard + +"@types/lodash@npm:*": + version: 4.17.5 + resolution: "@types/lodash@npm:4.17.5" + checksum: 10c0/55924803ed853e72261512bd3eaf2c5b16558c3817feb0a3125ef757afe46e54b86f33d1960e40b7606c0ddab91a96f47966bf5e6006b7abfd8994c13b04b19b + languageName: node + linkType: hard + "@types/mime@npm:^1": version: 1.3.5 resolution: "@types/mime@npm:1.3.5" @@ -2941,6 +2957,7 @@ __metadata: "@types/inquirer": "npm:^9.0.7" "@types/is-ci": "npm:^3.0.4" "@types/jju": "npm:^1.4.5" + "@types/lodash.clonedeep": "npm:^4" "@types/mime": "npm:^4.0.0" "@types/node": "npm:^20.11.20" "@types/semver": "npm:^7.5.8" @@ -2970,6 +2987,7 @@ __metadata: istextorbinary: "npm:~9.5.0" jju: "npm:~1.4.0" load-json-file: "npm:~7.0.1" + lodash.clonedeep: "npm:^4.5.0" mime: "npm:~4.0.1" mixpanel: "npm:~0.18.0" oclif: "npm:^4.4.18" @@ -6712,6 +6730,13 @@ __metadata: languageName: node linkType: hard +"lodash.clonedeep@npm:^4.5.0": + version: 4.5.0 + resolution: "lodash.clonedeep@npm:4.5.0" + checksum: 10c0/2caf0e4808f319d761d2939ee0642fa6867a4bbf2cfce43276698828380756b99d4c4fa226d881655e6ac298dd453fe12a5ec8ba49861777759494c534936985 + languageName: node + linkType: hard + "lodash.isequal@npm:^4.5.0": version: 4.5.0 resolution: "lodash.isequal@npm:4.5.0" From 49da417533d22268edd4e3f956f069d7de73f5f4 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 21 Jun 2024 19:01:26 +0300 Subject: [PATCH 4/5] chore: bleh --- src/commands/run.ts | 59 ++++++++++++++++++++++++++------------------- src/lib/consts.ts | 2 ++ 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/commands/run.ts b/src/commands/run.ts index 79cadd432..123b3c461 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -279,14 +279,14 @@ export class RunCommand extends ApifyCommand { // Increases default size of headers. The original limit was 80kb, but from node 10+ they decided to lower it to 8kb. // However they did not think about all the sites there with large headers, // so we put back the old limit of 80kb, which seems to work just fine. - if (isNodeVersionSupported(currentNodeVersion)) { - env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000'; - } else { - warning({ - message: `You are running Node.js version ${currentNodeVersion}, which is no longer supported. ` + if (isNodeVersionSupported(currentNodeVersion)) { + env.NODE_OPTIONS = env.NODE_OPTIONS ? `${env.NODE_OPTIONS} --max-http-header-size=80000` : '--max-http-header-size=80000'; + } else { + warning({ + message: `You are running Node.js version ${currentNodeVersion}, which is no longer supported. ` + `Please upgrade to Node.js version ${minimumSupportedNodeVersion} or later.`, - }); - } + }); + } this.telemetryData.actorNodejsVersion = currentNodeVersion; this.telemetryData.actorLanguage = LANGUAGE.NODEJS; @@ -310,18 +310,21 @@ export class RunCommand extends ApifyCommand { + 'For more information about that call "apify help run".'); } - await execWithLog(getNpmCmd(), ['run', entrypoint], { env, cwd }); + await execWithLog(getNpmCmd(), ['run', entrypoint], { env, cwd }); + } + } else { + error({ + message: + `No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js Actors locally.`, + }); } - } else { - error(`No Node.js detected! Please install Node.js ${minimumSupportedNodeVersion} or higher to be able to run Node.js Actors locally.`); - } - } else if (language === LANGUAGE.PYTHON) { - const pythonVersion = languageVersion; - this.telemetryData.actorPythonVersion = pythonVersion; - this.telemetryData.actorLanguage = LANGUAGE.PYTHON; - if (pythonVersion) { - if (isPythonVersionSupported(pythonVersion)) { - const pythonCommand = getPythonCommand(cwd); + } else if (language === LANGUAGE.PYTHON) { + const pythonVersion = languageVersion; + this.telemetryData.actorPythonVersion = pythonVersion; + this.telemetryData.actorLanguage = LANGUAGE.PYTHON; + if (pythonVersion) { + if (isPythonVersionSupported(pythonVersion)) { + const pythonCommand = getPythonCommand(cwd); if (isScrapyProject && !this.flags.entrypoint) { const project = new ScrapyProjectAnalyzer(cwd); @@ -331,17 +334,23 @@ export class RunCommand extends ApifyCommand { } } - if (runType === RunType.Module) { - await execWithLog(pythonCommand, ['-m', entrypoint], { env, cwd }); + if (runType === RunType.Module) { + await execWithLog(pythonCommand, ['-m', entrypoint], { env, cwd }); + } else { + await execWithLog(pythonCommand, [entrypoint], { env, cwd }); + } } else { - await execWithLog(pythonCommand, [entrypoint], { env, cwd }); + error({ message: `Python Actors require Python 3.8 or higher, but you have Python ${pythonVersion}!` }); + error({ message: 'Please install Python 3.8 or higher to be able to run Python Actors locally.' }); } } else { - error(`Python Actors require Python 3.8 or higher, but you have Python ${pythonVersion}!`); - error('Please install Python 3.8 or higher to be able to run Python Actors locally.'); + error({ message: 'No Python detected! Please install Python 3.8 or higher to be able to run Python Actors locally.' }); } - } else { - error('No Python detected! Please install Python 3.8 or higher to be able to run Python Actors locally.'); + } + } finally { + // Delete the temporary input file (but maybe we should keep it?) + if (envVariableOverride) { + await deleteFile(join(process.cwd(), getLocalKeyValueStorePath(), `${envVariableOverride}.json`)); } } } diff --git a/src/lib/consts.ts b/src/lib/consts.ts index d9c6e1459..34de08b60 100644 --- a/src/lib/consts.ts +++ b/src/lib/consts.ts @@ -96,4 +96,6 @@ export enum CommandExitCodes { RunAborted = 3, NoFilesToPush = 4, + + InvalidInput = 5, } From 151249b33bd730db4f8b0d13f0a5f66b05d94ce6 Mon Sep 17 00:00:00 2001 From: Vlad Frangu Date: Fri, 21 Jun 2024 20:03:26 +0300 Subject: [PATCH 5/5] chore: add tests --- src/commands/run.ts | 5 +- test/__setup__/hooks/useProcessCwdMock.ts | 13 +++ test/__setup__/input-schemas/defaults.json | 24 +++++ .../missing-required-property.json | 17 +++ test/__setup__/input-schemas/prefills.json | 24 +++++ test/commands/run.test.ts | 101 +++++++++++++++++- 6 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 test/__setup__/input-schemas/defaults.json create mode 100644 test/__setup__/input-schemas/missing-required-property.json create mode 100644 test/__setup__/input-schemas/prefills.json diff --git a/src/commands/run.ts b/src/commands/run.ts index 123b3c461..a961129cc 100644 --- a/src/commands/run.ts +++ b/src/commands/run.ts @@ -383,7 +383,10 @@ export class RunCommand extends ApifyCommand { const errors = validateInputUsingValidator(compiledInputSchema, inputSchema, fullInputOverride); if (errors.length > 0) { - throw new Error(`The input provided is invalid. Please fix the following errors:\n${errors.map((e) => ` - ${e.message}`).join('\n')}`); + throw new Error(`The input provided is invalid. Please fix the following errors:\n${ + errors.map((e) => ` - ${ + e.message.replace('Field input.', 'Field ') + }`).join('\n')}`); } await writeFile(inputFilePath, JSON.stringify(fullInputOverride, null, 2)); diff --git a/test/__setup__/hooks/useProcessCwdMock.ts b/test/__setup__/hooks/useProcessCwdMock.ts index e8c1630d5..5afeb4553 100644 --- a/test/__setup__/hooks/useProcessCwdMock.ts +++ b/test/__setup__/hooks/useProcessCwdMock.ts @@ -12,6 +12,19 @@ export function useProcessCwdMock(cwdMock: () => string) { }; }); + vitest.doMock('process', async (importActual) => { + const actual = await importActual(); + + return { + ...actual, + cwd: cwdMock, + default: { + ...actual, + cwd: cwdMock, + }, + }; + }); + const processCwdSpy = vitest.spyOn(process, 'cwd'); processCwdSpy.mockImplementation(cwdMock); } diff --git a/test/__setup__/input-schemas/defaults.json b/test/__setup__/input-schemas/defaults.json new file mode 100644 index 000000000..ad888c52b --- /dev/null +++ b/test/__setup__/input-schemas/defaults.json @@ -0,0 +1,24 @@ +{ + "title": "Defaults", + "description": "Ensures defaults also get filled into the input", + "type": "object", + "schemaVersion": 1, + "properties": { + "awesome": { + "title": "Are you awesome", + "type": "boolean", + "description": "yesnt", + "editor": "checkbox" + }, + "help": { + "title": "optional", + "type": "string", + "description": "A message, stop looking in these files", + "default": "this_maze_is_not_meant_for_you", + "editor": "textfield" + } + }, + "required": [ + "awesome" + ] +} diff --git a/test/__setup__/input-schemas/missing-required-property.json b/test/__setup__/input-schemas/missing-required-property.json new file mode 100644 index 000000000..d0946217b --- /dev/null +++ b/test/__setup__/input-schemas/missing-required-property.json @@ -0,0 +1,17 @@ +{ + "title": "required", + "description": "Ensures cli throws when required fields are missing", + "type": "object", + "schemaVersion": 1, + "properties": { + "awesome": { + "title": "Are you awesome", + "type": "boolean", + "description": "yesnt", + "editor": "checkbox" + } + }, + "required": [ + "awesome" + ] +} diff --git a/test/__setup__/input-schemas/prefills.json b/test/__setup__/input-schemas/prefills.json new file mode 100644 index 000000000..c95263ac2 --- /dev/null +++ b/test/__setup__/input-schemas/prefills.json @@ -0,0 +1,24 @@ +{ + "title": "Defaults", + "description": "Ensures defaults also get filled into the input", + "type": "object", + "schemaVersion": 1, + "properties": { + "awesome": { + "title": "Are you awesome", + "type": "boolean", + "description": "yesnt", + "editor": "checkbox" + }, + "help": { + "title": "optional", + "type": "string", + "description": "A message, stop looking in these files", + "prefill": "this_maze_is_not_meant_for_you", + "editor": "textfield" + } + }, + "required": [ + "awesome" + ] +} diff --git a/test/commands/run.test.ts b/test/commands/run.test.ts index 81bae00a1..34f2b2dd5 100644 --- a/test/commands/run.test.ts +++ b/test/commands/run.test.ts @@ -1,6 +1,8 @@ -import { existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { copyFileSync, existsSync, readFileSync, writeFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { APIFY_ENV_VARS } from '@apify/consts'; +import { captureOutput } from '@oclif/test'; import { loadJsonFileSync } from 'load-json-file'; import { writeJsonFileSync } from 'write-json-file'; @@ -12,6 +14,21 @@ import { useAuthSetup } from '../__setup__/hooks/useAuthSetup.js'; import { useTempPath } from '../__setup__/hooks/useTempPath.js'; const actName = 'run-my-actor'; +const pathToDefaultsInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/defaults.json', import.meta.url)); +const pathToMissingRequiredInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/missing-required-property.json', import.meta.url)); +const pathToPrefillsInputSchema = fileURLToPath(new URL('../__setup__/input-schemas/prefills.json', import.meta.url)); + +const INPUT_SCHEMA_ACTOR_SRC = ` +import { Actor } from 'apify'; + +Actor.main(async () => { + const input = await Actor.getInput(); + + await Actor.setValue('OUTPUT', input); + + console.log('Done.'); +}); +`; useAuthSetup({ perTest: true }); @@ -233,4 +250,86 @@ describe('apify run', () => { throw new Error('Can not run Actor without storage folder!'); } }); + + describe('input tests', () => { + const actPath = joinPath('src/main.js'); + const inputSchemaPath = joinPath('INPUT_SCHEMA.json'); + const inputPath = joinPath(getLocalKeyValueStorePath(), 'INPUT.json'); + const outputPath = joinPath(getLocalKeyValueStorePath(), 'OUTPUT.json'); + const handPassedInput = JSON.stringify({ awesome: null }); + + beforeAll(() => { + writeFileSync(actPath, INPUT_SCHEMA_ACTOR_SRC, { flag: 'w' }); + }); + + it('throws when required field is not provided', async () => { + writeFileSync(inputPath, '{}', { flag: 'w' }); + copyFileSync(pathToMissingRequiredInputSchema, inputSchemaPath); + + const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url)); + + expect(error).toBeDefined(); + expect(error!.message).toMatch(/Field awesome is required/i); + }); + + it('throws when required field has wrong type', async () => { + writeFileSync(inputPath, '{"awesome": 42}', { flag: 'w' }); + copyFileSync(pathToDefaultsInputSchema, inputSchemaPath); + + const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url)); + + expect(error).toBeDefined(); + expect(error!.message).toMatch(/Field awesome must be boolean/i); + }); + + it('throws when passing manual input, but local file has correct input', async () => { + writeFileSync(inputPath, '{"awesome": true}', { flag: 'w' }); + copyFileSync(pathToDefaultsInputSchema, inputSchemaPath); + + const { error } = await captureOutput(async () => RunCommand.run(['--input', handPassedInput], import.meta.url)); + + expect(error).toBeDefined(); + expect(error!.message).toMatch(/Field awesome must be boolean/i); + }); + + it('throws when input has default field of wrong type', async () => { + writeFileSync(inputPath, '{"awesome": true, "help": 123}', { flag: 'w' }); + copyFileSync(pathToDefaultsInputSchema, inputSchemaPath); + + const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url)); + + expect(error).toBeDefined(); + expect(error!.message).toMatch(/Field help must be string/i); + }); + + it('throws when input has prefilled field of wrong type', async () => { + writeFileSync(inputPath, '{"awesome": true, "help": 123}', { flag: 'w' }); + copyFileSync(pathToPrefillsInputSchema, inputSchemaPath); + + const { error } = await captureOutput(async () => RunCommand.run([], import.meta.url)); + + expect(error).toBeDefined(); + expect(error!.message).toMatch(/Field help must be string/i); + }); + + it('automatically inserts missing defaulted fields', async () => { + writeFileSync(inputPath, '{"awesome": true}', { flag: 'w' }); + copyFileSync(pathToDefaultsInputSchema, inputSchemaPath); + + await RunCommand.run([], import.meta.url); + + const output = loadJsonFileSync(outputPath); + expect(output).toStrictEqual({ awesome: true, help: 'this_maze_is_not_meant_for_you' }); + }); + + it('automatically inserts missing prefilled fields', async () => { + writeFileSync(inputPath, '{"awesome": true}', { flag: 'w' }); + copyFileSync(pathToPrefillsInputSchema, inputSchemaPath); + + await RunCommand.run([], import.meta.url); + + const output = loadJsonFileSync(outputPath); + expect(output).toStrictEqual({ awesome: true, help: 'this_maze_is_not_meant_for_you' }); + }); + }); });