From ad2aa9fba4a02022870e1cd1336459c0258a528a Mon Sep 17 00:00:00 2001 From: "Alex Rock (Koala)" Date: Fri, 27 Sep 2024 12:10:12 -0600 Subject: [PATCH 01/12] koala: initial commit --- validators/NumberValidator/README.MD | 75 ++++++++++ validators/NumberValidator/metadata.json | 63 +++++++++ validators/NumberValidator/package.json | 64 +++++++++ validators/NumberValidator/rollup.config.mjs | 5 + validators/NumberValidator/src/index.ts | 139 +++++++++++++++++++ 5 files changed, 346 insertions(+) create mode 100644 validators/NumberValidator/README.MD create mode 100644 validators/NumberValidator/metadata.json create mode 100644 validators/NumberValidator/package.json create mode 100644 validators/NumberValidator/rollup.config.mjs create mode 100644 validators/NumberValidator/src/index.ts diff --git a/validators/NumberValidator/README.MD b/validators/NumberValidator/README.MD new file mode 100644 index 000000000..cb09c37a4 --- /dev/null +++ b/validators/NumberValidator/README.MD @@ -0,0 +1,75 @@ +# Flatfile Number Validation Plugin + +This Flatfile Listener plugin provides comprehensive number validation capabilities for your data import processes. It offers a wide range of validation options to ensure that numeric data meets specific criteria before being accepted. + +## Features + +- Min/Max value validation +- Integer-only validation +- Precision and scale validation +- Currency formatting validation +- Step (increment) validation +- Special number type validation (prime, even, odd) +- Rounding and truncation options +- Customizable thousands separator and decimal point +- Inclusive/exclusive range validation + +## Installation + +To install the plugin, use npm: + +```bash +npm install @flatfile/plugin-number-validation +``` + +## Example Usage + +```javascript +import { FlatfileListener } from '@flatfile/listener'; +import { numberValidationPlugin } from '@flatfile/plugin-number-validation'; + +const listener = new FlatfileListener(); + +listener.use( + numberValidationPlugin({ + min: 0, + max: 1000, + inclusive: true, + integerOnly: true, + precision: 10, + scale: 2, + currency: true, + step: 5, + thousandsSeparator: ',', + decimalPoint: '.', + specialTypes: ['even'], + round: true + }) +); +``` + +## Configuration + +The `numberValidationPlugin` accepts a configuration object with the following options: + +- `min`: Minimum allowed value +- `max`: Maximum allowed value +- `inclusive`: Whether the min/max range is inclusive +- `integerOnly`: Allow only integer values +- `precision`: Total number of digits allowed +- `scale`: Number of decimal places allowed +- `currency`: Validate as a currency value +- `step`: Increment value (e.g., multiples of 5) +- `thousandsSeparator`: Character used as thousands separator +- `decimalPoint`: Character used as decimal point +- `specialTypes`: Array of special number types to validate ('prime', 'even', 'odd') +- `round`: Round the number to the nearest integer +- `truncate`: Truncate the decimal part of the number + +## Behavior + +The plugin listens for the 'record:created' event and performs validation on a specified number field (default field name is 'numberField'). It applies the configured validations and throws an error if any validation fails. + +The plugin also handles number parsing, considering the specified thousands separator and decimal point. It can optionally round or truncate the number before applying validations. + +If all validations pass, the plugin logs a success message. If any validation fails, it logs an error message and throws an error with a descriptive message about the validation failure. \ No newline at end of file diff --git a/validators/NumberValidator/metadata.json b/validators/NumberValidator/metadata.json new file mode 100644 index 000000000..b8b3be515 --- /dev/null +++ b/validators/NumberValidator/metadata.json @@ -0,0 +1,63 @@ +{ + "timestamp": "2024-09-27T17-59-35-891Z", + "task": "Create a Number Validator Flatfile Listener plugin:\n - Implement range validation for numbers (min, max, inclusive/exclusive)\n - Add support for integer-only validation\n - Implement precision and scale validation for decimal numbers\n - Add options for handling currency values (e.g., two decimal places)\n - Implement custom step validation (e.g., multiples of 0.5)\n - Add support for scientific notation and exponential numbers\n - Implement options for thousands separators and decimal points\n - Add validation for special number types (e.g., prime, even/odd)\n - Implement custom error messages for different validation failures\n - Add options for rounding and truncation of numbers", + "summary": "This is a Flatfile Listener plugin for number validation. It includes various validation options such as min/max values, precision, scale, currency formatting, special number types, and options for rounding and truncation. The plugin listens to the 'record:created' event and performs validation on a specified number field.", + "steps": [ + [ + "Retrieve the basic structure of a Flatfile Listener plugin and identify the appropriate event topic for number validation.\n", + "#E1", + "PineconeAssistant", + "Provide the basic structure of a Flatfile Listener plugin for number validation, including the correct event topic to use.", + "Plan: Retrieve the basic structure of a Flatfile Listener plugin and identify the appropriate event topic for number validation.\n#E1 = PineconeAssistant[Provide the basic structure of a Flatfile Listener plugin for number validation, including the correct event topic to use.]" + ], + [ + "Implement range validation for numbers (min, max, inclusive/exclusive) and integer-only validation.\n", + "#E2", + "PineconeAssistant", + "Add code to the Flatfile Listener from #E1 to implement range validation (min, max, inclusive/exclusive) and integer-only validation for numbers.", + "Plan: Implement range validation for numbers (min, max, inclusive/exclusive) and integer-only validation.\n#E2 = PineconeAssistant[Add code to the Flatfile Listener from #E1 to implement range validation (min, max, inclusive/exclusive) and integer-only validation for numbers.]" + ], + [ + "Implement precision and scale validation for decimal numbers and add options for handling currency values.\n", + "#E3", + "PineconeAssistant", + "Extend the Flatfile Listener from #E2 to include precision and scale validation for decimal numbers, and add options for handling currency values (e.g., two decimal places).", + "Plan: Implement precision and scale validation for decimal numbers and add options for handling currency values.\n#E3 = PineconeAssistant[Extend the Flatfile Listener from #E2 to include precision and scale validation for decimal numbers, and add options for handling currency values (e.g., two decimal places).]" + ], + [ + "Implement custom step validation and support for scientific notation and exponential numbers.\n", + "#E4", + "PineconeAssistant", + "Add code to the Flatfile Listener from #E3 to implement custom step validation (e.g., multiples of 0.5) and support for scientific notation and exponential numbers.", + "Plan: Implement custom step validation and support for scientific notation and exponential numbers.\n#E4 = PineconeAssistant[Add code to the Flatfile Listener from #E3 to implement custom step validation (e.g., multiples of 0.5) and support for scientific notation and exponential numbers.]" + ], + [ + "Implement options for thousands separators and decimal points, and add validation for special number types.\n", + "#E5", + "PineconeAssistant", + "Extend the Flatfile Listener from #E4 to include options for thousands separators and decimal points, and add validation for special number types (e.g., prime, even/odd).", + "Plan: Implement options for thousands separators and decimal points, and add validation for special number types.\n#E5 = PineconeAssistant[Extend the Flatfile Listener from #E4 to include options for thousands separators and decimal points, and add validation for special number types (e.g., prime, even/odd).]" + ], + [ + "Implement custom error messages for different validation failures and add options for rounding and truncation of numbers.\n", + "#E6", + "PineconeAssistant", + "Complete the Flatfile Listener from #E5 by implementing custom error messages for different validation failures and adding options for rounding and truncation of numbers.", + "Plan: Implement custom error messages for different validation failures and add options for rounding and truncation of numbers.\n#E6 = PineconeAssistant[Complete the Flatfile Listener from #E5 by implementing custom error messages for different validation failures and adding options for rounding and truncation of numbers.]" + ], + [ + "Review and finalize the Number Validator Flatfile Listener plugin.\n", + "#E7", + "LLM", + "Review the complete Number Validator Flatfile Listener plugin from #E6. Ensure all requirements are met, the code is valid, and only valid Event Topics are used. Remove any unused imports or plugins. Provide the final, optimized version of the plugin.", + "Plan: Review and finalize the Number Validator Flatfile Listener plugin.\n#E7 = LLM[Review the complete Number Validator Flatfile Listener plugin from #E6. Ensure all requirements are met, the code is valid, and only valid Event Topics are used. Remove any unused imports or plugins. Provide the final, optimized version of the plugin.]" + ] + ], + "metrics": { + "tokens": { + "plan": 4508, + "state": 5744, + "total": 10252 + } + } +} \ No newline at end of file diff --git a/validators/NumberValidator/package.json b/validators/NumberValidator/package.json new file mode 100644 index 000000000..a758c525c --- /dev/null +++ b/validators/NumberValidator/package.json @@ -0,0 +1,64 @@ +{ + "name": "@flatfile/plugin-validate-number", + "version": "1.0.0", + "description": "A Flatfile Listener plugin for number validation with various options", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "browser": { + "./dist/index.js": "./dist/index.browser.js", + "./dist/index.mjs": "./dist/index.browser.mjs" + }, + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "browser": { + "require": "./dist/index.browser.js", + "import": "./dist/index.browser.mjs" + }, + "default": "./dist/index.mjs" + }, + "source": "./src/index.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + }, + "keywords": [ + "flatfile", + "plugin", + "number", + "validation", + "flatfile-plugins", + "category-transform" + ], + "author": "Your Name", + "license": "MIT", + "dependencies": { + "@flatfile/plugin-record-hook": "^1.7.0" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + }, + "devDependencies": { + "@flatfile/rollup-config": "^0.1.1" + }, + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/number-validation" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ] +} diff --git a/validators/NumberValidator/rollup.config.mjs b/validators/NumberValidator/rollup.config.mjs new file mode 100644 index 000000000..fafa813c6 --- /dev/null +++ b/validators/NumberValidator/rollup.config.mjs @@ -0,0 +1,5 @@ +import { buildConfig } from '@flatfile/rollup-config' + +const config = buildConfig({}) + +export default config diff --git a/validators/NumberValidator/src/index.ts b/validators/NumberValidator/src/index.ts new file mode 100644 index 000000000..a7a5796a4 --- /dev/null +++ b/validators/NumberValidator/src/index.ts @@ -0,0 +1,139 @@ +import { FlatfileListener, FlatfileEvent } from '@flatfile/listener' +import { logInfo, logError } from '@flatfile/util-common' + +interface NumberValidationConfig { + min?: number + max?: number + inclusive?: boolean + integerOnly?: boolean + precision?: number + scale?: number + currency?: boolean + step?: number + thousandsSeparator?: string + decimalPoint?: string + specialTypes?: string[] + round?: boolean + truncate?: boolean +} + +export function numberValidationPlugin(config: NumberValidationConfig) { + return (listener: FlatfileListener) => { + listener.use((handler) => { + handler.on('record:created', async (event: FlatfileEvent) => { + const { record } = event.payload + const numberField = record.get('numberField') // Replace 'numberField' with your actual field name + + try { + let numberValue = numberField + .replace(config.thousandsSeparator || ',', '') + .replace(config.decimalPoint || '.', '.') + numberValue = parseFloat(numberValue) + + if (isNaN(numberValue)) { + throw new Error('The field must be a number') + } + + if (config.round) { + numberValue = Math.round(numberValue) + } + + if (config.truncate) { + numberValue = Math.trunc(numberValue) + } + + if (config.integerOnly && !Number.isInteger(numberValue)) { + throw new Error('The field must be an integer') + } + + if (config.min !== undefined) { + if ( + config.inclusive + ? numberValue < config.min + : numberValue <= config.min + ) { + throw new Error( + `The field must be greater than ${ + config.inclusive ? 'or equal to ' : '' + }${config.min}` + ) + } + } + + if (config.max !== undefined) { + if ( + config.inclusive + ? numberValue > config.max + : numberValue >= config.max + ) { + throw new Error( + `The field must be less than ${ + config.inclusive ? 'or equal to ' : '' + }${config.max}` + ) + } + } + + if (config.precision !== undefined && config.scale !== undefined) { + const [integerPart, decimalPart] = numberValue.toString().split('.') + if (integerPart.length > config.precision - config.scale) { + throw new Error( + `The field must have at most ${ + config.precision - config.scale + } digits before the decimal point` + ) + } + if (decimalPart && decimalPart.length > config.scale) { + throw new Error( + `The field must have at most ${config.scale} digits after the decimal point` + ) + } + } + + if ( + config.currency && + !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) + ) { + throw new Error( + 'The field must be a valid currency value with at most two decimal places' + ) + } + + if (config.step !== undefined && numberValue % config.step !== 0) { + throw new Error(`The field must be a multiple of ${config.step}`) + } + + if (config.specialTypes) { + if ( + config.specialTypes.includes('prime') && + !isPrime(numberValue) + ) { + throw new Error('The field must be a prime number') + } + if (config.specialTypes.includes('even') && numberValue % 2 !== 0) { + throw new Error('The field must be an even number') + } + if (config.specialTypes.includes('odd') && numberValue % 2 === 0) { + throw new Error('The field must be an odd number') + } + } + + logInfo('number-validation', 'Number validation passed') + } catch (error) { + logError( + 'number-validation', + `Error processing event: ${error.message}` + ) + throw error + } + }) + }) + } +} + +function isPrime(num: number): boolean { + for (let i = 2, sqrt = Math.sqrt(num); i <= sqrt; i++) { + if (num % i === 0) return false + } + return num > 1 +} From 723b01ee3a735c2bc59c52883d70f0e70307d779 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Tue, 1 Oct 2024 01:11:09 -0600 Subject: [PATCH 02/12] feat: add tests --- .../number}/README.MD | 10 +- .../number}/metadata.json | 0 .../number}/package.json | 0 .../number}/rollup.config.mjs | 0 .../number/src/ValidateNumber.e2e.spec.ts | 191 ++++++++++++++++++ validate/number/src/index.ts | 165 +++++++++++++++ validators/NumberValidator/src/index.ts | 139 ------------- 7 files changed, 361 insertions(+), 144 deletions(-) rename {validators/NumberValidator => validate/number}/README.MD (87%) rename {validators/NumberValidator => validate/number}/metadata.json (100%) rename {validators/NumberValidator => validate/number}/package.json (100%) rename {validators/NumberValidator => validate/number}/rollup.config.mjs (100%) create mode 100644 validate/number/src/ValidateNumber.e2e.spec.ts create mode 100644 validate/number/src/index.ts delete mode 100644 validators/NumberValidator/src/index.ts diff --git a/validators/NumberValidator/README.MD b/validate/number/README.MD similarity index 87% rename from validators/NumberValidator/README.MD rename to validate/number/README.MD index cb09c37a4..7189e3917 100644 --- a/validators/NumberValidator/README.MD +++ b/validate/number/README.MD @@ -26,12 +26,12 @@ npm install @flatfile/plugin-number-validation ```javascript import { FlatfileListener } from '@flatfile/listener'; -import { numberValidationPlugin } from '@flatfile/plugin-number-validation'; +import { validateNumber } from '@flatfile/plugin-validate-number'; const listener = new FlatfileListener(); listener.use( - numberValidationPlugin({ + validateNumber({ min: 0, max: 1000, inclusive: true, @@ -50,7 +50,7 @@ listener.use( ## Configuration -The `numberValidationPlugin` accepts a configuration object with the following options: +The `validateNumber` accepts a configuration object with the following options: - `min`: Minimum allowed value - `max`: Maximum allowed value @@ -68,8 +68,8 @@ The `numberValidationPlugin` accepts a configuration object with the following o ## Behavior -The plugin listens for the 'record:created' event and performs validation on a specified number field (default field name is 'numberField'). It applies the configured validations and throws an error if any validation fails. +The plugin listens for the 'commit:created' event and performs validation on a specified number field (default field name is 'numberField'). It applies the configured validations and throws an error if any validation fails. The plugin also handles number parsing, considering the specified thousands separator and decimal point. It can optionally round or truncate the number before applying validations. -If all validations pass, the plugin logs a success message. If any validation fails, it logs an error message and throws an error with a descriptive message about the validation failure. \ No newline at end of file +If all validations pass, the plugin logs a success message. If any validation fails, it logs an error message and throws an error with a descriptive message about the validation failure. diff --git a/validators/NumberValidator/metadata.json b/validate/number/metadata.json similarity index 100% rename from validators/NumberValidator/metadata.json rename to validate/number/metadata.json diff --git a/validators/NumberValidator/package.json b/validate/number/package.json similarity index 100% rename from validators/NumberValidator/package.json rename to validate/number/package.json diff --git a/validators/NumberValidator/rollup.config.mjs b/validate/number/rollup.config.mjs similarity index 100% rename from validators/NumberValidator/rollup.config.mjs rename to validate/number/rollup.config.mjs diff --git a/validate/number/src/ValidateNumber.e2e.spec.ts b/validate/number/src/ValidateNumber.e2e.spec.ts new file mode 100644 index 000000000..2332fb3c4 --- /dev/null +++ b/validate/number/src/ValidateNumber.e2e.spec.ts @@ -0,0 +1,191 @@ +import { FlatfileClient } from '@flatfile/api' +import { + createRecords, + deleteSpace, + getRecords, + setupListener, + setupSimpleWorkbook, + setupSpace, +} from '@flatfile/utils-testing' +import { validateNumber } from './index' + +const api = new FlatfileClient() + +describe('ValidateNumber e2e', () => { + const listener = setupListener() + + let spaceId: string + let sheetId: string + + beforeAll(async () => { + const space = await setupSpace() + spaceId = space.id + const workbook = await setupSimpleWorkbook(space.id, [ + { key: 'amount', type: 'number' }, + { key: 'price', type: 'number' }, + { key: 'quantity', type: 'number' }, + ]) + sheetId = workbook.sheets![0].id + }) + + afterAll(async () => { + await deleteSpace(spaceId) + }) + + afterEach(async () => { + listener.reset() + const records = await getRecords(sheetId) + if (records.length > 0) { + const ids = records.map((record) => record.id) + await api.records.delete(sheetId, { ids }) + } + }) + + describe('validateNumber()', () => { + it('validates min and max values', async () => { + listener.use( + validateNumber({ + fields: ['amount'], + min: 0, + max: 100, + }) + ) + + await createRecords(sheetId, [ + { amount: -10 }, + { amount: 50 }, + { amount: 150 }, + ]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].valid).toBeFalsy() + expect(records[1].valid).toBeTruthy() + expect(records[2].valid).toBeFalsy() + }) + + it('validates integer-only values', async () => { + listener.use( + validateNumber({ + fields: ['quantity'], + integerOnly: true, + }) + ) + + await createRecords(sheetId, [{ quantity: 5 }, { quantity: 3.14 }]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].valid).toBeTruthy() + expect(records[1].valid).toBeFalsy() + }) + + it('validates precision and scale', async () => { + listener.use( + validateNumber({ + fields: ['price'], + precision: 5, + scale: 2, + }) + ) + + await createRecords(sheetId, [ + { price: 123.45 }, + { price: 1234.56 }, + { price: 12.345 }, + ]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].valid).toBeTruthy() + expect(records[1].valid).toBeFalsy() + expect(records[2].valid).toBeFalsy() + }) + + it('validates step values', async () => { + listener.use( + validateNumber({ + fields: ['amount'], + step: 0.5, + }) + ) + + await createRecords(sheetId, [ + { amount: 1.5 }, + { amount: 2.25 }, + { amount: 3 }, + ]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].valid).toBeTruthy() + expect(records[1].valid).toBeFalsy() + expect(records[2].valid).toBeTruthy() + }) + + it('handles currency values', async () => { + listener.use( + validateNumber({ + fields: ['price'], + currency: true, + }) + ) + + await createRecords(sheetId, [ + { price: 10.0 }, + { price: 15.5 }, + { price: 20.555 }, + ]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].valid).toBeTruthy() + expect(records[1].valid).toBeTruthy() + expect(records[2].valid).toBeFalsy() + }) + + it('validates special number types', async () => { + listener.use( + validateNumber({ + fields: ['amount'], + specialTypes: ['even'], + }) + ) + + await createRecords(sheetId, [ + { amount: 2 }, + { amount: 3 }, + { amount: 4 }, + ]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].valid).toBeTruthy() + expect(records[1].valid).toBeFalsy() + expect(records[2].valid).toBeTruthy() + }) + + it('handles rounding', async () => { + listener.use( + validateNumber({ + fields: ['amount'], + round: true, + }) + ) + + await createRecords(sheetId, [{ amount: 1.4 }, { amount: 1.6 }]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].values.amount.value).toBe(1) + expect(records[1].values.amount.value).toBe(2) + }) + }) +}) diff --git a/validate/number/src/index.ts b/validate/number/src/index.ts new file mode 100644 index 000000000..50ad5c5a5 --- /dev/null +++ b/validate/number/src/index.ts @@ -0,0 +1,165 @@ +import { FlatfileListener, FlatfileEvent } from '@flatfile/listener' +import { logInfo, logError } from '@flatfile/util-common' +import { recordHook } from '@flatfile/plugin-record-hook' + +interface NumberValidationConfig { + fields: string[] // Specify fields to validate + min?: number + max?: number + inclusive?: boolean + integerOnly?: boolean + precision?: number + scale?: number + currency?: boolean + step?: number + thousandsSeparator?: string + decimalPoint?: string + specialTypes?: string[] + round?: boolean + truncate?: boolean + sheetSlug?: string // Specify the sheet slug +} + +export function validateNumber(config: NumberValidationConfig) { + return (listener: FlatfileListener) => { + listener.use( + recordHook( + config.sheetSlug || '**', + async (record, event: FlatfileEvent) => { + for (const field of config.fields) { + const numberField = record.get(field) + + try { + let numberValue: string | number = String(numberField) + .replace(config.thousandsSeparator || ',', '') + .replace(config.decimalPoint || '.', '.') + numberValue = parseFloat(numberValue) + + if (isNaN(numberValue)) { + throw new Error(`The field '${field}' must be a number`) + } + + if (config.round) { + numberValue = Math.round(numberValue) + } + + if (config.truncate) { + numberValue = Math.trunc(numberValue) + } + + if (config.integerOnly && !Number.isInteger(numberValue)) { + throw new Error(`The field '${field}' must be an integer`) + } + + if (config.min !== undefined) { + if ( + config.inclusive + ? numberValue < config.min + : numberValue <= config.min + ) { + throw new Error( + `The field '${field}' must be greater than ${ + config.inclusive ? 'or equal to ' : '' + }${config.min}` + ) + } + } + + if (config.max !== undefined) { + if ( + config.inclusive + ? numberValue > config.max + : numberValue >= config.max + ) { + throw new Error( + `The field '${field}' must be less than ${ + config.inclusive ? 'or equal to ' : '' + }${config.max}` + ) + } + } + + if ( + config.precision !== undefined && + config.scale !== undefined + ) { + const [integerPart, decimalPart] = numberValue + .toString() + .split('.') + if (integerPart.length > config.precision - config.scale) { + throw new Error( + `The field '${field}' must have at most ${ + config.precision - config.scale + } digits before the decimal point` + ) + } + if (decimalPart && decimalPart.length > config.scale) { + throw new Error( + `The field '${field}' must have at most ${config.scale} digits after the decimal point` + ) + } + } + + if ( + config.currency && + !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) + ) { + throw new Error( + `The field '${field}' must be a valid currency value with at most two decimal places` + ) + } + + if ( + config.step !== undefined && + numberValue % config.step !== 0 + ) { + throw new Error( + `The field '${field}' must be a multiple of ${config.step}` + ) + } + + if (config.specialTypes) { + if ( + config.specialTypes.includes('prime') && + !isPrime(numberValue) + ) { + throw new Error(`The field '${field}' must be a prime number`) + } + if ( + config.specialTypes.includes('even') && + numberValue % 2 !== 0 + ) { + throw new Error(`The field '${field}' must be an even number`) + } + if ( + config.specialTypes.includes('odd') && + numberValue % 2 === 0 + ) { + throw new Error(`The field '${field}' must be an odd number`) + } + } + + logInfo( + 'number-validation', + `Number validation passed for field '${field}'` + ) + } catch (error) { + logError( + 'number-validation', + `Error processing event for field '${field}': ${error.message}` + ) + throw error + } + } + } + ) + ) + } +} + +export default function isPrime(num: number): boolean { + for (let i = 2, sqrt = Math.sqrt(num); i <= sqrt; i++) { + if (num % i === 0) return false + } + return num > 1 +} diff --git a/validators/NumberValidator/src/index.ts b/validators/NumberValidator/src/index.ts deleted file mode 100644 index a7a5796a4..000000000 --- a/validators/NumberValidator/src/index.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { FlatfileListener, FlatfileEvent } from '@flatfile/listener' -import { logInfo, logError } from '@flatfile/util-common' - -interface NumberValidationConfig { - min?: number - max?: number - inclusive?: boolean - integerOnly?: boolean - precision?: number - scale?: number - currency?: boolean - step?: number - thousandsSeparator?: string - decimalPoint?: string - specialTypes?: string[] - round?: boolean - truncate?: boolean -} - -export function numberValidationPlugin(config: NumberValidationConfig) { - return (listener: FlatfileListener) => { - listener.use((handler) => { - handler.on('record:created', async (event: FlatfileEvent) => { - const { record } = event.payload - const numberField = record.get('numberField') // Replace 'numberField' with your actual field name - - try { - let numberValue = numberField - .replace(config.thousandsSeparator || ',', '') - .replace(config.decimalPoint || '.', '.') - numberValue = parseFloat(numberValue) - - if (isNaN(numberValue)) { - throw new Error('The field must be a number') - } - - if (config.round) { - numberValue = Math.round(numberValue) - } - - if (config.truncate) { - numberValue = Math.trunc(numberValue) - } - - if (config.integerOnly && !Number.isInteger(numberValue)) { - throw new Error('The field must be an integer') - } - - if (config.min !== undefined) { - if ( - config.inclusive - ? numberValue < config.min - : numberValue <= config.min - ) { - throw new Error( - `The field must be greater than ${ - config.inclusive ? 'or equal to ' : '' - }${config.min}` - ) - } - } - - if (config.max !== undefined) { - if ( - config.inclusive - ? numberValue > config.max - : numberValue >= config.max - ) { - throw new Error( - `The field must be less than ${ - config.inclusive ? 'or equal to ' : '' - }${config.max}` - ) - } - } - - if (config.precision !== undefined && config.scale !== undefined) { - const [integerPart, decimalPart] = numberValue.toString().split('.') - if (integerPart.length > config.precision - config.scale) { - throw new Error( - `The field must have at most ${ - config.precision - config.scale - } digits before the decimal point` - ) - } - if (decimalPart && decimalPart.length > config.scale) { - throw new Error( - `The field must have at most ${config.scale} digits after the decimal point` - ) - } - } - - if ( - config.currency && - !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) - ) { - throw new Error( - 'The field must be a valid currency value with at most two decimal places' - ) - } - - if (config.step !== undefined && numberValue % config.step !== 0) { - throw new Error(`The field must be a multiple of ${config.step}`) - } - - if (config.specialTypes) { - if ( - config.specialTypes.includes('prime') && - !isPrime(numberValue) - ) { - throw new Error('The field must be a prime number') - } - if (config.specialTypes.includes('even') && numberValue % 2 !== 0) { - throw new Error('The field must be an even number') - } - if (config.specialTypes.includes('odd') && numberValue % 2 === 0) { - throw new Error('The field must be an odd number') - } - } - - logInfo('number-validation', 'Number validation passed') - } catch (error) { - logError( - 'number-validation', - `Error processing event: ${error.message}` - ) - throw error - } - }) - }) - } -} - -function isPrime(num: number): boolean { - for (let i = 2, sqrt = Math.sqrt(num); i <= sqrt; i++) { - if (num % i === 0) return false - } - return num > 1 -} From 81c3535be9100d52f13f6f74d0f7a14e5b276255 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Tue, 1 Oct 2024 02:29:24 -0600 Subject: [PATCH 03/12] feat: fix tests --- .../number/src/ValidateNumber.e2e.spec.ts | 95 ++++++-- validate/number/src/index.ts | 223 +++++++++--------- 2 files changed, 191 insertions(+), 127 deletions(-) diff --git a/validate/number/src/ValidateNumber.e2e.spec.ts b/validate/number/src/ValidateNumber.e2e.spec.ts index 2332fb3c4..ee6d67caa 100644 --- a/validate/number/src/ValidateNumber.e2e.spec.ts +++ b/validate/number/src/ValidateNumber.e2e.spec.ts @@ -60,9 +60,13 @@ describe('ValidateNumber e2e', () => { const records = await getRecords(sheetId) - expect(records[0].valid).toBeFalsy() - expect(records[1].valid).toBeTruthy() - expect(records[2].valid).toBeFalsy() + expect(records[0].values.amount?.messages?.[0].message).toContain( + 'must be greater than' + ) + expect(records[1].values.amount.messages).toEqual([]) + expect(records[2].values.amount.messages?.[0].message).toContain( + 'must be less than' + ) }) it('validates integer-only values', async () => { @@ -73,13 +77,20 @@ describe('ValidateNumber e2e', () => { }) ) - await createRecords(sheetId, [{ quantity: 5 }, { quantity: 3.14 }]) + await createRecords(sheetId, [ + { quantity: 5 }, + { quantity: 3.14 }, + { quantity: '3' }, + ]) await listener.waitFor('commit:created') const records = await getRecords(sheetId) - expect(records[0].valid).toBeTruthy() - expect(records[1].valid).toBeFalsy() + expect(records[0].values.quantity.messages).toEqual([]) + expect(records[1].values.quantity.messages?.[0].message).toContain( + 'must be an integer' + ) + expect(records[2].values.quantity.value).toEqual(3) }) it('validates precision and scale', async () => { @@ -100,9 +111,13 @@ describe('ValidateNumber e2e', () => { const records = await getRecords(sheetId) - expect(records[0].valid).toBeTruthy() - expect(records[1].valid).toBeFalsy() - expect(records[2].valid).toBeFalsy() + expect(records[0].values.price.messages).toEqual([]) + expect(records[1].values.price.messages?.[0].message).toContain( + 'must have at most 3 digits before the decimal point' + ) + expect(records[2].values.price.messages?.[0].message).toContain( + 'must have at most 2 digits after the decimal point' + ) }) it('validates step values', async () => { @@ -122,9 +137,11 @@ describe('ValidateNumber e2e', () => { const records = await getRecords(sheetId) - expect(records[0].valid).toBeTruthy() - expect(records[1].valid).toBeFalsy() - expect(records[2].valid).toBeTruthy() + expect(records[0].values.amount.messages).toEqual([]) + expect(records[1].values.amount.messages?.[0].message).toContain( + 'must be a multiple of 0.5' + ) + expect(records[2].values.amount.messages).toEqual([]) }) it('handles currency values', async () => { @@ -144,9 +161,11 @@ describe('ValidateNumber e2e', () => { const records = await getRecords(sheetId) - expect(records[0].valid).toBeTruthy() - expect(records[1].valid).toBeTruthy() - expect(records[2].valid).toBeFalsy() + expect(records[0].values.price.messages).toEqual([]) + expect(records[1].values.price.messages).toEqual([]) + expect(records[2].values.price.messages?.[0].message).toContain( + 'must be a valid currency value' + ) }) it('validates special number types', async () => { @@ -166,9 +185,11 @@ describe('ValidateNumber e2e', () => { const records = await getRecords(sheetId) - expect(records[0].valid).toBeTruthy() - expect(records[1].valid).toBeFalsy() - expect(records[2].valid).toBeTruthy() + expect(records[0].values.amount.messages).toEqual([]) + expect(records[1].values.amount.messages?.[0].message).toContain( + 'must be an even number' + ) + expect(records[2].values.amount.messages).toEqual([]) }) it('handles rounding', async () => { @@ -187,5 +208,43 @@ describe('ValidateNumber e2e', () => { expect(records[0].values.amount.value).toBe(1) expect(records[1].values.amount.value).toBe(2) }) + + it('handles truncation', async () => { + listener.use( + validateNumber({ + fields: ['amount'], + truncate: true, + }) + ) + + await createRecords(sheetId, [{ amount: 1.4 }, { amount: 1.6 }]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].values.amount.value).toBe(1) + expect(records[1].values.amount.value).toBe(1) + }) + + it('handles custom separators', async () => { + listener.use( + validateNumber({ + fields: ['amount'], + thousandsSeparator: '.', + decimalPoint: ',', + }) + ) + + await createRecords(sheetId, [ + { amount: '1.000,50' }, + { amount: '2,500.75' }, + ]) + await listener.waitFor('commit:created') + + const records = await getRecords(sheetId) + + expect(records[0].values.amount.value).toBe(1000.5) + expect(records[1].values.amount.value).toBe(2.50075) + }) }) }) diff --git a/validate/number/src/index.ts b/validate/number/src/index.ts index 50ad5c5a5..725696328 100644 --- a/validate/number/src/index.ts +++ b/validate/number/src/index.ts @@ -1,7 +1,5 @@ -import { FlatfileListener, FlatfileEvent } from '@flatfile/listener' -import { logInfo, logError } from '@flatfile/util-common' -import { recordHook } from '@flatfile/plugin-record-hook' - +import { FlatfileListener } from '@flatfile/listener' +import { FlatfileRecord, recordHook } from '@flatfile/plugin-record-hook' interface NumberValidationConfig { fields: string[] // Specify fields to validate min?: number @@ -23,136 +21,143 @@ interface NumberValidationConfig { export function validateNumber(config: NumberValidationConfig) { return (listener: FlatfileListener) => { listener.use( - recordHook( - config.sheetSlug || '**', - async (record, event: FlatfileEvent) => { - for (const field of config.fields) { - const numberField = record.get(field) + recordHook(config.sheetSlug || '**', async (record: FlatfileRecord) => { + for (const field of config.fields) { + const numberField = record.get(field) - try { - let numberValue: string | number = String(numberField) - .replace(config.thousandsSeparator || ',', '') - .replace(config.decimalPoint || '.', '.') - numberValue = parseFloat(numberValue) + try { + let numberValue: string | number = String(numberField) + .replace(config.thousandsSeparator || ',', '') + .replace(config.decimalPoint || '.', '.') + numberValue = parseFloat(numberValue) - if (isNaN(numberValue)) { - throw new Error(`The field '${field}' must be a number`) - } + if (isNaN(numberValue)) { + record.addWarning(field, `The field '${field}' must be a number`) + } - if (config.round) { - numberValue = Math.round(numberValue) - } + if (config.round) { + numberValue = Math.round(numberValue) + } - if (config.truncate) { - numberValue = Math.trunc(numberValue) - } + if (config.truncate) { + numberValue = Math.trunc(numberValue) + } + + if (config.integerOnly && !Number.isInteger(numberValue)) { + record.addWarning( + field, + `The field '${field}' must be an integer` + ) + } - if (config.integerOnly && !Number.isInteger(numberValue)) { - throw new Error(`The field '${field}' must be an integer`) + if (config.min !== undefined) { + if ( + config.inclusive + ? numberValue < config.min + : numberValue <= config.min + ) { + record.addWarning( + field, + `The field '${field}' must be greater than ${ + config.inclusive ? 'or equal to ' : '' + }${config.min}` + ) } + } - if (config.min !== undefined) { - if ( - config.inclusive - ? numberValue < config.min - : numberValue <= config.min - ) { - throw new Error( - `The field '${field}' must be greater than ${ - config.inclusive ? 'or equal to ' : '' - }${config.min}` - ) - } + if (config.max !== undefined) { + if ( + config.inclusive + ? numberValue > config.max + : numberValue >= config.max + ) { + record.addWarning( + field, + `The field '${field}' must be less than ${ + config.inclusive ? 'or equal to ' : '' + }${config.max}` + ) } + } - if (config.max !== undefined) { - if ( - config.inclusive - ? numberValue > config.max - : numberValue >= config.max - ) { - throw new Error( - `The field '${field}' must be less than ${ - config.inclusive ? 'or equal to ' : '' - }${config.max}` - ) - } + if (config.precision !== undefined && config.scale !== undefined) { + const [integerPart, decimalPart] = numberValue + .toString() + .split('.') + if (integerPart.length > config.precision - config.scale) { + record.addWarning( + field, + `The field '${field}' must have at most ${ + config.precision - config.scale + } digits before the decimal point` + ) } + if (decimalPart && decimalPart.length > config.scale) { + record.addWarning( + field, + `The field '${field}' must have at most ${config.scale} digits after the decimal point` + ) + } + } + + if ( + config.currency && + !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) + ) { + record.addWarning( + field, + `The field '${field}' must be a valid currency value with at most two decimal places` + ) + } + + if (config.step !== undefined && numberValue % config.step !== 0) { + record.addWarning( + field, + `The field '${field}' must be a multiple of ${config.step}` + ) + } + if (config.specialTypes) { if ( - config.precision !== undefined && - config.scale !== undefined + config.specialTypes.includes('prime') && + !isPrime(numberValue) ) { - const [integerPart, decimalPart] = numberValue - .toString() - .split('.') - if (integerPart.length > config.precision - config.scale) { - throw new Error( - `The field '${field}' must have at most ${ - config.precision - config.scale - } digits before the decimal point` - ) - } - if (decimalPart && decimalPart.length > config.scale) { - throw new Error( - `The field '${field}' must have at most ${config.scale} digits after the decimal point` - ) - } + record.addWarning( + field, + `The field '${field}' must be a prime number` + ) } - if ( - config.currency && - !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) + config.specialTypes.includes('even') && + numberValue % 2 !== 0 ) { - throw new Error( - `The field '${field}' must be a valid currency value with at most two decimal places` + record.addWarning( + field, + `The field '${field}' must be an even number` ) } - if ( - config.step !== undefined && - numberValue % config.step !== 0 + config.specialTypes.includes('odd') && + numberValue % 2 === 0 ) { - throw new Error( - `The field '${field}' must be a multiple of ${config.step}` + record.addWarning( + field, + `The field '${field}' must be an odd number` ) } - - if (config.specialTypes) { - if ( - config.specialTypes.includes('prime') && - !isPrime(numberValue) - ) { - throw new Error(`The field '${field}' must be a prime number`) - } - if ( - config.specialTypes.includes('even') && - numberValue % 2 !== 0 - ) { - throw new Error(`The field '${field}' must be an even number`) - } - if ( - config.specialTypes.includes('odd') && - numberValue % 2 === 0 - ) { - throw new Error(`The field '${field}' must be an odd number`) - } - } - - logInfo( - 'number-validation', - `Number validation passed for field '${field}'` - ) - } catch (error) { - logError( - 'number-validation', - `Error processing event for field '${field}': ${error.message}` - ) - throw error } + record.set(field, numberValue) + + return record + } catch (error) { + record.addError( + field, + `Error processing event for field '${field}': ${error.message}` + ) + throw error } } - ) + }) ) } } From 8e78739a919afe7f8fa44a469796af47cbce0b7f Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Thu, 3 Oct 2024 16:51:49 -0600 Subject: [PATCH 04/12] feat: remove metadata.json and cleanup infocard in readme --- validate/number/README.MD | 12 +++++-- validate/number/metadata.json | 63 ----------------------------------- 2 files changed, 9 insertions(+), 66 deletions(-) delete mode 100644 validate/number/metadata.json diff --git a/validate/number/README.MD b/validate/number/README.MD index 7189e3917..4cdb11c7a 100644 --- a/validate/number/README.MD +++ b/validate/number/README.MD @@ -1,6 +1,12 @@ -# Flatfile Number Validation Plugin + +# @flatfile/plugin-validate-number -This Flatfile Listener plugin provides comprehensive number validation capabilities for your data import processes. It offers a wide range of validation options to ensure that numeric data meets specific criteria before being accepted. +The `@flatfile/plugin-validate-number` plugin provides comprehensive number validation capabilities for your data import processes. It offers a wide range of validation options to ensure that numeric data meets specific criteria before being accepted. + +**Event Type:** +`listener.on('commit:created')` + + ## Features @@ -19,7 +25,7 @@ This Flatfile Listener plugin provides comprehensive number validation capabilit To install the plugin, use npm: ```bash -npm install @flatfile/plugin-number-validation +npm install @flatfile/plugin-validate-number ``` ## Example Usage diff --git a/validate/number/metadata.json b/validate/number/metadata.json deleted file mode 100644 index b8b3be515..000000000 --- a/validate/number/metadata.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "timestamp": "2024-09-27T17-59-35-891Z", - "task": "Create a Number Validator Flatfile Listener plugin:\n - Implement range validation for numbers (min, max, inclusive/exclusive)\n - Add support for integer-only validation\n - Implement precision and scale validation for decimal numbers\n - Add options for handling currency values (e.g., two decimal places)\n - Implement custom step validation (e.g., multiples of 0.5)\n - Add support for scientific notation and exponential numbers\n - Implement options for thousands separators and decimal points\n - Add validation for special number types (e.g., prime, even/odd)\n - Implement custom error messages for different validation failures\n - Add options for rounding and truncation of numbers", - "summary": "This is a Flatfile Listener plugin for number validation. It includes various validation options such as min/max values, precision, scale, currency formatting, special number types, and options for rounding and truncation. The plugin listens to the 'record:created' event and performs validation on a specified number field.", - "steps": [ - [ - "Retrieve the basic structure of a Flatfile Listener plugin and identify the appropriate event topic for number validation.\n", - "#E1", - "PineconeAssistant", - "Provide the basic structure of a Flatfile Listener plugin for number validation, including the correct event topic to use.", - "Plan: Retrieve the basic structure of a Flatfile Listener plugin and identify the appropriate event topic for number validation.\n#E1 = PineconeAssistant[Provide the basic structure of a Flatfile Listener plugin for number validation, including the correct event topic to use.]" - ], - [ - "Implement range validation for numbers (min, max, inclusive/exclusive) and integer-only validation.\n", - "#E2", - "PineconeAssistant", - "Add code to the Flatfile Listener from #E1 to implement range validation (min, max, inclusive/exclusive) and integer-only validation for numbers.", - "Plan: Implement range validation for numbers (min, max, inclusive/exclusive) and integer-only validation.\n#E2 = PineconeAssistant[Add code to the Flatfile Listener from #E1 to implement range validation (min, max, inclusive/exclusive) and integer-only validation for numbers.]" - ], - [ - "Implement precision and scale validation for decimal numbers and add options for handling currency values.\n", - "#E3", - "PineconeAssistant", - "Extend the Flatfile Listener from #E2 to include precision and scale validation for decimal numbers, and add options for handling currency values (e.g., two decimal places).", - "Plan: Implement precision and scale validation for decimal numbers and add options for handling currency values.\n#E3 = PineconeAssistant[Extend the Flatfile Listener from #E2 to include precision and scale validation for decimal numbers, and add options for handling currency values (e.g., two decimal places).]" - ], - [ - "Implement custom step validation and support for scientific notation and exponential numbers.\n", - "#E4", - "PineconeAssistant", - "Add code to the Flatfile Listener from #E3 to implement custom step validation (e.g., multiples of 0.5) and support for scientific notation and exponential numbers.", - "Plan: Implement custom step validation and support for scientific notation and exponential numbers.\n#E4 = PineconeAssistant[Add code to the Flatfile Listener from #E3 to implement custom step validation (e.g., multiples of 0.5) and support for scientific notation and exponential numbers.]" - ], - [ - "Implement options for thousands separators and decimal points, and add validation for special number types.\n", - "#E5", - "PineconeAssistant", - "Extend the Flatfile Listener from #E4 to include options for thousands separators and decimal points, and add validation for special number types (e.g., prime, even/odd).", - "Plan: Implement options for thousands separators and decimal points, and add validation for special number types.\n#E5 = PineconeAssistant[Extend the Flatfile Listener from #E4 to include options for thousands separators and decimal points, and add validation for special number types (e.g., prime, even/odd).]" - ], - [ - "Implement custom error messages for different validation failures and add options for rounding and truncation of numbers.\n", - "#E6", - "PineconeAssistant", - "Complete the Flatfile Listener from #E5 by implementing custom error messages for different validation failures and adding options for rounding and truncation of numbers.", - "Plan: Implement custom error messages for different validation failures and add options for rounding and truncation of numbers.\n#E6 = PineconeAssistant[Complete the Flatfile Listener from #E5 by implementing custom error messages for different validation failures and adding options for rounding and truncation of numbers.]" - ], - [ - "Review and finalize the Number Validator Flatfile Listener plugin.\n", - "#E7", - "LLM", - "Review the complete Number Validator Flatfile Listener plugin from #E6. Ensure all requirements are met, the code is valid, and only valid Event Topics are used. Remove any unused imports or plugins. Provide the final, optimized version of the plugin.", - "Plan: Review and finalize the Number Validator Flatfile Listener plugin.\n#E7 = LLM[Review the complete Number Validator Flatfile Listener plugin from #E6. Ensure all requirements are met, the code is valid, and only valid Event Topics are used. Remove any unused imports or plugins. Provide the final, optimized version of the plugin.]" - ] - ], - "metrics": { - "tokens": { - "plan": 4508, - "state": 5744, - "total": 10252 - } - } -} \ No newline at end of file From f314acdabdcdfcab87af88e6bee06386b1ac5553 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Fri, 4 Oct 2024 13:08:22 -0600 Subject: [PATCH 05/12] feat: npm install after rebase --- package-lock.json | 111 ++++++++++++++++++++-------------------------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/package-lock.json b/package-lock.json index c8ba9bc83..3f892498f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -78,19 +78,6 @@ "@flatfile/listener": "^1.0.5" } }, - "flatfilers/playground": { - "name": "@private/playground", - "version": "0.0.0", - "license": "ISC", - "dependencies": { - "@flatfile/api": "^1.9.19", - "@flatfile/listener": "^1.1.0", - "modern-async": "^2.0.0" - }, - "devDependencies": { - "flatfile": "3.8.0" - } - }, "flatfilers/sandbox": { "name": "@private/sandbox", "version": "0.0.0", @@ -2915,8 +2902,6 @@ }, "node_modules/@flatfile/api": { "version": "1.9.19", - "resolved": "https://registry.npmjs.org/@flatfile/api/-/api-1.9.19.tgz", - "integrity": "sha512-2vblUl7YtiR14RtF/nPQwyWrhavtm6Zzjj5/gMHIvgoA7Wj6jEHxJQUoPCDh+gh5eDh4QcvrrCWO6tw5WzAW2g==", "dependencies": { "@flatfile/cross-env-config": "0.0.4", "@types/pako": "2.0.1", @@ -3068,8 +3053,7 @@ }, "node_modules/@flatfile/listener": { "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@flatfile/listener/-/listener-1.1.0.tgz", - "integrity": "sha512-LEzq0ucXmDy/tCM23dg9MTjtZtd8eGSzHLdaKwzeQWrwjN+XnZE/5wrpU+0vXxfDxTpuKFv8TGA/Bvxh/e+4yg==", + "license": "MIT", "dependencies": { "ansi-colors": "^4.1.3", "cross-fetch": "^4.0.0", @@ -3196,6 +3180,10 @@ "resolved": "validate/isbn", "link": true }, + "node_modules/@flatfile/plugin-validate-number": { + "resolved": "validate/number", + "link": true + }, "node_modules/@flatfile/plugin-validate-phone": { "resolved": "validate/phone", "link": true @@ -6142,10 +6130,6 @@ "node": ">=14" } }, - "node_modules/@private/playground": { - "resolved": "flatfilers/playground", - "link": true - }, "node_modules/@private/sandbox": { "resolved": "flatfilers/sandbox", "link": true @@ -7663,11 +7647,10 @@ }, "node_modules/@rollup/rollup-darwin-arm64": { "version": "4.22.5", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.22.5.tgz", - "integrity": "sha512-250ZGg4ipTL0TGvLlfACkIxS9+KLtIbn7BCZjsZj88zSg2Lvu3Xdw6dhAhfe/FjjXPVNCtcSp+WZjVsD3a/Zlw==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "darwin" @@ -7770,9 +7753,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.22.5", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.22.5.tgz", - "integrity": "sha512-N0jPPhHjGShcB9/XXZQWuWBKZQnC1F36Ce3sDqWpujsGjDz/CQtOL9LgTrJ+rJC8MJeesMWrMWVLKKNR/tMOCA==", + "version": "4.24.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.24.0.tgz", + "integrity": "sha512-ZXFk7M72R0YYFN5q13niV0B7G8/5dcQ9JDp8keJSfr3GoZeXEoMHP/HlvqROA3OMbMdfr19IjCeNAnPUG93b6A==", "cpu": [ "x64" ], @@ -8001,8 +7984,7 @@ }, "node_modules/@types/estree": { "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==" + "license": "MIT" }, "node_modules/@types/flat": { "version": "5.0.5", @@ -8045,9 +8027,8 @@ }, "node_modules/@types/jest": { "version": "29.5.13", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.13.tgz", - "integrity": "sha512-wd+MVEZCHt23V0/L642O5APvspWply/rGY5BcW4SUETo2UzPU3Z26qr8jC2qxpimI2jjx9h7+2cj2FwIr01bXg==", "dev": true, + "license": "MIT", "dependencies": { "expect": "^29.0.0", "pretty-format": "^29.0.0" @@ -8176,8 +8157,6 @@ }, "node_modules/@what3words/api": { "version": "5.4.0", - "resolved": "https://registry.npmjs.org/@what3words/api/-/api-5.4.0.tgz", - "integrity": "sha512-e8BIyUtpvL460q2qcNivFNIPOOb16UN/tGZQge3BagE0QRGI2kNxO6dGappRnvRYdEJXzGG3AdbO8sBC2hj+yQ==", "engines": { "node": ">=18" }, @@ -11137,9 +11116,7 @@ } }, "node_modules/flatfile": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/flatfile/-/flatfile-3.8.0.tgz", - "integrity": "sha512-7QrCUlIaywKIbMMKDmxKZC3EG8iF/iY45gqrjGfQkgKL56GjK/Zq7ib/sFKc4N1oDZLIai9Jpuhzd14PKmVpGg==", + "version": "3.6.8", "dev": true, "dependencies": { "@flatfile/cross-env-config": "^0.0.6", @@ -12606,8 +12583,7 @@ }, "node_modules/isbn3": { "version": "1.2.0", - "resolved": "https://registry.npmjs.org/isbn3/-/isbn3-1.2.0.tgz", - "integrity": "sha512-ackTSv/bE4MxrMIoqwl9FihRFHKdoM+E3hxYeRoYozoaCFuhxjtlS/27kkdUD75bG+CwO6Z1pYxxVYY+66rqAA==", + "license": "MIT", "bin": { "isbn": "bin/isbn", "isbn-audit": "bin/isbn-audit", @@ -12828,9 +12804,8 @@ }, "node_modules/jest": { "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", - "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, + "license": "MIT", "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -14580,9 +14555,9 @@ } }, "node_modules/libphonenumber-js": { - "version": "1.11.10", - "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.10.tgz", - "integrity": "sha512-yHEtzDlG2VbhuxMIo/sAUY+lyL5YThqD+gHT3YyelaRWRQv1VeyEk94jDvviRT4PYmLJR9cHxvtrx9h2f4OAIQ==" + "version": "1.11.11", + "resolved": "https://registry.npmjs.org/libphonenumber-js/-/libphonenumber-js-1.11.11.tgz", + "integrity": "sha512-mF3KaORjJQR6JBNcOkluDcJKhtoQT4VTLRMrX1v/wlBayL4M8ybwEDeryyPcrSEJmD0rVwHUbBarpZwN5NfPFQ==" }, "node_modules/lighthouse-logger": { "version": "1.4.2", @@ -17798,8 +17773,7 @@ }, "node_modules/rollup": { "version": "4.22.5", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.22.5.tgz", - "integrity": "sha512-WoinX7GeQOFMGznEcWA1WrTQCd/tpEbMkc3nuMs9BT0CPjMdSjPMTVClwWd4pgSQwJdP65SK9mTCNvItlr5o7w==", + "license": "MIT", "dependencies": { "@types/estree": "1.0.6" }, @@ -17873,6 +17847,18 @@ "rollup": "*" } }, + "node_modules/rollup/node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.22.5", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.22.5.tgz", + "integrity": "sha512-N0jPPhHjGShcB9/XXZQWuWBKZQnC1F36Ce3sDqWpujsGjDz/CQtOL9LgTrJ+rJC8MJeesMWrMWVLKKNR/tMOCA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ] + }, "node_modules/run-async": { "version": "3.0.0", "dev": true, @@ -19229,9 +19215,8 @@ }, "node_modules/ts-jest": { "version": "29.2.5", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.2.5.tgz", - "integrity": "sha512-KD8zB2aAZrcKIdGk4OwpJggeLcH1FgrICqDSROWqlnJXGCXK4Mn6FcdK2B6670Xr73lHMG1kHw8R87A0ecZ+vA==", "dev": true, + "license": "MIT", "dependencies": { "bs-logger": "^0.2.6", "ejs": "^3.1.10", @@ -20247,7 +20232,7 @@ }, "plugins/automap": { "name": "@flatfile/plugin-automap", - "version": "0.5.1", + "version": "0.5.2", "license": "ISC", "dependencies": { "@flatfile/common-plugin-utils": "^1.0.2", @@ -20812,8 +20797,7 @@ }, "utils/common/node_modules/@flatfile/cross-env-config": { "version": "0.0.6", - "resolved": "https://registry.npmjs.org/@flatfile/cross-env-config/-/cross-env-config-0.0.6.tgz", - "integrity": "sha512-/8GpBVxjzTlAmFni08ELQfIrfuwZ+yIsAsTcYOzazbIAMp1ShKNB/1RfaIzRGhOF5OTWbj37mAL/z/tu6+rvpQ==" + "license": "ISC" }, "utils/extractor": { "name": "@flatfile/util-extractor", @@ -20919,35 +20903,34 @@ "@flatfile/listener": "^1.1.0" } }, - "validate/phone": { - "name": "@flatfile/plugin-validate-phone", - "version": "0.0.0", - "license": "ISC", + "validate/number": { + "name": "@flatfile/plugin-validate-number", + "version": "1.0.0", + "license": "MIT", "dependencies": { - "@flatfile/plugin-record-hook": "^1.7.0", - "libphonenumber-js": "^1.11.10" + "@flatfile/plugin-record-hook": "^1.7.0" }, "devDependencies": { "@flatfile/rollup-config": "^0.1.1" }, - "engines": { - "node": ">= 16" - }, "peerDependencies": { "@flatfile/listener": "^1.0.5" } }, - "validators/isbn": { - "name": "@flatfile/plugin-isbn-validator", - "version": "1.0.0", - "extraneous": true, - "license": "MIT", + "validate/phone": { + "name": "@flatfile/plugin-validate-phone", + "version": "0.0.0", + "license": "ISC", "dependencies": { - "@flatfile/plugin-record-hook": "^1.7.0" + "@flatfile/plugin-record-hook": "^1.7.0", + "libphonenumber-js": "^1.11.10" }, "devDependencies": { "@flatfile/rollup-config": "^0.1.1" }, + "engines": { + "node": ">= 16" + }, "peerDependencies": { "@flatfile/listener": "^1.0.5" } From 92ba513983e3609f38212cb5239b9b3d0437b338 Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Fri, 4 Oct 2024 14:08:34 -0500 Subject: [PATCH 06/12] Update validate/number/package.json --- validate/number/package.json | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/validate/number/package.json b/validate/number/package.json index a758c525c..3c139646c 100644 --- a/validate/number/package.json +++ b/validate/number/package.json @@ -1,27 +1,30 @@ { "name": "@flatfile/plugin-validate-number", "version": "1.0.0", - "description": "A Flatfile Listener plugin for number validation with various options", - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "description": "A Flatfile Listener plugin for number validation", + "engines": { + "node": ">= 16" + }, "browser": { - "./dist/index.js": "./dist/index.browser.js", + "./dist/index.cjs": "./dist/index.browser.cjs", "./dist/index.mjs": "./dist/index.browser.mjs" }, "exports": { "types": "./dist/index.d.ts", "node": { "import": "./dist/index.mjs", - "require": "./dist/index.js" + "require": "./dist/index.cjs" }, "browser": { - "require": "./dist/index.browser.js", + "require": "./dist/index.browser.cjs", "import": "./dist/index.browser.mjs" }, "default": "./dist/index.mjs" }, + "main": "./dist/index.cjs", + "module": "./dist/index.mjs", "source": "./src/index.ts", + "types": "./dist/index.d.ts", "files": [ "dist/**" ], From e5f67b9ee945f7a0cddd633c5813ad40ecfb4c58 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Fri, 4 Oct 2024 13:13:03 -0600 Subject: [PATCH 07/12] feat: move logic to a separate file --- .../number/src/ValidateNumber.e2e.spec.ts | 2 +- validate/number/src/index.ts | 171 +----------------- validate/number/src/validateNumber.ts | 170 +++++++++++++++++ 3 files changed, 172 insertions(+), 171 deletions(-) create mode 100644 validate/number/src/validateNumber.ts diff --git a/validate/number/src/ValidateNumber.e2e.spec.ts b/validate/number/src/ValidateNumber.e2e.spec.ts index ee6d67caa..f26c56959 100644 --- a/validate/number/src/ValidateNumber.e2e.spec.ts +++ b/validate/number/src/ValidateNumber.e2e.spec.ts @@ -7,7 +7,7 @@ import { setupSimpleWorkbook, setupSpace, } from '@flatfile/utils-testing' -import { validateNumber } from './index' +import { validateNumber } from './validateNumber' const api = new FlatfileClient() diff --git a/validate/number/src/index.ts b/validate/number/src/index.ts index 725696328..51c734006 100644 --- a/validate/number/src/index.ts +++ b/validate/number/src/index.ts @@ -1,170 +1 @@ -import { FlatfileListener } from '@flatfile/listener' -import { FlatfileRecord, recordHook } from '@flatfile/plugin-record-hook' -interface NumberValidationConfig { - fields: string[] // Specify fields to validate - min?: number - max?: number - inclusive?: boolean - integerOnly?: boolean - precision?: number - scale?: number - currency?: boolean - step?: number - thousandsSeparator?: string - decimalPoint?: string - specialTypes?: string[] - round?: boolean - truncate?: boolean - sheetSlug?: string // Specify the sheet slug -} - -export function validateNumber(config: NumberValidationConfig) { - return (listener: FlatfileListener) => { - listener.use( - recordHook(config.sheetSlug || '**', async (record: FlatfileRecord) => { - for (const field of config.fields) { - const numberField = record.get(field) - - try { - let numberValue: string | number = String(numberField) - .replace(config.thousandsSeparator || ',', '') - .replace(config.decimalPoint || '.', '.') - numberValue = parseFloat(numberValue) - - if (isNaN(numberValue)) { - record.addWarning(field, `The field '${field}' must be a number`) - } - - if (config.round) { - numberValue = Math.round(numberValue) - } - - if (config.truncate) { - numberValue = Math.trunc(numberValue) - } - - if (config.integerOnly && !Number.isInteger(numberValue)) { - record.addWarning( - field, - `The field '${field}' must be an integer` - ) - } - - if (config.min !== undefined) { - if ( - config.inclusive - ? numberValue < config.min - : numberValue <= config.min - ) { - record.addWarning( - field, - `The field '${field}' must be greater than ${ - config.inclusive ? 'or equal to ' : '' - }${config.min}` - ) - } - } - - if (config.max !== undefined) { - if ( - config.inclusive - ? numberValue > config.max - : numberValue >= config.max - ) { - record.addWarning( - field, - `The field '${field}' must be less than ${ - config.inclusive ? 'or equal to ' : '' - }${config.max}` - ) - } - } - - if (config.precision !== undefined && config.scale !== undefined) { - const [integerPart, decimalPart] = numberValue - .toString() - .split('.') - if (integerPart.length > config.precision - config.scale) { - record.addWarning( - field, - `The field '${field}' must have at most ${ - config.precision - config.scale - } digits before the decimal point` - ) - } - if (decimalPart && decimalPart.length > config.scale) { - record.addWarning( - field, - `The field '${field}' must have at most ${config.scale} digits after the decimal point` - ) - } - } - - if ( - config.currency && - !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) - ) { - record.addWarning( - field, - `The field '${field}' must be a valid currency value with at most two decimal places` - ) - } - - if (config.step !== undefined && numberValue % config.step !== 0) { - record.addWarning( - field, - `The field '${field}' must be a multiple of ${config.step}` - ) - } - - if (config.specialTypes) { - if ( - config.specialTypes.includes('prime') && - !isPrime(numberValue) - ) { - record.addWarning( - field, - `The field '${field}' must be a prime number` - ) - } - if ( - config.specialTypes.includes('even') && - numberValue % 2 !== 0 - ) { - record.addWarning( - field, - `The field '${field}' must be an even number` - ) - } - if ( - config.specialTypes.includes('odd') && - numberValue % 2 === 0 - ) { - record.addWarning( - field, - `The field '${field}' must be an odd number` - ) - } - } - record.set(field, numberValue) - - return record - } catch (error) { - record.addError( - field, - `Error processing event for field '${field}': ${error.message}` - ) - throw error - } - } - }) - ) - } -} - -export default function isPrime(num: number): boolean { - for (let i = 2, sqrt = Math.sqrt(num); i <= sqrt; i++) { - if (num % i === 0) return false - } - return num > 1 -} +export * from './validateNumber' diff --git a/validate/number/src/validateNumber.ts b/validate/number/src/validateNumber.ts new file mode 100644 index 000000000..05937136e --- /dev/null +++ b/validate/number/src/validateNumber.ts @@ -0,0 +1,170 @@ +import { FlatfileListener } from '@flatfile/listener' +import { FlatfileRecord, recordHook } from '@flatfile/plugin-record-hook' +export interface NumberValidationConfig { + fields: string[] // Specify fields to validate + min?: number + max?: number + inclusive?: boolean + integerOnly?: boolean + precision?: number + scale?: number + currency?: boolean + step?: number + thousandsSeparator?: string + decimalPoint?: string + specialTypes?: string[] + round?: boolean + truncate?: boolean + sheetSlug?: string // Specify the sheet slug +} + +export function validateNumber(config: NumberValidationConfig) { + return (listener: FlatfileListener) => { + listener.use( + recordHook(config.sheetSlug || '**', async (record: FlatfileRecord) => { + for (const field of config.fields) { + const numberField = record.get(field) + + try { + let numberValue: string | number = String(numberField) + .replace(config.thousandsSeparator || ',', '') + .replace(config.decimalPoint || '.', '.') + numberValue = parseFloat(numberValue) + + if (isNaN(numberValue)) { + record.addWarning(field, `The field '${field}' must be a number`) + } + + if (config.round) { + numberValue = Math.round(numberValue) + } + + if (config.truncate) { + numberValue = Math.trunc(numberValue) + } + + if (config.integerOnly && !Number.isInteger(numberValue)) { + record.addWarning( + field, + `The field '${field}' must be an integer` + ) + } + + if (config.min !== undefined) { + if ( + config.inclusive + ? numberValue < config.min + : numberValue <= config.min + ) { + record.addWarning( + field, + `The field '${field}' must be greater than ${ + config.inclusive ? 'or equal to ' : '' + }${config.min}` + ) + } + } + + if (config.max !== undefined) { + if ( + config.inclusive + ? numberValue > config.max + : numberValue >= config.max + ) { + record.addWarning( + field, + `The field '${field}' must be less than ${ + config.inclusive ? 'or equal to ' : '' + }${config.max}` + ) + } + } + + if (config.precision !== undefined && config.scale !== undefined) { + const [integerPart, decimalPart] = numberValue + .toString() + .split('.') + if (integerPart.length > config.precision - config.scale) { + record.addWarning( + field, + `The field '${field}' must have at most ${ + config.precision - config.scale + } digits before the decimal point` + ) + } + if (decimalPart && decimalPart.length > config.scale) { + record.addWarning( + field, + `The field '${field}' must have at most ${config.scale} digits after the decimal point` + ) + } + } + + if ( + config.currency && + !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) + ) { + record.addWarning( + field, + `The field '${field}' must be a valid currency value with at most two decimal places` + ) + } + + if (config.step !== undefined && numberValue % config.step !== 0) { + record.addWarning( + field, + `The field '${field}' must be a multiple of ${config.step}` + ) + } + + if (config.specialTypes) { + if ( + config.specialTypes.includes('prime') && + !isPrime(numberValue) + ) { + record.addWarning( + field, + `The field '${field}' must be a prime number` + ) + } + if ( + config.specialTypes.includes('even') && + numberValue % 2 !== 0 + ) { + record.addWarning( + field, + `The field '${field}' must be an even number` + ) + } + if ( + config.specialTypes.includes('odd') && + numberValue % 2 === 0 + ) { + record.addWarning( + field, + `The field '${field}' must be an odd number` + ) + } + } + record.set(field, numberValue) + + return record + } catch (error) { + record.addError( + field, + `Error processing event for field '${field}': ${error.message}` + ) + throw error + } + } + }) + ) + } +} + +export default function isPrime(num: number): boolean { + for (let i = 2, sqrt = Math.sqrt(num); i <= sqrt; i++) { + if (num % i === 0) return false + } + return num > 1 +} From f147cb12fffbf564749236d1ea6bb4b166d281f0 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Fri, 4 Oct 2024 13:16:07 -0600 Subject: [PATCH 08/12] feat: rename --- validate/number/src/index.ts | 2 +- ...ateNumber.e2e.spec.ts => validate.number.plugin.e2e.spec.ts} | 2 +- .../number/src/{validateNumber.ts => validate.number.plugin.ts} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename validate/number/src/{ValidateNumber.e2e.spec.ts => validate.number.plugin.e2e.spec.ts} (99%) rename validate/number/src/{validateNumber.ts => validate.number.plugin.ts} (100%) diff --git a/validate/number/src/index.ts b/validate/number/src/index.ts index 51c734006..4c5f44f21 100644 --- a/validate/number/src/index.ts +++ b/validate/number/src/index.ts @@ -1 +1 @@ -export * from './validateNumber' +export * from './validate.number.plugin' diff --git a/validate/number/src/ValidateNumber.e2e.spec.ts b/validate/number/src/validate.number.plugin.e2e.spec.ts similarity index 99% rename from validate/number/src/ValidateNumber.e2e.spec.ts rename to validate/number/src/validate.number.plugin.e2e.spec.ts index f26c56959..c0fcc592b 100644 --- a/validate/number/src/ValidateNumber.e2e.spec.ts +++ b/validate/number/src/validate.number.plugin.e2e.spec.ts @@ -7,7 +7,7 @@ import { setupSimpleWorkbook, setupSpace, } from '@flatfile/utils-testing' -import { validateNumber } from './validateNumber' +import { validateNumber } from './validate.number.plugin' const api = new FlatfileClient() diff --git a/validate/number/src/validateNumber.ts b/validate/number/src/validate.number.plugin.ts similarity index 100% rename from validate/number/src/validateNumber.ts rename to validate/number/src/validate.number.plugin.ts From 9be07040c4a5da7b62a86e3d4bd139dd85fa03a6 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Tue, 8 Oct 2024 14:56:15 -0400 Subject: [PATCH 09/12] feat: refactor for unit tests --- validate/number/jest.config.js | 16 ++ validate/number/package.json | 4 +- .../src/validate.number.plugin.e2e.spec.ts | 250 ----------------- .../number/src/validate.number.plugin.spec.ts | 155 +++++++++++ validate/number/src/validate.number.plugin.ts | 255 +++++++++--------- 5 files changed, 294 insertions(+), 386 deletions(-) create mode 100644 validate/number/jest.config.js delete mode 100644 validate/number/src/validate.number.plugin.e2e.spec.ts create mode 100644 validate/number/src/validate.number.plugin.spec.ts diff --git a/validate/number/jest.config.js b/validate/number/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/validate/number/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/validate/number/package.json b/validate/number/package.json index 3c139646c..af65cd9d9 100644 --- a/validate/number/package.json +++ b/validate/number/package.json @@ -33,7 +33,9 @@ "build:watch": "rollup -c --watch", "build:prod": "NODE_ENV=production rollup -c", "check": "tsc ./**/*.ts --noEmit --esModuleInterop", - "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" }, "keywords": [ "flatfile", diff --git a/validate/number/src/validate.number.plugin.e2e.spec.ts b/validate/number/src/validate.number.plugin.e2e.spec.ts deleted file mode 100644 index c0fcc592b..000000000 --- a/validate/number/src/validate.number.plugin.e2e.spec.ts +++ /dev/null @@ -1,250 +0,0 @@ -import { FlatfileClient } from '@flatfile/api' -import { - createRecords, - deleteSpace, - getRecords, - setupListener, - setupSimpleWorkbook, - setupSpace, -} from '@flatfile/utils-testing' -import { validateNumber } from './validate.number.plugin' - -const api = new FlatfileClient() - -describe('ValidateNumber e2e', () => { - const listener = setupListener() - - let spaceId: string - let sheetId: string - - beforeAll(async () => { - const space = await setupSpace() - spaceId = space.id - const workbook = await setupSimpleWorkbook(space.id, [ - { key: 'amount', type: 'number' }, - { key: 'price', type: 'number' }, - { key: 'quantity', type: 'number' }, - ]) - sheetId = workbook.sheets![0].id - }) - - afterAll(async () => { - await deleteSpace(spaceId) - }) - - afterEach(async () => { - listener.reset() - const records = await getRecords(sheetId) - if (records.length > 0) { - const ids = records.map((record) => record.id) - await api.records.delete(sheetId, { ids }) - } - }) - - describe('validateNumber()', () => { - it('validates min and max values', async () => { - listener.use( - validateNumber({ - fields: ['amount'], - min: 0, - max: 100, - }) - ) - - await createRecords(sheetId, [ - { amount: -10 }, - { amount: 50 }, - { amount: 150 }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.amount?.messages?.[0].message).toContain( - 'must be greater than' - ) - expect(records[1].values.amount.messages).toEqual([]) - expect(records[2].values.amount.messages?.[0].message).toContain( - 'must be less than' - ) - }) - - it('validates integer-only values', async () => { - listener.use( - validateNumber({ - fields: ['quantity'], - integerOnly: true, - }) - ) - - await createRecords(sheetId, [ - { quantity: 5 }, - { quantity: 3.14 }, - { quantity: '3' }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.quantity.messages).toEqual([]) - expect(records[1].values.quantity.messages?.[0].message).toContain( - 'must be an integer' - ) - expect(records[2].values.quantity.value).toEqual(3) - }) - - it('validates precision and scale', async () => { - listener.use( - validateNumber({ - fields: ['price'], - precision: 5, - scale: 2, - }) - ) - - await createRecords(sheetId, [ - { price: 123.45 }, - { price: 1234.56 }, - { price: 12.345 }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.price.messages).toEqual([]) - expect(records[1].values.price.messages?.[0].message).toContain( - 'must have at most 3 digits before the decimal point' - ) - expect(records[2].values.price.messages?.[0].message).toContain( - 'must have at most 2 digits after the decimal point' - ) - }) - - it('validates step values', async () => { - listener.use( - validateNumber({ - fields: ['amount'], - step: 0.5, - }) - ) - - await createRecords(sheetId, [ - { amount: 1.5 }, - { amount: 2.25 }, - { amount: 3 }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.amount.messages).toEqual([]) - expect(records[1].values.amount.messages?.[0].message).toContain( - 'must be a multiple of 0.5' - ) - expect(records[2].values.amount.messages).toEqual([]) - }) - - it('handles currency values', async () => { - listener.use( - validateNumber({ - fields: ['price'], - currency: true, - }) - ) - - await createRecords(sheetId, [ - { price: 10.0 }, - { price: 15.5 }, - { price: 20.555 }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.price.messages).toEqual([]) - expect(records[1].values.price.messages).toEqual([]) - expect(records[2].values.price.messages?.[0].message).toContain( - 'must be a valid currency value' - ) - }) - - it('validates special number types', async () => { - listener.use( - validateNumber({ - fields: ['amount'], - specialTypes: ['even'], - }) - ) - - await createRecords(sheetId, [ - { amount: 2 }, - { amount: 3 }, - { amount: 4 }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.amount.messages).toEqual([]) - expect(records[1].values.amount.messages?.[0].message).toContain( - 'must be an even number' - ) - expect(records[2].values.amount.messages).toEqual([]) - }) - - it('handles rounding', async () => { - listener.use( - validateNumber({ - fields: ['amount'], - round: true, - }) - ) - - await createRecords(sheetId, [{ amount: 1.4 }, { amount: 1.6 }]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.amount.value).toBe(1) - expect(records[1].values.amount.value).toBe(2) - }) - - it('handles truncation', async () => { - listener.use( - validateNumber({ - fields: ['amount'], - truncate: true, - }) - ) - - await createRecords(sheetId, [{ amount: 1.4 }, { amount: 1.6 }]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.amount.value).toBe(1) - expect(records[1].values.amount.value).toBe(1) - }) - - it('handles custom separators', async () => { - listener.use( - validateNumber({ - fields: ['amount'], - thousandsSeparator: '.', - decimalPoint: ',', - }) - ) - - await createRecords(sheetId, [ - { amount: '1.000,50' }, - { amount: '2,500.75' }, - ]) - await listener.waitFor('commit:created') - - const records = await getRecords(sheetId) - - expect(records[0].values.amount.value).toBe(1000.5) - expect(records[1].values.amount.value).toBe(2.50075) - }) - }) -}) diff --git a/validate/number/src/validate.number.plugin.spec.ts b/validate/number/src/validate.number.plugin.spec.ts new file mode 100644 index 000000000..50613f5ca --- /dev/null +++ b/validate/number/src/validate.number.plugin.spec.ts @@ -0,0 +1,155 @@ +import { + validateNumberField, + NumberValidationConfig, +} from './validate.number.plugin' + +describe('validateNumberField', () => { + const defaultConfig: NumberValidationConfig = {} + + it('should validate a basic number', () => { + const result = validateNumberField('123', defaultConfig) + expect(result).toEqual({ + value: 123, + errors: [], + warnings: [], + }) + }) + + it('should handle non-numeric input', () => { + const result = validateNumberField('abc', defaultConfig) + expect(result).toEqual({ + value: null, + errors: ['Must be a number'], + warnings: [], + }) + }) + + it('should validate min and max constraints', () => { + const config: NumberValidationConfig = { min: 0, max: 100 } + expect(validateNumberField('50', config)).toEqual({ + value: 50, + errors: [], + warnings: [], + }) + expect(validateNumberField('-1', config)).toEqual({ + value: -1, + errors: [], + warnings: ['Must be greater than 0'], + }) + expect(validateNumberField('101', config)).toEqual({ + value: 101, + errors: [], + warnings: ['Must be less than 100'], + }) + }) + + it('should validate integer constraint', () => { + const config: NumberValidationConfig = { integerOnly: true } + expect(validateNumberField('10', config)).toEqual({ + value: 10, + errors: [], + warnings: [], + }) + expect(validateNumberField('10.5', config)).toEqual({ + value: 10.5, + errors: [], + warnings: ['Must be an integer'], + }) + }) + + it('should validate precision and scale', () => { + const config: NumberValidationConfig = { precision: 5, scale: 2 } + expect(validateNumberField('123.45', config)).toEqual({ + value: 123.45, + errors: [], + warnings: [], + }) + expect(validateNumberField('1234.567', config)).toEqual({ + value: 1234.567, + errors: [], + warnings: [ + 'Must have at most 3 digits before the decimal point', + 'Must have at most 2 digits after the decimal point', + ], + }) + }) + + it('should validate currency format', () => { + const config: NumberValidationConfig = { currency: true } + expect(validateNumberField('100.00', config)).toEqual({ + value: 100, + errors: [], + warnings: [], + }) + expect(validateNumberField('100.123', config)).toEqual({ + value: 100.123, + errors: [], + warnings: [ + 'Must be a valid currency value with at most two decimal places', + ], + }) + }) + + it('should validate step constraint', () => { + const config: NumberValidationConfig = { step: 0.5 } + expect(validateNumberField('2.5', config)).toEqual({ + value: 2.5, + errors: [], + warnings: [], + }) + expect(validateNumberField('2.7', config)).toEqual({ + value: 2.7, + errors: [], + warnings: ['Must be a multiple of 0.5'], + }) + }) + + it('should handle thousands separator and decimal point', () => { + const config: NumberValidationConfig = { + thousandsSeparator: ',', + decimalPoint: '.', + } + expect(validateNumberField('1,234.56', config)).toEqual({ + value: 1234.56, + errors: [], + warnings: [], + }) + }) + + it('should validate special types', () => { + const config: NumberValidationConfig = { specialTypes: ['prime', 'odd'] } + expect(validateNumberField('7', config)).toEqual({ + value: 7, + errors: [], + warnings: [], + }) + expect(validateNumberField('9', config)).toEqual({ + value: 9, + errors: [], + warnings: ['Must be a prime number'], + }) + expect(validateNumberField('2', config)).toEqual({ + value: 2, + errors: [], + warnings: ['Must be an odd number'], + }) + }) + + it('should round numbers when configured', () => { + const config: NumberValidationConfig = { round: true } + expect(validateNumberField('3.7', config)).toEqual({ + value: 4, + errors: [], + warnings: [], + }) + }) + + it('should truncate numbers when configured', () => { + const config: NumberValidationConfig = { truncate: true } + expect(validateNumberField('3.7', config)).toEqual({ + value: 3, + errors: [], + warnings: [], + }) + }) +}) diff --git a/validate/number/src/validate.number.plugin.ts b/validate/number/src/validate.number.plugin.ts index 05937136e..2ee8bf7ad 100644 --- a/validate/number/src/validate.number.plugin.ts +++ b/validate/number/src/validate.number.plugin.ts @@ -1,7 +1,7 @@ import { FlatfileListener } from '@flatfile/listener' import { FlatfileRecord, recordHook } from '@flatfile/plugin-record-hook' + export interface NumberValidationConfig { - fields: string[] // Specify fields to validate min?: number max?: number inclusive?: boolean @@ -15,154 +15,139 @@ export interface NumberValidationConfig { specialTypes?: string[] round?: boolean truncate?: boolean - sheetSlug?: string // Specify the sheet slug } -export function validateNumber(config: NumberValidationConfig) { +export interface NumberValidationResult { + value: number | null + errors: string[] + warnings: string[] +} + +export function validateNumberField( + value: string | number, + config: NumberValidationConfig +): NumberValidationResult { + const result: NumberValidationResult = { + value: null, + errors: [], + warnings: [], + } + + try { + let numberValue: string | number = String(value) + .replace(config.thousandsSeparator || ',', '') + .replace(config.decimalPoint || '.', '.') + numberValue = parseFloat(numberValue) + + if (isNaN(numberValue)) { + result.errors.push('Must be a number') + return result + } + + if (config.round) { + numberValue = Math.round(numberValue) + } + + if (config.truncate) { + numberValue = Math.trunc(numberValue) + } + + if (config.integerOnly && !Number.isInteger(numberValue)) { + result.warnings.push('Must be an integer') + } + + if (config.min !== undefined) { + if ( + config.inclusive ? numberValue < config.min : numberValue <= config.min + ) { + result.warnings.push( + `Must be greater than ${config.inclusive ? 'or equal to ' : ''}${config.min}` + ) + } + } + + if (config.max !== undefined) { + if ( + config.inclusive ? numberValue > config.max : numberValue >= config.max + ) { + result.warnings.push( + `Must be less than ${config.inclusive ? 'or equal to ' : ''}${config.max}` + ) + } + } + + if (config.precision !== undefined && config.scale !== undefined) { + const [integerPart, decimalPart] = numberValue.toString().split('.') + if (integerPart.length > config.precision - config.scale) { + result.warnings.push( + `Must have at most ${config.precision - config.scale} digits before the decimal point` + ) + } + if (decimalPart && decimalPart.length > config.scale) { + result.warnings.push( + `Must have at most ${config.scale} digits after the decimal point` + ) + } + } + + if (config.currency && !/^\d+(\.\d{1,2})?$/.test(numberValue.toString())) { + result.warnings.push( + 'Must be a valid currency value with at most two decimal places' + ) + } + + if (config.step !== undefined && numberValue % config.step !== 0) { + result.warnings.push(`Must be a multiple of ${config.step}`) + } + + if (config.specialTypes) { + if (config.specialTypes.includes('prime') && !isPrime(numberValue)) { + result.warnings.push('Must be a prime number') + } + if (config.specialTypes.includes('even') && numberValue % 2 !== 0) { + result.warnings.push('Must be an even number') + } + if (config.specialTypes.includes('odd') && numberValue % 2 === 0) { + result.warnings.push('Must be an odd number') + } + } + + result.value = numberValue + } catch (error) { + result.errors.push(`Error processing value: ${error.message}`) + } + + return result +} + +export function validateNumber( + config: NumberValidationConfig & { fields: string[]; sheetSlug?: string } +) { return (listener: FlatfileListener) => { listener.use( recordHook(config.sheetSlug || '**', async (record: FlatfileRecord) => { for (const field of config.fields) { - const numberField = record.get(field) - - try { - let numberValue: string | number = String(numberField) - .replace(config.thousandsSeparator || ',', '') - .replace(config.decimalPoint || '.', '.') - numberValue = parseFloat(numberValue) - - if (isNaN(numberValue)) { - record.addWarning(field, `The field '${field}' must be a number`) - } - - if (config.round) { - numberValue = Math.round(numberValue) - } - - if (config.truncate) { - numberValue = Math.trunc(numberValue) - } - - if (config.integerOnly && !Number.isInteger(numberValue)) { - record.addWarning( - field, - `The field '${field}' must be an integer` - ) - } - - if (config.min !== undefined) { - if ( - config.inclusive - ? numberValue < config.min - : numberValue <= config.min - ) { - record.addWarning( - field, - `The field '${field}' must be greater than ${ - config.inclusive ? 'or equal to ' : '' - }${config.min}` - ) - } - } - - if (config.max !== undefined) { - if ( - config.inclusive - ? numberValue > config.max - : numberValue >= config.max - ) { - record.addWarning( - field, - `The field '${field}' must be less than ${ - config.inclusive ? 'or equal to ' : '' - }${config.max}` - ) - } - } - - if (config.precision !== undefined && config.scale !== undefined) { - const [integerPart, decimalPart] = numberValue - .toString() - .split('.') - if (integerPart.length > config.precision - config.scale) { - record.addWarning( - field, - `The field '${field}' must have at most ${ - config.precision - config.scale - } digits before the decimal point` - ) - } - if (decimalPart && decimalPart.length > config.scale) { - record.addWarning( - field, - `The field '${field}' must have at most ${config.scale} digits after the decimal point` - ) - } - } - - if ( - config.currency && - !/^\d+(\.\d{1,2})?$/.test(numberValue.toString()) - ) { - record.addWarning( - field, - `The field '${field}' must be a valid currency value with at most two decimal places` - ) - } - - if (config.step !== undefined && numberValue % config.step !== 0) { - record.addWarning( - field, - `The field '${field}' must be a multiple of ${config.step}` - ) - } - - if (config.specialTypes) { - if ( - config.specialTypes.includes('prime') && - !isPrime(numberValue) - ) { - record.addWarning( - field, - `The field '${field}' must be a prime number` - ) - } - if ( - config.specialTypes.includes('even') && - numberValue % 2 !== 0 - ) { - record.addWarning( - field, - `The field '${field}' must be an even number` - ) - } - if ( - config.specialTypes.includes('odd') && - numberValue % 2 === 0 - ) { - record.addWarning( - field, - `The field '${field}' must be an odd number` - ) - } - } - record.set(field, numberValue) - + const value = record.get(field) + if (typeof value !== 'string' && typeof value !== 'number') { return record - } catch (error) { - record.addError( - field, - `Error processing event for field '${field}': ${error.message}` - ) - throw error } + const result = validateNumberField(value, config) + if (result.value !== null) { + record.set(field, result.value) + } + + result.errors.forEach((error) => record.addError(field, error)) + result.warnings.forEach((warning) => + record.addWarning(field, warning) + ) } + return record }) ) } } -export default function isPrime(num: number): boolean { +export function isPrime(num: number): boolean { for (let i = 2, sqrt = Math.sqrt(num); i <= sqrt; i++) { if (num % i === 0) return false } From f48b9733a3c4931ac30dbcaff98f10f18c2a9598 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Tue, 8 Oct 2024 17:04:49 -0400 Subject: [PATCH 10/12] feat: update package.json --- validate/number/package.json | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/validate/number/package.json b/validate/number/package.json index af65cd9d9..552cd4c1a 100644 --- a/validate/number/package.json +++ b/validate/number/package.json @@ -38,15 +38,11 @@ "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" }, "keywords": [ - "flatfile", - "plugin", - "number", - "validation", "flatfile-plugins", "category-transform" ], - "author": "Your Name", - "license": "MIT", + "author": "Flatfile, Inc", + "license": "ISC", "dependencies": { "@flatfile/plugin-record-hook": "^1.7.0" }, @@ -59,7 +55,7 @@ "repository": { "type": "git", "url": "https://github.com/FlatFilers/flatfile-plugins.git", - "directory": "plugins/number-validation" + "directory": "validate/number" }, "browserslist": [ "> 0.5%", From a02bbdc7d4397a4d3a00d28bf7bc2216a47fd8e2 Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Tue, 8 Oct 2024 16:15:28 -0500 Subject: [PATCH 11/12] Apply suggestions from code review --- validate/number/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validate/number/package.json b/validate/number/package.json index 552cd4c1a..6860a8cab 100644 --- a/validate/number/package.json +++ b/validate/number/package.json @@ -1,6 +1,6 @@ { "name": "@flatfile/plugin-validate-number", - "version": "1.0.0", + "version": "0.0.0", "description": "A Flatfile Listener plugin for number validation", "engines": { "node": ">= 16" From a99f28c4e8c6afaba1bed5e73ed1d90e02a53c95 Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Tue, 8 Oct 2024 17:18:17 -0400 Subject: [PATCH 12/12] Update package.json --- validate/number/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/validate/number/package.json b/validate/number/package.json index 6860a8cab..7e820e53e 100644 --- a/validate/number/package.json +++ b/validate/number/package.json @@ -1,6 +1,7 @@ { "name": "@flatfile/plugin-validate-number", "version": "0.0.0", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/validate/number", "description": "A Flatfile Listener plugin for number validation", "engines": { "node": ">= 16"