diff --git a/enrich/geocode/README.MD b/enrich/geocode/README.MD new file mode 100644 index 000000000..db9c07370 --- /dev/null +++ b/enrich/geocode/README.MD @@ -0,0 +1,56 @@ + +# @flatfile/plugin-enrich-geocode + +The `@flatfile/plugin-enrich-geocode` plugin for geocoding addresses using the Google Maps Geocoding API. This plugin enables automatic geocoding of addresses, reverse geocoding of coordinates, and enrichment of location data within your Flatfile imports. + +**Event Type:** +`listener.on('commit:created')` + + + +## Features + +- Forward geocoding: Convert addresses to latitude and longitude coordinates +- Reverse geocoding: Convert latitude and longitude coordinates to formatted addresses +- Automatic geocoding based on available data +- Customizable field mapping +- Error handling for invalid inputs and API errors +- Extraction of additional location components (country, postal code) + +## Parameters + +The `enrichGeocode` function accepts a configuration object with the following properties: + +- `sheetSlug` (string): The slug of the sheet containing address data. Default: "addresses" +- `addressField` (string): The field name for the address input. Default: "address" +- `latitudeField` (string): The field name for latitude. Default: "latitude" +- `longitudeField` (string): The field name for longitude. Default: "longitude" +- `autoGeocode` (boolean): Whether to automatically geocode records. Default: true + +## Installation + +To install the plugin, use npm: + +```bash +npm install @flatfile/plugin-enrich-geocode +``` + +## Example Usage + +```javascript +import type { FlatfileListener } from "@flatfile/listener"; +import enrichGeocode from "@flatfile/plugin-enrich-geocode"; + +export default function (listener: FlatfileListener) { + listener.use( + enrichGeocode(listener, { + sheetSlug: 'customer_addresses', + addressField: 'full_address', + latitudeField: 'lat', + longitudeField: 'lng' + }) + ); + + // ... rest of your Flatfile setup +} +``` diff --git a/enrich/geocode/jest.config.js b/enrich/geocode/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/enrich/geocode/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/enrich/geocode/package.json b/enrich/geocode/package.json new file mode 100644 index 000000000..3d4836c9d --- /dev/null +++ b/enrich/geocode/package.json @@ -0,0 +1,73 @@ +{ + "name": "@flatfile/plugin-enrich-geocode", + "version": "0.0.0", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/enrich/geocode", + "description": "A Flatfile plugin for geocoding addresses using the Google Maps Geocoding API", + "registryMetadata": { + "category": "transform" + }, + "engines": { + "node": ">= 16" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ], + "browser": { + "./dist/index.cjs": "./dist/index.browser.cjs", + "./dist/index.mjs": "./dist/index.browser.mjs" + }, + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.cjs" + }, + "browser": { + "require": "./dist/index.browser.cjs", + "import": "./dist/index.browser.mjs" + }, + "default": "./dist/index.mjs" + }, + "main": "./dist/index.cjs", + "module": "./dist/index.mjs", + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" + }, + "keywords": [ + "flatfile-plugins", + "category-enrich" + ], + "author": "Flatfile, Inc.", + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "enrich/geocode" + }, + "license": "ISC", + "dependencies": { + "@flatfile/plugin-record-hook": "^1.6.1", + "cross-fetch": "^4.0.0" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + }, + "devDependencies": { + "@flatfile/hooks": "^1.5.0", + "@flatfile/rollup-config": "^0.1.1", + "@types/node": "^22.6.1", + "typescript": "^5.6.2" + } +} diff --git a/enrich/geocode/rollup.config.mjs b/enrich/geocode/rollup.config.mjs new file mode 100644 index 000000000..fafa813c6 --- /dev/null +++ b/enrich/geocode/rollup.config.mjs @@ -0,0 +1,5 @@ +import { buildConfig } from '@flatfile/rollup-config' + +const config = buildConfig({}) + +export default config diff --git a/enrich/geocode/src/enrich.geocode.plugin.spec.ts b/enrich/geocode/src/enrich.geocode.plugin.spec.ts new file mode 100644 index 000000000..cf4339b7d --- /dev/null +++ b/enrich/geocode/src/enrich.geocode.plugin.spec.ts @@ -0,0 +1,138 @@ +import fetch from 'cross-fetch' +import { performGeocoding } from './enrich.geocode.plugin' + +jest.mock('cross-fetch') + +describe('performGeocoding', () => { + const apiKey = 'test_api_key' + + beforeEach(() => { + jest.clearAllMocks() + }) + + it('should perform forward geocoding successfully', async () => { + const mockResponse = { + status: 'OK', + results: [ + { + geometry: { + location: { lat: 40.7128, lng: -74.006 }, + }, + formatted_address: 'New York, NY, USA', + address_components: [ + { types: ['country'], long_name: 'United States' }, + { types: ['postal_code'], long_name: '10001' }, + ], + }, + ], + } + ;(fetch as jest.MockedFunction).mockResolvedValue({ + json: jest.fn().mockResolvedValue(mockResponse), + } as any) + + const result = await performGeocoding({ address: 'New York' }, apiKey) + + expect(fetch).toHaveBeenCalledWith( + 'https://maps.googleapis.com/maps/api/geocode/json?key=test_api_key&address=New+York' + ) + expect(result).toEqual({ + latitude: 40.7128, + longitude: -74.006, + formatted_address: 'New York, NY, USA', + country: 'United States', + postal_code: '10001', + }) + }) + + it('should perform reverse geocoding successfully', async () => { + const mockResponse = { + status: 'OK', + results: [ + { + geometry: { + location: { lat: 48.8584, lng: 2.2945 }, + }, + formatted_address: 'Eiffel Tower, Paris, France', + address_components: [ + { types: ['country'], long_name: 'France' }, + { types: ['postal_code'], long_name: '75007' }, + ], + }, + ], + } + ;(fetch as jest.MockedFunction).mockResolvedValue({ + json: jest.fn().mockResolvedValue(mockResponse), + } as any) + + const result = await performGeocoding( + { latitude: 48.8584, longitude: 2.2945 }, + apiKey + ) + + expect(fetch).toHaveBeenCalledWith( + 'https://maps.googleapis.com/maps/api/geocode/json?key=test_api_key&latlng=48.8584%2C2.2945' + ) + expect(result).toEqual({ + latitude: 48.8584, + longitude: 2.2945, + formatted_address: 'Eiffel Tower, Paris, France', + country: 'France', + postal_code: '75007', + }) + }) + + it('should handle zero results', async () => { + const mockResponse = { + status: 'ZERO_RESULTS', + results: [], + } + ;(fetch as jest.MockedFunction).mockResolvedValue({ + json: jest.fn().mockResolvedValue(mockResponse), + } as any) + + const result = await performGeocoding( + { address: 'NonexistentPlace' }, + apiKey + ) + + expect(result).toEqual({ + message: 'No results found for the given input', + field: 'address', + }) + }) + + it('should handle API errors', async () => { + ;(fetch as jest.MockedFunction).mockRejectedValue( + new Error('API error') + ) + + const result = await performGeocoding({ address: 'New York' }, apiKey) + + expect(result).toEqual({ + message: 'API error: API error', + field: 'address', + }) + }) + + it('should handle unexpected errors', async () => { + ;(fetch as jest.MockedFunction).mockRejectedValue( + new Error('Network error') + ) + + const result = await performGeocoding({ address: 'New York' }, apiKey) + + expect(result).toEqual({ + message: 'API error: Network error', + field: 'address', + }) + }) + + it('should return an error when neither address nor coordinates are provided', async () => { + const result = await performGeocoding({}, apiKey) + + expect(result).toEqual({ + message: 'Either address or both latitude and longitude are required', + field: 'input', + }) + }) +}) diff --git a/enrich/geocode/src/enrich.geocode.plugin.ts b/enrich/geocode/src/enrich.geocode.plugin.ts new file mode 100644 index 000000000..797b741a2 --- /dev/null +++ b/enrich/geocode/src/enrich.geocode.plugin.ts @@ -0,0 +1,131 @@ +import { bulkRecordHook } from '@flatfile/plugin-record-hook' +import fetch from 'cross-fetch' + +interface GeocodingConfig { + sheetSlug: string + addressField: string + latitudeField: string + longitudeField: string + autoGeocode: boolean +} + +interface GeocodingResult { + latitude: number + longitude: number + formatted_address: string + country?: string + postal_code?: string +} + +interface GeocodingError { + message: string + field: string +} + +export async function performGeocoding( + input: { address?: string; latitude?: number; longitude?: number }, + apiKey: string +): Promise { + try { + let response + let url = 'https://maps.googleapis.com/maps/api/geocode/json' + let params: Record = { key: apiKey } + + if (input.address) { + // Forward geocoding + params.address = input.address + } else if (input.latitude && input.longitude) { + // Reverse geocoding + params.latlng = `${input.latitude},${input.longitude}` + } else { + return { + message: 'Either address or both latitude and longitude are required', + field: 'input', + } + } + + const queryString = new URLSearchParams(params).toString() + response = await fetch(`${url}?${queryString}`) + const data = await response.json() + + if (data.status === 'OK') { + const result = data.results[0] + const { lat, lng } = result.geometry.location + const geocodingResult: GeocodingResult = { + latitude: lat, + longitude: lng, + formatted_address: result.formatted_address, + } + + // Extract additional components + for (const component of result.address_components) { + if (component.types.includes('country')) { + geocodingResult.country = component.long_name + } + if (component.types.includes('postal_code')) { + geocodingResult.postal_code = component.long_name + } + } + + return geocodingResult + } else if (data.status === 'ZERO_RESULTS') { + return { + message: 'No results found for the given input', + field: input.address ? 'address' : 'coordinates', + } + } else { + return { + message: `Geocoding failed: ${data.status}`, + field: input.address ? 'address' : 'coordinates', + } + } + } catch (error) { + console.error('Error calling Google Maps Geocoding API:', error) + if (error instanceof Error) { + return { + message: `API error: ${error.message}`, + field: input.address ? 'address' : 'coordinates', + } + } else { + return { + message: 'An unexpected error occurred while geocoding', + field: input.address ? 'address' : 'coordinates', + } + } + } +} + +export function enrichGeocode(config: GeocodingConfig) { + return bulkRecordHook(config.sheetSlug, async (records, event) => { + const googleMapsApiKey = + process.env.GOOGLE_MAPS_API_KEY || + (await event.secrets('GOOGLE_MAPS_API_KEY')) + for (const record of records) { + const address = record.get(config.addressField) as string + const latitude = record.get(config.latitudeField) as number + const longitude = record.get(config.longitudeField) as number + + if (!config.autoGeocode) { + return record + } + + const result = await performGeocoding( + { address, latitude, longitude }, + googleMapsApiKey + ) + + if ('message' in result) { + record.addError(result.field, result.message) + } else { + record.set(config.latitudeField, result.latitude) + record.set(config.longitudeField, result.longitude) + record.set('formatted_address', result.formatted_address) + if (result.country) record.set('country', result.country) + if (result.postal_code) record.set('postal_code', result.postal_code) + } + } + return records + }) +} + +export default enrichGeocode diff --git a/enrich/geocode/src/index.ts b/enrich/geocode/src/index.ts new file mode 100644 index 000000000..ed3c7621c --- /dev/null +++ b/enrich/geocode/src/index.ts @@ -0,0 +1 @@ +export * from './enrich.geocode.plugin'; diff --git a/import/rss/README.MD b/import/rss/README.MD index 1df1fe135..407abaab4 100644 --- a/import/rss/README.MD +++ b/import/rss/README.MD @@ -36,40 +36,42 @@ npm install @flatfile/plugin-import-rss ### JavaScript ```javascript -import { FlatfileListener } from '@flatfile/listener'; import { rssImport } from "@flatfile/plugin-import-rss"; -const listener = new FlatfileListener(); - -listener.use( - rssImport({ - operation: "importRSSFeed", - feeds: [ - { - sheetSlug: "rss_feed_data", - rssFeedUrl: "https://example.com/rss-feed" - } - ] - }) -); - -listener.mount(); +export default function (listener) { + listener.use( + rssImport({ + operation: "importRSSFeed", + feeds: [ + { + sheetSlug: "rss_feed_data", + rssFeedUrl: "https://example.com/rss-feed" + } + ] + }) + ); +} ``` ### TypeScript ```typescript -import { rssImport, RSSImportConfig } from "@flatfile/plugin-import-rss"; - -listener.use(rssImport({ - operation: "importRSSFeed", - feeds: [ - { - sheetSlug: "rss_feed_data", - rssFeedUrl: "https://example.com/rss-feed" - } - ] -})); +import { FlatfileListener } from '@flatfile/listener'; +import { rssImport } from "@flatfile/plugin-import-rss"; + +export default function (listener: FlatfileListener) { + listener.use( + rssImport({ + operation: "importRSSFeed", + feeds: [ + { + sheetSlug: "rss_feed_data", + rssFeedUrl: "https://example.com/rss-feed" + } + ] + }) + ); +} ``` ## Example Usage diff --git a/package-lock.json b/package-lock.json index 69c06be83..5df264996 100644 --- a/package-lock.json +++ b/package-lock.json @@ -82,6 +82,42 @@ "@flatfile/listener": "^1.0.5" } }, + "enrich/geocode": { + "name": "@flatfile/plugin-enrich-geocode", + "version": "0.0.0", + "license": "ISC", + "dependencies": { + "@flatfile/plugin-record-hook": "^1.6.1", + "cross-fetch": "^4.0.0" + }, + "devDependencies": { + "@flatfile/hooks": "^1.5.0", + "@flatfile/rollup-config": "^0.1.1", + "@types/node": "^22.6.1", + "typescript": "^5.6.2" + }, + "engines": { + "node": ">= 16" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + } + }, + "enrich/geocode/node_modules/@types/node": { + "version": "22.7.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.5.tgz", + "integrity": "sha512-jML7s2NAzMWc//QSJ1a3prpk78cOPchGvXJsC3C6R6PSMoooztvRVQEz89gmBTBY1SPMaqo5teB4uNHPdetShQ==", + "dev": true, + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "enrich/geocode/node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true + }, "enrich/sentiment": { "name": "@flatfile/plugin-enrich-sentiment", "version": "0.0.0", @@ -156,6 +192,7 @@ } }, "import/rss": { + "name": "@flatfile/plugin-import-rss", "version": "0.0.0", "license": "ISC", "dependencies": { @@ -3211,6 +3248,10 @@ "resolved": "plugins/dxp-configure", "link": true }, + "node_modules/@flatfile/plugin-enrich-geocode": { + "resolved": "enrich/geocode", + "link": true + }, "node_modules/@flatfile/plugin-enrich-sentiment": { "resolved": "enrich/sentiment", "link": true diff --git a/validate/boolean/README.MD b/validate/boolean/README.MD index a1eab871f..5f93d8f73 100644 --- a/validate/boolean/README.MD +++ b/validate/boolean/README.MD @@ -32,15 +32,15 @@ npm install @flatfile/plugin-validate-boolean ```javascript import { validateBoolean } from '@flatfile/plugin-validate-boolean'; -const booleanValidator = validateBoolean({ - fields: ['isActive', 'hasSubscription'], - validationType: 'truthy', - language: 'en', - handleNull: 'false', - convertNonBoolean: true -}); - -listener.use(booleanValidator); +listener.use( + validateBoolean({ + fields: ['isActive', 'hasSubscription'], + validationType: 'truthy', + language: 'en', + handleNull: 'false', + convertNonBoolean: true + } +)); ``` ## Configuration diff --git a/validate/date/jest.config.js b/validate/date/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/validate/date/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/validate/date/package.json b/validate/date/package.json index 806ebbdda..5a5900ada 100644 --- a/validate/date/package.json +++ b/validate/date/package.json @@ -42,7 +42,9 @@ "build:watch": "rollup -c --watch", "build:prod": "NODE_ENV=production rollup -c", "check": "tsc ./**/*.ts --noEmit --esModuleInterop", - "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" }, "keywords": [ "flatfile-plugins",