From 0d368ccbcbe19234d711ce95cbbfe87788d352ed Mon Sep 17 00:00:00 2001 From: "Alex Rock (Koala)" Date: Tue, 24 Sep 2024 01:48:49 -0600 Subject: [PATCH 1/6] koala: initial commit --- validators/GoogleGeocoding/README.MD | 67 ++++++++++ validators/GoogleGeocoding/metadata.json | 77 ++++++++++++ validators/GoogleGeocoding/package.json | 68 ++++++++++ validators/GoogleGeocoding/rollup.config.mjs | 49 ++++++++ validators/GoogleGeocoding/src/index.ts | 123 +++++++++++++++++++ 5 files changed, 384 insertions(+) create mode 100644 validators/GoogleGeocoding/README.MD create mode 100644 validators/GoogleGeocoding/metadata.json create mode 100644 validators/GoogleGeocoding/package.json create mode 100644 validators/GoogleGeocoding/rollup.config.mjs create mode 100644 validators/GoogleGeocoding/src/index.ts diff --git a/validators/GoogleGeocoding/README.MD b/validators/GoogleGeocoding/README.MD new file mode 100644 index 000000000..5f67679e6 --- /dev/null +++ b/validators/GoogleGeocoding/README.MD @@ -0,0 +1,67 @@ +# Flatfile Geocoding Plugin + +A Flatfile Listener plugin for geocoding addresses using the Google Maps Geocoding API. This plugin enables automatic geocoding of addresses, reverse geocoding of coordinates, and enrichment of location data within your Flatfile imports. + +## Features + +- Forward geocoding: Convert addresses to latitude and longitude coordinates +- Reverse geocoding: Convert latitude and longitude coordinates to formatted addresses +- Automatic geocoding based on available data +- Customizable field mapping +- Error handling for invalid inputs and API errors +- Extraction of additional location components (country, postal code) + +## Installation + +To install the plugin, use npm: + +```bash +npm install @flatfile/plugin-geocoding +``` + +## Example Usage + +```javascript +import { FlatfileListener } from "@flatfile/listener"; +import geocodingPlugin from "@flatfile/plugin-geocoding"; + +const listener = new FlatfileListener(); + +geocodingPlugin(listener, { + apiKey: 'YOUR_GOOGLE_MAPS_API_KEY', + sheetSlug: 'customer_addresses', + addressField: 'full_address', + latitudeField: 'lat', + longitudeField: 'lng' +}); + +// ... rest of your Flatfile setup +``` + +## Configuration + +The plugin accepts the following configuration options: + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `sheetSlug` | string | "addresses" | The slug of the sheet containing address data | +| `addressField` | string | "address" | The field name for the address input | +| `latitudeField` | string | "latitude" | The field name for latitude | +| `longitudeField` | string | "longitude" | The field name for longitude | +| `autoGeocode` | boolean | true | Whether to automatically geocode records | +| `apiKey` | string | "YOUR_API_KEY_HERE" | Your Google Maps API key | + +## Behavior + +1. The plugin checks for the presence of an address or both latitude and longitude fields. +2. If `autoGeocode` is true: + - For records with an address, it performs forward geocoding. + - For records with latitude and longitude, it performs reverse geocoding. +3. The plugin updates the record with: + - Latitude and longitude (for forward geocoding) + - Formatted address (for both forward and reverse geocoding) + - Additional components: country and postal code +4. If geocoding fails or returns no results, appropriate errors are added to the record. +5. The plugin handles API errors and adds descriptive error messages to the record. + +Note: Ensure you have a valid Google Maps API key with Geocoding API enabled to use this plugin. \ No newline at end of file diff --git a/validators/GoogleGeocoding/metadata.json b/validators/GoogleGeocoding/metadata.json new file mode 100644 index 000000000..f62f4b2be --- /dev/null +++ b/validators/GoogleGeocoding/metadata.json @@ -0,0 +1,77 @@ +{ + "timestamp": "2024-09-24T07-20-20-112Z", + "task": "Create a Google Geocoding Flatfile Listener plugin:\n - Implement a RecordHook to process address fields\n - Use Google Maps Geocoding API to convert addresses to coordinates\n - Handle reverse geocoding (coordinates to addresses) if needed\n - Add latitude, longitude, and formatted address to new fields\n - Implement error handling for failed geocoding attempts\n - Give the user reasonable config options to specify the Sheet Slug, the Field(s) that are the address(es), whether the geocoding should be done automatically", + "summary": "This code implements a Flatfile Listener plugin for geocoding addresses using the Google Maps Geocoding API. It allows for user customization of sheet and field names, supports both forward and reverse geocoding, and includes error handling.", + "steps": [ + [ + "Retrieve information about Flatfile Listeners and RecordHook plugin.\n", + "#E1", + "PineconeAssistant", + "Explain Flatfile Listeners and RecordHook plugin with code examples", + "Plan: Retrieve information about Flatfile Listeners and RecordHook plugin.\n#E1 = PineconeAssistant[Explain Flatfile Listeners and RecordHook plugin with code examples]" + ], + [ + "Get information about Google Maps Geocoding API and its usage.\n", + "#E2", + "Google", + "Google Maps Geocoding API usage JavaScript", + "Plan: Get information about Google Maps Geocoding API and its usage.\n#E2 = Google[Google Maps Geocoding API usage JavaScript]" + ], + [ + "Create the basic structure of the Flatfile Listener plugin with RecordHook.\n", + "#E3", + "LLM", + "Create a basic Flatfile Listener plugin structure with RecordHook using the information from #E1 and #E2", + "Plan: Create the basic structure of the Flatfile Listener plugin with RecordHook.\n#E3 = LLM[Create a basic Flatfile Listener plugin structure with RecordHook using the information from #E1 and #E2]" + ], + [ + "Implement the Google Maps Geocoding API call within the RecordHook.\n", + "#E4", + "LLM", + "Implement Google Maps Geocoding API call in the RecordHook using the structure from #E3 and API information from #E2", + "Plan: Implement the Google Maps Geocoding API call within the RecordHook.\n#E4 = LLM[Implement Google Maps Geocoding API call in the RecordHook using the structure from #E3 and API information from #E2]" + ], + [ + "Add error handling and reverse geocoding functionality.\n", + "#E5", + "LLM", + "Add error handling and reverse geocoding functionality to the code from #E4", + "Plan: Add error handling and reverse geocoding functionality.\n#E5 = LLM[Add error handling and reverse geocoding functionality to the code from #E4]" + ], + [ + "Implement config options for user customization.\n", + "#E6", + "LLM", + "Add config options for Sheet Slug, address fields, and automatic geocoding to the code from #E5", + "Plan: Implement config options for user customization.\n#E6 = LLM[Add config options for Sheet Slug, address fields, and automatic geocoding to the code from #E5]" + ], + [ + "Finalize the plugin code and ensure all requirements are met.\n", + "#E7", + "LLM", + "Review and finalize the plugin code from #E6, ensuring all requirements are met and the code is valid", + "Plan: Finalize the plugin code and ensure all requirements are met.\n#E7 = LLM[Review and finalize the plugin code from #E6, ensuring all requirements are met and the code is valid]" + ], + [ + "Validate the Event Topics used in the Listener.\n", + "#E8", + "PineconeAssistant", + "Verify that the Event Topics used in the Listener from #E7 are valid according to the event.topics.fact file", + "Plan: Validate the Event Topics used in the Listener.\n#E8 = PineconeAssistant[Verify that the Event Topics used in the Listener from #E7 are valid according to the event.topics.fact file]" + ], + [ + "Remove any unused imports and perform final code cleanup.\n", + "#E9", + "LLM", + "Review the code from #E7 and #E8, remove any unused imports, and perform final code cleanup", + "Plan: Remove any unused imports and perform final code cleanup.\n#E9 = LLM[Review the code from #E7 and #E8, remove any unused imports, and perform final code cleanup]" + ] + ], + "metrics": { + "tokens": { + "plan": 3866, + "state": 4034, + "total": 7900 + } + } +} \ No newline at end of file diff --git a/validators/GoogleGeocoding/package.json b/validators/GoogleGeocoding/package.json new file mode 100644 index 000000000..001091318 --- /dev/null +++ b/validators/GoogleGeocoding/package.json @@ -0,0 +1,68 @@ +{ + "name": "@flatfile/plugin-geocoding", + "version": "1.0.0", + "description": "A Flatfile plugin for geocoding addresses using the Google Maps Geocoding API", + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "browser": { + "./dist/index.js": "./dist/index.browser.js", + "./dist/index.mjs": "./dist/index.browser.mjs" + }, + "exports": { + "types": "./dist/index.d.ts", + "node": { + "import": "./dist/index.mjs", + "require": "./dist/index.js" + }, + "browser": { + "require": "./dist/index.browser.js", + "import": "./dist/index.browser.mjs" + }, + "default": "./dist/index.mjs" + }, + "source": "./src/index.ts", + "files": [ + "dist/**" + ], + "scripts": { + "build": "rollup -c", + "build:watch": "rollup -c --watch", + "build:prod": "NODE_ENV=production rollup -c", + "check": "tsc ./**/*.ts --noEmit --esModuleInterop", + "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + }, + "keywords": [ + "flatfile", + "plugin", + "geocoding", + "google-maps", + "flatfile-plugins", + "category-transform" + ], + "author": "Your Name", + "license": "MIT", + "dependencies": { + "@flatfile/plugin-record-hook": "^1.6.1", + "axios": "^1.7.7" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + }, + "devDependencies": { + "@flatfile/hooks": "^1.5.0", + "@flatfile/rollup-config": "^0.1.1", + "@types/node": "^22.6.1", + "typescript": "^5.6.2" + }, + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "plugins/geocoding" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ] +} \ No newline at end of file diff --git a/validators/GoogleGeocoding/rollup.config.mjs b/validators/GoogleGeocoding/rollup.config.mjs new file mode 100644 index 000000000..9548ce4d6 --- /dev/null +++ b/validators/GoogleGeocoding/rollup.config.mjs @@ -0,0 +1,49 @@ +import { buildConfig } from '@flatfile/rollup-config'; +import { nodeResolve } from '@rollup/plugin-node-resolve'; +import commonjs from '@rollup/plugin-commonjs'; +import typescript from '@rollup/plugin-typescript'; +import json from '@rollup/plugin-json'; + +const umdExternals = [ + '@flatfile/api', + '@flatfile/hooks', + '@flatfile/listener', + '@flatfile/util-common', + '@flatfile/plugin-record-hook', + 'axios' +]; + +const config = buildConfig({ + input: 'src/index.ts', // Adjust this if your entry file is different + includeUmd: true, + umdConfig: { + name: 'FlatfileGeocodingPlugin', + external: umdExternals + }, + plugins: [ + nodeResolve({ + browser: true, + preferBuiltins: false, + }), + commonjs(), + json(), + typescript({ + tsconfig: './tsconfig.json', + declaration: true, + declarationDir: 'dist/types', + }), + ] +}); + +// Add external dependencies +const external = [ + ...umdExternals, + 'axios', +]; + +// Modify each build configuration to include external dependencies +config.forEach(buildConfig => { + buildConfig.external = external; +}); + +export default config; \ No newline at end of file diff --git a/validators/GoogleGeocoding/src/index.ts b/validators/GoogleGeocoding/src/index.ts new file mode 100644 index 000000000..53c3ceb4b --- /dev/null +++ b/validators/GoogleGeocoding/src/index.ts @@ -0,0 +1,123 @@ +import { FlatfileListener } from '@flatfile/listener' +import { recordHook } from '@flatfile/plugin-record-hook' +import axios from 'axios' + +interface GeocodingConfig { + sheetSlug: string + addressField: string + latitudeField: string + longitudeField: string + autoGeocode: boolean + apiKey: string +} + +const defaultConfig: GeocodingConfig = { + sheetSlug: 'addresses', + addressField: 'address', + latitudeField: 'latitude', + longitudeField: 'longitude', + autoGeocode: true, + apiKey: 'YOUR_API_KEY_HERE', +} + +export default function geocodingPlugin( + listener: FlatfileListener, + userConfig: Partial = {} +) { + const config: GeocodingConfig = { ...defaultConfig, ...userConfig } + + listener.use( + recordHook(config.sheetSlug, async (record, event) => { + const address = record.get(config.addressField) as string + const latitude = record.get(config.latitudeField) as number + const longitude = record.get(config.longitudeField) as number + + if (!address && (!latitude || !longitude)) { + record.addError( + config.addressField, + `Either ${config.addressField} or both ${config.latitudeField} and ${config.longitudeField} are required` + ) + return record + } + + if (!config.autoGeocode) { + return record + } + + try { + let response + if (address) { + // Forward geocoding + response = await axios.get( + 'https://maps.googleapis.com/maps/api/geocode/json', + { + params: { + address: address, + key: config.apiKey, + }, + } + ) + } else { + // Reverse geocoding + response = await axios.get( + 'https://maps.googleapis.com/maps/api/geocode/json', + { + params: { + latlng: `${latitude},${longitude}`, + key: config.apiKey, + }, + } + ) + } + + if (response.data.status === 'OK') { + const result = response.data.results[0] + const { lat, lng } = result.geometry.location + + record.set(config.latitudeField, lat) + record.set(config.longitudeField, lng) + record.set('formatted_address', result.formatted_address) + + // Extract additional components + for (const component of result.address_components) { + if (component.types.includes('country')) { + record.set('country', component.long_name) + } + if (component.types.includes('postal_code')) { + record.set('postal_code', component.long_name) + } + } + } else if (response.data.status === 'ZERO_RESULTS') { + record.addError( + address ? config.addressField : 'coordinates', + 'No results found for the given input' + ) + } else { + record.addError( + address ? config.addressField : 'coordinates', + `Geocoding failed: ${response.data.status}` + ) + } + } catch (error) { + console.error('Error calling Google Maps Geocoding API:', error) + if (axios.isAxiosError(error) && error.response) { + record.addError( + address ? config.addressField : 'coordinates', + `API error: ${error.response.status} - ${ + error.response.data.error_message || 'Unknown error' + }` + ) + } else { + record.addError( + address ? config.addressField : 'coordinates', + 'An unexpected error occurred while geocoding' + ) + } + } + + return record + }) + ) + + return listener +} From 539ba5e62161673b64d5d52343a04f39ffc5265e Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Thu, 10 Oct 2024 15:42:38 -0400 Subject: [PATCH 2/6] feat: cleanup and add tests to geocoding plugin --- .../geocode}/README.MD | 36 ++-- enrich/geocode/jest.config.js | 16 ++ .../geocode}/metadata.json | 0 .../geocode}/package.json | 8 +- enrich/geocode/rollup.config.mjs | 5 + .../geocode/src/enrich.geocode.plugin.spec.ts | 146 ++++++++++++++++ enrich/geocode/src/enrich.geocode.plugin.ts | 158 ++++++++++++++++++ enrich/geocode/src/index.ts | 1 + validate/boolean/README.MD | 18 +- validate/date/package.json | 4 +- validators/GoogleGeocoding/rollup.config.mjs | 49 ------ validators/GoogleGeocoding/src/index.ts | 123 -------------- 12 files changed, 364 insertions(+), 200 deletions(-) rename {validators/GoogleGeocoding => enrich/geocode}/README.MD (72%) create mode 100644 enrich/geocode/jest.config.js rename {validators/GoogleGeocoding => enrich/geocode}/metadata.json (100%) rename {validators/GoogleGeocoding => enrich/geocode}/package.json (85%) create mode 100644 enrich/geocode/rollup.config.mjs create mode 100644 enrich/geocode/src/enrich.geocode.plugin.spec.ts create mode 100644 enrich/geocode/src/enrich.geocode.plugin.ts create mode 100644 enrich/geocode/src/index.ts delete mode 100644 validators/GoogleGeocoding/rollup.config.mjs delete mode 100644 validators/GoogleGeocoding/src/index.ts diff --git a/validators/GoogleGeocoding/README.MD b/enrich/geocode/README.MD similarity index 72% rename from validators/GoogleGeocoding/README.MD rename to enrich/geocode/README.MD index 5f67679e6..c810cb6c8 100644 --- a/validators/GoogleGeocoding/README.MD +++ b/enrich/geocode/README.MD @@ -1,6 +1,12 @@ -# Flatfile Geocoding Plugin + +# @flatfile/plugin-enrich-geocode -A Flatfile Listener plugin for geocoding addresses using the Google Maps Geocoding API. This plugin enables automatic geocoding of addresses, reverse geocoding of coordinates, and enrichment of location data within your Flatfile imports. +The `@flatfile/plugin-enrich-geocode` plugin for geocoding addresses using the Google Maps Geocoding API. This plugin enables automatic geocoding of addresses, reverse geocoding of coordinates, and enrichment of location data within your Flatfile imports. + +**Event Type:** +`listener.on('commit:created')` + + ## Features @@ -16,24 +22,24 @@ A Flatfile Listener plugin for geocoding addresses using the Google Maps Geocodi To install the plugin, use npm: ```bash -npm install @flatfile/plugin-geocoding +npm install @flatfile/plugin-enrich-geocode ``` ## Example Usage ```javascript import { FlatfileListener } from "@flatfile/listener"; -import geocodingPlugin from "@flatfile/plugin-geocoding"; - -const listener = new FlatfileListener(); - -geocodingPlugin(listener, { - apiKey: 'YOUR_GOOGLE_MAPS_API_KEY', - sheetSlug: 'customer_addresses', - addressField: 'full_address', - latitudeField: 'lat', - longitudeField: 'lng' -}); +import enrichGeocode from "@flatfile/plugin-enrich-geocode"; + +listener.use( + enrichGeocode(listener, { + apiKey: 'YOUR_GOOGLE_MAPS_API_KEY', + sheetSlug: 'customer_addresses', + addressField: 'full_address', + latitudeField: 'lat', + longitudeField: 'lng' + }) +); // ... rest of your Flatfile setup ``` @@ -64,4 +70,4 @@ The plugin accepts the following configuration options: 4. If geocoding fails or returns no results, appropriate errors are added to the record. 5. The plugin handles API errors and adds descriptive error messages to the record. -Note: Ensure you have a valid Google Maps API key with Geocoding API enabled to use this plugin. \ No newline at end of file +Note: Ensure you have a valid Google Maps API key with Geocoding API enabled to use this plugin. diff --git a/enrich/geocode/jest.config.js b/enrich/geocode/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/enrich/geocode/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} diff --git a/validators/GoogleGeocoding/metadata.json b/enrich/geocode/metadata.json similarity index 100% rename from validators/GoogleGeocoding/metadata.json rename to enrich/geocode/metadata.json diff --git a/validators/GoogleGeocoding/package.json b/enrich/geocode/package.json similarity index 85% rename from validators/GoogleGeocoding/package.json rename to enrich/geocode/package.json index 001091318..89ca263a0 100644 --- a/validators/GoogleGeocoding/package.json +++ b/enrich/geocode/package.json @@ -1,5 +1,5 @@ { - "name": "@flatfile/plugin-geocoding", + "name": "@flatfile/plugin-enrich-geocode", "version": "1.0.0", "description": "A Flatfile plugin for geocoding addresses using the Google Maps Geocoding API", "main": "./dist/index.js", @@ -30,7 +30,9 @@ "build:watch": "rollup -c --watch", "build:prod": "NODE_ENV=production rollup -c", "check": "tsc ./**/*.ts --noEmit --esModuleInterop", - "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" }, "keywords": [ "flatfile", @@ -65,4 +67,4 @@ "last 2 versions", "not dead" ] -} \ No newline at end of file +} diff --git a/enrich/geocode/rollup.config.mjs b/enrich/geocode/rollup.config.mjs new file mode 100644 index 000000000..fafa813c6 --- /dev/null +++ b/enrich/geocode/rollup.config.mjs @@ -0,0 +1,5 @@ +import { buildConfig } from '@flatfile/rollup-config' + +const config = buildConfig({}) + +export default config diff --git a/enrich/geocode/src/enrich.geocode.plugin.spec.ts b/enrich/geocode/src/enrich.geocode.plugin.spec.ts new file mode 100644 index 000000000..c50b66f46 --- /dev/null +++ b/enrich/geocode/src/enrich.geocode.plugin.spec.ts @@ -0,0 +1,146 @@ +import axios from 'axios'; +import { performGeocoding } from './enrich.geocode.plugin'; + +jest.mock('axios'); +const mockedAxios = axios as jest.Mocked; + +describe('performGeocoding', () => { + const apiKey = 'test_api_key'; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should perform forward geocoding successfully', async () => { + const mockResponse = { + data: { + status: 'OK', + results: [ + { + geometry: { + location: { lat: 40.7128, lng: -74.0060 }, + }, + formatted_address: 'New York, NY, USA', + address_components: [ + { types: ['country'], long_name: 'United States' }, + { types: ['postal_code'], long_name: '10001' }, + ], + }, + ], + }, + }; + mockedAxios.get.mockResolvedValue(mockResponse); + + const result = await performGeocoding({ address: 'New York' }, apiKey); + + expect(mockedAxios.get).toHaveBeenCalledWith( + 'https://maps.googleapis.com/maps/api/geocode/json', + { + params: { + address: 'New York', + key: apiKey, + }, + } + ); + expect(result).toEqual({ + latitude: 40.7128, + longitude: -74.0060, + formatted_address: 'New York, NY, USA', + country: 'United States', + postal_code: '10001', + }); + }); + + it('should perform reverse geocoding successfully', async () => { + const mockResponse = { + data: { + status: 'OK', + results: [ + { + geometry: { + location: { lat: 48.8584, lng: 2.2945 }, + }, + formatted_address: 'Eiffel Tower, Paris, France', + address_components: [ + { types: ['country'], long_name: 'France' }, + { types: ['postal_code'], long_name: '75007' }, + ], + }, + ], + }, + }; + mockedAxios.get.mockResolvedValue(mockResponse); + + const result = await performGeocoding({ latitude: 48.8584, longitude: 2.2945 }, apiKey); + + expect(mockedAxios.get).toHaveBeenCalledWith( + 'https://maps.googleapis.com/maps/api/geocode/json', + { + params: { + latlng: '48.8584,2.2945', + key: apiKey, + }, + } + ); + expect(result).toEqual({ + latitude: 48.8584, + longitude: 2.2945, + formatted_address: 'Eiffel Tower, Paris, France', + country: 'France', + postal_code: '75007', + }); + }); + + it('should handle zero results', async () => { + const mockResponse = { + data: { + status: 'ZERO_RESULTS', + results: [], + }, + }; + mockedAxios.get.mockResolvedValue(mockResponse); + + const result = await performGeocoding({ address: 'NonexistentPlace' }, apiKey); + + expect(result).toEqual({ + message: 'No results found for the given input', + field: 'address', + }); + }); + + it('should handle API errors', async () => { + mockedAxios.get.mockRejectedValue({ + response: { + status: 400, + data: { error_message: 'Invalid request' }, + }, + }); + + const result = await performGeocoding({ address: 'New York' }, apiKey); + + expect(result).toEqual({ + message: 'An unexpected error occurred while geocoding', + field: 'address', + }); + }); + + it('should handle unexpected errors', async () => { + mockedAxios.get.mockRejectedValue(new Error('Network error')); + + const result = await performGeocoding({ address: 'New York' }, apiKey); + + expect(result).toEqual({ + message: 'An unexpected error occurred while geocoding', + field: 'address', + }); + }); + + it('should return an error when neither address nor coordinates are provided', async () => { + const result = await performGeocoding({}, apiKey); + + expect(result).toEqual({ + message: 'Either address or both latitude and longitude are required', + field: 'input', + }); + }); +}); diff --git a/enrich/geocode/src/enrich.geocode.plugin.ts b/enrich/geocode/src/enrich.geocode.plugin.ts new file mode 100644 index 000000000..cf0f546d5 --- /dev/null +++ b/enrich/geocode/src/enrich.geocode.plugin.ts @@ -0,0 +1,158 @@ +import { FlatfileListener } from '@flatfile/listener' +import { recordHook } from '@flatfile/plugin-record-hook' +import axios from 'axios' + +interface GeocodingConfig { + sheetSlug: string + addressField: string + latitudeField: string + longitudeField: string + autoGeocode: boolean + apiKey: string +} + +const defaultConfig: GeocodingConfig = { + sheetSlug: 'addresses', + addressField: 'address', + latitudeField: 'latitude', + longitudeField: 'longitude', + autoGeocode: true, + apiKey: 'YOUR_API_KEY_HERE', +} + +interface GeocodingResult { + latitude: number + longitude: number + formatted_address: string + country?: string + postal_code?: string +} + +interface GeocodingError { + message: string + field: string +} + +export async function performGeocoding( + input: { address?: string; latitude?: number; longitude?: number }, + apiKey: string +): Promise { + try { + let response + if (input.address) { + // Forward geocoding + response = await axios.get( + 'https://maps.googleapis.com/maps/api/geocode/json', + { + params: { + address: input.address, + key: apiKey, + }, + } + ) + } else if (input.latitude && input.longitude) { + // Reverse geocoding + response = await axios.get( + 'https://maps.googleapis.com/maps/api/geocode/json', + { + params: { + latlng: `${input.latitude},${input.longitude}`, + key: apiKey, + }, + } + ) + } else { + return { + message: 'Either address or both latitude and longitude are required', + field: 'input', + } + } + + if (response.data.status === 'OK') { + const result = response.data.results[0] + const { lat, lng } = result.geometry.location + const geocodingResult: GeocodingResult = { + latitude: lat, + longitude: lng, + formatted_address: result.formatted_address, + } + + // Extract additional components + for (const component of result.address_components) { + if (component.types.includes('country')) { + geocodingResult.country = component.long_name + } + if (component.types.includes('postal_code')) { + geocodingResult.postal_code = component.long_name + } + } + + return geocodingResult + } else if (response.data.status === 'ZERO_RESULTS') { + return { + message: 'No results found for the given input', + field: input.address ? 'address' : 'coordinates', + } + } else { + return { + message: `Geocoding failed: ${response.data.status}`, + field: input.address ? 'address' : 'coordinates', + } + } + } catch (error) { + // console.error('Error calling Google Maps Geocoding API:', error) + if (axios.isAxiosError(error) && error.response) { + return { + message: `API error: ${error.response.status} - ${ + error.response.data.error_message || 'Unknown error' + }`, + field: input.address ? 'address' : 'coordinates', + } + } else { + return { + message: 'An unexpected error occurred while geocoding', + field: input.address ? 'address' : 'coordinates', + } + } + } +} + +export function enrichGeocode( + listener: FlatfileListener, + userConfig: Partial = {} +) { + const config: GeocodingConfig = { ...defaultConfig, ...userConfig } + + listener.use( + recordHook(config.sheetSlug, async (record, event) => { + const address = record.get(config.addressField) as string + const latitude = record.get(config.latitudeField) as number + const longitude = record.get(config.longitudeField) as number + + if (!config.autoGeocode) { + return record + } + + const result = await performGeocoding( + { address, latitude, longitude }, + config.apiKey + ) + + if ('message' in result) { + record.addError(result.field, result.message) + } else { + record.set(config.latitudeField, result.latitude) + record.set(config.longitudeField, result.longitude) + record.set('formatted_address', result.formatted_address) + if (result.country) record.set('country', result.country) + if (result.postal_code) record.set('postal_code', result.postal_code) + } + + return record + }) + ) + + return listener +} + +export default enrichGeocode diff --git a/enrich/geocode/src/index.ts b/enrich/geocode/src/index.ts new file mode 100644 index 000000000..ed3c7621c --- /dev/null +++ b/enrich/geocode/src/index.ts @@ -0,0 +1 @@ +export * from './enrich.geocode.plugin'; diff --git a/validate/boolean/README.MD b/validate/boolean/README.MD index a1eab871f..5f93d8f73 100644 --- a/validate/boolean/README.MD +++ b/validate/boolean/README.MD @@ -32,15 +32,15 @@ npm install @flatfile/plugin-validate-boolean ```javascript import { validateBoolean } from '@flatfile/plugin-validate-boolean'; -const booleanValidator = validateBoolean({ - fields: ['isActive', 'hasSubscription'], - validationType: 'truthy', - language: 'en', - handleNull: 'false', - convertNonBoolean: true -}); - -listener.use(booleanValidator); +listener.use( + validateBoolean({ + fields: ['isActive', 'hasSubscription'], + validationType: 'truthy', + language: 'en', + handleNull: 'false', + convertNonBoolean: true + } +)); ``` ## Configuration diff --git a/validate/date/package.json b/validate/date/package.json index 806ebbdda..5a5900ada 100644 --- a/validate/date/package.json +++ b/validate/date/package.json @@ -42,7 +42,9 @@ "build:watch": "rollup -c --watch", "build:prod": "NODE_ENV=production rollup -c", "check": "tsc ./**/*.ts --noEmit --esModuleInterop", - "test": "jest ./**/*.spec.ts --config=../../jest.config.js --runInBand" + "test": "jest src/*.spec.ts --detectOpenHandles", + "test:unit": "jest src/*.spec.ts --testPathIgnorePatterns=.*\\.e2e\\.spec\\.ts$ --detectOpenHandles", + "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" }, "keywords": [ "flatfile-plugins", diff --git a/validators/GoogleGeocoding/rollup.config.mjs b/validators/GoogleGeocoding/rollup.config.mjs deleted file mode 100644 index 9548ce4d6..000000000 --- a/validators/GoogleGeocoding/rollup.config.mjs +++ /dev/null @@ -1,49 +0,0 @@ -import { buildConfig } from '@flatfile/rollup-config'; -import { nodeResolve } from '@rollup/plugin-node-resolve'; -import commonjs from '@rollup/plugin-commonjs'; -import typescript from '@rollup/plugin-typescript'; -import json from '@rollup/plugin-json'; - -const umdExternals = [ - '@flatfile/api', - '@flatfile/hooks', - '@flatfile/listener', - '@flatfile/util-common', - '@flatfile/plugin-record-hook', - 'axios' -]; - -const config = buildConfig({ - input: 'src/index.ts', // Adjust this if your entry file is different - includeUmd: true, - umdConfig: { - name: 'FlatfileGeocodingPlugin', - external: umdExternals - }, - plugins: [ - nodeResolve({ - browser: true, - preferBuiltins: false, - }), - commonjs(), - json(), - typescript({ - tsconfig: './tsconfig.json', - declaration: true, - declarationDir: 'dist/types', - }), - ] -}); - -// Add external dependencies -const external = [ - ...umdExternals, - 'axios', -]; - -// Modify each build configuration to include external dependencies -config.forEach(buildConfig => { - buildConfig.external = external; -}); - -export default config; \ No newline at end of file diff --git a/validators/GoogleGeocoding/src/index.ts b/validators/GoogleGeocoding/src/index.ts deleted file mode 100644 index 53c3ceb4b..000000000 --- a/validators/GoogleGeocoding/src/index.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { FlatfileListener } from '@flatfile/listener' -import { recordHook } from '@flatfile/plugin-record-hook' -import axios from 'axios' - -interface GeocodingConfig { - sheetSlug: string - addressField: string - latitudeField: string - longitudeField: string - autoGeocode: boolean - apiKey: string -} - -const defaultConfig: GeocodingConfig = { - sheetSlug: 'addresses', - addressField: 'address', - latitudeField: 'latitude', - longitudeField: 'longitude', - autoGeocode: true, - apiKey: 'YOUR_API_KEY_HERE', -} - -export default function geocodingPlugin( - listener: FlatfileListener, - userConfig: Partial = {} -) { - const config: GeocodingConfig = { ...defaultConfig, ...userConfig } - - listener.use( - recordHook(config.sheetSlug, async (record, event) => { - const address = record.get(config.addressField) as string - const latitude = record.get(config.latitudeField) as number - const longitude = record.get(config.longitudeField) as number - - if (!address && (!latitude || !longitude)) { - record.addError( - config.addressField, - `Either ${config.addressField} or both ${config.latitudeField} and ${config.longitudeField} are required` - ) - return record - } - - if (!config.autoGeocode) { - return record - } - - try { - let response - if (address) { - // Forward geocoding - response = await axios.get( - 'https://maps.googleapis.com/maps/api/geocode/json', - { - params: { - address: address, - key: config.apiKey, - }, - } - ) - } else { - // Reverse geocoding - response = await axios.get( - 'https://maps.googleapis.com/maps/api/geocode/json', - { - params: { - latlng: `${latitude},${longitude}`, - key: config.apiKey, - }, - } - ) - } - - if (response.data.status === 'OK') { - const result = response.data.results[0] - const { lat, lng } = result.geometry.location - - record.set(config.latitudeField, lat) - record.set(config.longitudeField, lng) - record.set('formatted_address', result.formatted_address) - - // Extract additional components - for (const component of result.address_components) { - if (component.types.includes('country')) { - record.set('country', component.long_name) - } - if (component.types.includes('postal_code')) { - record.set('postal_code', component.long_name) - } - } - } else if (response.data.status === 'ZERO_RESULTS') { - record.addError( - address ? config.addressField : 'coordinates', - 'No results found for the given input' - ) - } else { - record.addError( - address ? config.addressField : 'coordinates', - `Geocoding failed: ${response.data.status}` - ) - } - } catch (error) { - console.error('Error calling Google Maps Geocoding API:', error) - if (axios.isAxiosError(error) && error.response) { - record.addError( - address ? config.addressField : 'coordinates', - `API error: ${error.response.status} - ${ - error.response.data.error_message || 'Unknown error' - }` - ) - } else { - record.addError( - address ? config.addressField : 'coordinates', - 'An unexpected error occurred while geocoding' - ) - } - } - - return record - }) - ) - - return listener -} From 1912bcae11022ad0ed7f2c9fcc4f5ab1fffc9f13 Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Thu, 10 Oct 2024 15:46:02 -0400 Subject: [PATCH 3/6] feat: install deps --- .../geocode/src/enrich.geocode.plugin.spec.ts | 2 +- package-lock.json | 42 +++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/enrich/geocode/src/enrich.geocode.plugin.spec.ts b/enrich/geocode/src/enrich.geocode.plugin.spec.ts index c50b66f46..ae8238630 100644 --- a/enrich/geocode/src/enrich.geocode.plugin.spec.ts +++ b/enrich/geocode/src/enrich.geocode.plugin.spec.ts @@ -119,7 +119,7 @@ describe('performGeocoding', () => { const result = await performGeocoding({ address: 'New York' }, apiKey); expect(result).toEqual({ - message: 'An unexpected error occurred while geocoding', + message: 'API error: 400 - Invalid request', field: 'address', }); }); diff --git a/package-lock.json b/package-lock.json index 69c06be83..b71ca1335 100644 --- a/package-lock.json +++ b/package-lock.json @@ -82,6 +82,38 @@ "@flatfile/listener": "^1.0.5" } }, + "enrich/geocode": { + "version": "1.0.0", + "license": "MIT", + "dependencies": { + "@flatfile/plugin-record-hook": "^1.6.1", + "axios": "^1.7.7" + }, + "devDependencies": { + "@flatfile/hooks": "^1.5.0", + "@flatfile/rollup-config": "^0.1.1", + "@types/node": "^22.6.1", + "typescript": "^5.6.2" + }, + "peerDependencies": { + "@flatfile/listener": "^1.0.5" + } + }, + "enrich/geocode/node_modules/@types/node": { + "version": "22.7.5", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.5.tgz", + "integrity": "sha512-jML7s2NAzMWc//QSJ1a3prpk78cOPchGvXJsC3C6R6PSMoooztvRVQEz89gmBTBY1SPMaqo5teB4uNHPdetShQ==", + "dev": true, + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "enrich/geocode/node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true + }, "enrich/sentiment": { "name": "@flatfile/plugin-enrich-sentiment", "version": "0.0.0", @@ -132,6 +164,7 @@ "flatfilers/playground": { "name": "@private/playground", "version": "0.0.0", + "extraneous": true, "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.19", @@ -156,6 +189,7 @@ } }, "import/rss": { + "name": "@flatfile/plugin-import-rss", "version": "0.0.0", "license": "ISC", "dependencies": { @@ -3211,6 +3245,10 @@ "resolved": "plugins/dxp-configure", "link": true }, + "node_modules/@flatfile/plugin-enrich-geocode": { + "resolved": "enrich/geocode", + "link": true + }, "node_modules/@flatfile/plugin-enrich-sentiment": { "resolved": "enrich/sentiment", "link": true @@ -6245,10 +6283,6 @@ "node": ">=14" } }, - "node_modules/@private/playground": { - "resolved": "flatfilers/playground", - "link": true - }, "node_modules/@private/sandbox": { "resolved": "flatfilers/sandbox", "link": true From 29853cd6c40032a4511221bca1b6a495d406c16f Mon Sep 17 00:00:00 2001 From: Alex Rock Date: Thu, 10 Oct 2024 16:01:17 -0400 Subject: [PATCH 4/6] feat: fix tests --- enrich/geocode/src/enrich.geocode.plugin.spec.ts | 2 +- validate/date/jest.config.js | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 validate/date/jest.config.js diff --git a/enrich/geocode/src/enrich.geocode.plugin.spec.ts b/enrich/geocode/src/enrich.geocode.plugin.spec.ts index ae8238630..c50b66f46 100644 --- a/enrich/geocode/src/enrich.geocode.plugin.spec.ts +++ b/enrich/geocode/src/enrich.geocode.plugin.spec.ts @@ -119,7 +119,7 @@ describe('performGeocoding', () => { const result = await performGeocoding({ address: 'New York' }, apiKey); expect(result).toEqual({ - message: 'API error: 400 - Invalid request', + message: 'An unexpected error occurred while geocoding', field: 'address', }); }); diff --git a/validate/date/jest.config.js b/validate/date/jest.config.js new file mode 100644 index 000000000..e6d7ca40b --- /dev/null +++ b/validate/date/jest.config.js @@ -0,0 +1,16 @@ +module.exports = { + testEnvironment: 'node', + + transform: { + '^.+\\.tsx?$': 'ts-jest', + }, + setupFiles: ['../../test/dotenv-config.js'], + setupFilesAfterEnv: [ + '../../test/betterConsoleLog.js', + '../../test/unit.cleanup.js', + ], + testTimeout: 60_000, + globalSetup: '../../test/setup-global.js', + forceExit: true, + passWithNoTests: true, +} From cb5153458aad43fb22101441932e0f927fa25017 Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Thu, 10 Oct 2024 16:40:30 -0400 Subject: [PATCH 5/6] cleanup --- enrich/geocode/README.MD | 63 +++---- enrich/geocode/metadata.json | 77 -------- enrich/geocode/package.json | 55 +++--- .../geocode/src/enrich.geocode.plugin.spec.ts | 178 +++++++++--------- enrich/geocode/src/enrich.geocode.plugin.ts | 83 +++----- import/rss/README.MD | 56 +++--- package-lock.json | 15 +- 7 files changed, 205 insertions(+), 322 deletions(-) delete mode 100644 enrich/geocode/metadata.json diff --git a/enrich/geocode/README.MD b/enrich/geocode/README.MD index c810cb6c8..db9c07370 100644 --- a/enrich/geocode/README.MD +++ b/enrich/geocode/README.MD @@ -17,6 +17,16 @@ The `@flatfile/plugin-enrich-geocode` plugin for geocoding addresses using the G - Error handling for invalid inputs and API errors - Extraction of additional location components (country, postal code) +## Parameters + +The `enrichGeocode` function accepts a configuration object with the following properties: + +- `sheetSlug` (string): The slug of the sheet containing address data. Default: "addresses" +- `addressField` (string): The field name for the address input. Default: "address" +- `latitudeField` (string): The field name for latitude. Default: "latitude" +- `longitudeField` (string): The field name for longitude. Default: "longitude" +- `autoGeocode` (boolean): Whether to automatically geocode records. Default: true + ## Installation To install the plugin, use npm: @@ -28,46 +38,19 @@ npm install @flatfile/plugin-enrich-geocode ## Example Usage ```javascript -import { FlatfileListener } from "@flatfile/listener"; +import type { FlatfileListener } from "@flatfile/listener"; import enrichGeocode from "@flatfile/plugin-enrich-geocode"; -listener.use( - enrichGeocode(listener, { - apiKey: 'YOUR_GOOGLE_MAPS_API_KEY', - sheetSlug: 'customer_addresses', - addressField: 'full_address', - latitudeField: 'lat', - longitudeField: 'lng' - }) -); - -// ... rest of your Flatfile setup +export default function (listener: FlatfileListener) { + listener.use( + enrichGeocode(listener, { + sheetSlug: 'customer_addresses', + addressField: 'full_address', + latitudeField: 'lat', + longitudeField: 'lng' + }) + ); + + // ... rest of your Flatfile setup +} ``` - -## Configuration - -The plugin accepts the following configuration options: - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `sheetSlug` | string | "addresses" | The slug of the sheet containing address data | -| `addressField` | string | "address" | The field name for the address input | -| `latitudeField` | string | "latitude" | The field name for latitude | -| `longitudeField` | string | "longitude" | The field name for longitude | -| `autoGeocode` | boolean | true | Whether to automatically geocode records | -| `apiKey` | string | "YOUR_API_KEY_HERE" | Your Google Maps API key | - -## Behavior - -1. The plugin checks for the presence of an address or both latitude and longitude fields. -2. If `autoGeocode` is true: - - For records with an address, it performs forward geocoding. - - For records with latitude and longitude, it performs reverse geocoding. -3. The plugin updates the record with: - - Latitude and longitude (for forward geocoding) - - Formatted address (for both forward and reverse geocoding) - - Additional components: country and postal code -4. If geocoding fails or returns no results, appropriate errors are added to the record. -5. The plugin handles API errors and adds descriptive error messages to the record. - -Note: Ensure you have a valid Google Maps API key with Geocoding API enabled to use this plugin. diff --git a/enrich/geocode/metadata.json b/enrich/geocode/metadata.json deleted file mode 100644 index f62f4b2be..000000000 --- a/enrich/geocode/metadata.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "timestamp": "2024-09-24T07-20-20-112Z", - "task": "Create a Google Geocoding Flatfile Listener plugin:\n - Implement a RecordHook to process address fields\n - Use Google Maps Geocoding API to convert addresses to coordinates\n - Handle reverse geocoding (coordinates to addresses) if needed\n - Add latitude, longitude, and formatted address to new fields\n - Implement error handling for failed geocoding attempts\n - Give the user reasonable config options to specify the Sheet Slug, the Field(s) that are the address(es), whether the geocoding should be done automatically", - "summary": "This code implements a Flatfile Listener plugin for geocoding addresses using the Google Maps Geocoding API. It allows for user customization of sheet and field names, supports both forward and reverse geocoding, and includes error handling.", - "steps": [ - [ - "Retrieve information about Flatfile Listeners and RecordHook plugin.\n", - "#E1", - "PineconeAssistant", - "Explain Flatfile Listeners and RecordHook plugin with code examples", - "Plan: Retrieve information about Flatfile Listeners and RecordHook plugin.\n#E1 = PineconeAssistant[Explain Flatfile Listeners and RecordHook plugin with code examples]" - ], - [ - "Get information about Google Maps Geocoding API and its usage.\n", - "#E2", - "Google", - "Google Maps Geocoding API usage JavaScript", - "Plan: Get information about Google Maps Geocoding API and its usage.\n#E2 = Google[Google Maps Geocoding API usage JavaScript]" - ], - [ - "Create the basic structure of the Flatfile Listener plugin with RecordHook.\n", - "#E3", - "LLM", - "Create a basic Flatfile Listener plugin structure with RecordHook using the information from #E1 and #E2", - "Plan: Create the basic structure of the Flatfile Listener plugin with RecordHook.\n#E3 = LLM[Create a basic Flatfile Listener plugin structure with RecordHook using the information from #E1 and #E2]" - ], - [ - "Implement the Google Maps Geocoding API call within the RecordHook.\n", - "#E4", - "LLM", - "Implement Google Maps Geocoding API call in the RecordHook using the structure from #E3 and API information from #E2", - "Plan: Implement the Google Maps Geocoding API call within the RecordHook.\n#E4 = LLM[Implement Google Maps Geocoding API call in the RecordHook using the structure from #E3 and API information from #E2]" - ], - [ - "Add error handling and reverse geocoding functionality.\n", - "#E5", - "LLM", - "Add error handling and reverse geocoding functionality to the code from #E4", - "Plan: Add error handling and reverse geocoding functionality.\n#E5 = LLM[Add error handling and reverse geocoding functionality to the code from #E4]" - ], - [ - "Implement config options for user customization.\n", - "#E6", - "LLM", - "Add config options for Sheet Slug, address fields, and automatic geocoding to the code from #E5", - "Plan: Implement config options for user customization.\n#E6 = LLM[Add config options for Sheet Slug, address fields, and automatic geocoding to the code from #E5]" - ], - [ - "Finalize the plugin code and ensure all requirements are met.\n", - "#E7", - "LLM", - "Review and finalize the plugin code from #E6, ensuring all requirements are met and the code is valid", - "Plan: Finalize the plugin code and ensure all requirements are met.\n#E7 = LLM[Review and finalize the plugin code from #E6, ensuring all requirements are met and the code is valid]" - ], - [ - "Validate the Event Topics used in the Listener.\n", - "#E8", - "PineconeAssistant", - "Verify that the Event Topics used in the Listener from #E7 are valid according to the event.topics.fact file", - "Plan: Validate the Event Topics used in the Listener.\n#E8 = PineconeAssistant[Verify that the Event Topics used in the Listener from #E7 are valid according to the event.topics.fact file]" - ], - [ - "Remove any unused imports and perform final code cleanup.\n", - "#E9", - "LLM", - "Review the code from #E7 and #E8, remove any unused imports, and perform final code cleanup", - "Plan: Remove any unused imports and perform final code cleanup.\n#E9 = LLM[Review the code from #E7 and #E8, remove any unused imports, and perform final code cleanup]" - ] - ], - "metrics": { - "tokens": { - "plan": 3866, - "state": 4034, - "total": 7900 - } - } -} \ No newline at end of file diff --git a/enrich/geocode/package.json b/enrich/geocode/package.json index 89ca263a0..3d4836c9d 100644 --- a/enrich/geocode/package.json +++ b/enrich/geocode/package.json @@ -1,27 +1,39 @@ { "name": "@flatfile/plugin-enrich-geocode", - "version": "1.0.0", + "version": "0.0.0", + "url": "https://github.com/FlatFilers/flatfile-plugins/tree/main/enrich/geocode", "description": "A Flatfile plugin for geocoding addresses using the Google Maps Geocoding API", - "main": "./dist/index.js", - "module": "./dist/index.mjs", - "types": "./dist/index.d.ts", + "registryMetadata": { + "category": "transform" + }, + "engines": { + "node": ">= 16" + }, + "browserslist": [ + "> 0.5%", + "last 2 versions", + "not dead" + ], "browser": { - "./dist/index.js": "./dist/index.browser.js", + "./dist/index.cjs": "./dist/index.browser.cjs", "./dist/index.mjs": "./dist/index.browser.mjs" }, "exports": { "types": "./dist/index.d.ts", "node": { "import": "./dist/index.mjs", - "require": "./dist/index.js" + "require": "./dist/index.cjs" }, "browser": { - "require": "./dist/index.browser.js", + "require": "./dist/index.browser.cjs", "import": "./dist/index.browser.mjs" }, "default": "./dist/index.mjs" }, + "main": "./dist/index.cjs", + "module": "./dist/index.mjs", "source": "./src/index.ts", + "types": "./dist/index.d.ts", "files": [ "dist/**" ], @@ -35,18 +47,19 @@ "test:e2e": "jest src/*.e2e.spec.ts --detectOpenHandles" }, "keywords": [ - "flatfile", - "plugin", - "geocoding", - "google-maps", "flatfile-plugins", - "category-transform" + "category-enrich" ], - "author": "Your Name", - "license": "MIT", + "author": "Flatfile, Inc.", + "repository": { + "type": "git", + "url": "https://github.com/FlatFilers/flatfile-plugins.git", + "directory": "enrich/geocode" + }, + "license": "ISC", "dependencies": { "@flatfile/plugin-record-hook": "^1.6.1", - "axios": "^1.7.7" + "cross-fetch": "^4.0.0" }, "peerDependencies": { "@flatfile/listener": "^1.0.5" @@ -56,15 +69,5 @@ "@flatfile/rollup-config": "^0.1.1", "@types/node": "^22.6.1", "typescript": "^5.6.2" - }, - "repository": { - "type": "git", - "url": "https://github.com/FlatFilers/flatfile-plugins.git", - "directory": "plugins/geocoding" - }, - "browserslist": [ - "> 0.5%", - "last 2 versions", - "not dead" - ] + } } diff --git a/enrich/geocode/src/enrich.geocode.plugin.spec.ts b/enrich/geocode/src/enrich.geocode.plugin.spec.ts index c50b66f46..9ef15dd52 100644 --- a/enrich/geocode/src/enrich.geocode.plugin.spec.ts +++ b/enrich/geocode/src/enrich.geocode.plugin.spec.ts @@ -1,146 +1,138 @@ -import axios from 'axios'; -import { performGeocoding } from './enrich.geocode.plugin'; +import fetch from 'cross-fetch' +import { performGeocoding } from './enrich.geocode.plugin' -jest.mock('axios'); -const mockedAxios = axios as jest.Mocked; +jest.mock('cross-fetch') describe('performGeocoding', () => { - const apiKey = 'test_api_key'; + const apiKey = 'test_api_key' beforeEach(() => { - jest.clearAllMocks(); - }); + jest.clearAllMocks() + }) it('should perform forward geocoding successfully', async () => { const mockResponse = { - data: { - status: 'OK', - results: [ - { - geometry: { - location: { lat: 40.7128, lng: -74.0060 }, - }, - formatted_address: 'New York, NY, USA', - address_components: [ - { types: ['country'], long_name: 'United States' }, - { types: ['postal_code'], long_name: '10001' }, - ], + status: 'OK', + results: [ + { + geometry: { + location: { lat: 40.7128, lng: -74.006 }, }, - ], - }, - }; - mockedAxios.get.mockResolvedValue(mockResponse); - - const result = await performGeocoding({ address: 'New York' }, apiKey); - - expect(mockedAxios.get).toHaveBeenCalledWith( - 'https://maps.googleapis.com/maps/api/geocode/json', - { - params: { - address: 'New York', - key: apiKey, + formatted_address: 'New York, NY, USA', + address_components: [ + { types: ['country'], long_name: 'United States' }, + { types: ['postal_code'], long_name: '10001' }, + ], }, - } - ); + ], + } + ;(fetch as jest.MockedFunction).mockResolvedValue({ + json: jest.fn().mockResolvedValue(mockResponse), + } as any) + + const result = await performGeocoding({ address: 'New York' }, apiKey) + + expect(fetch).toHaveBeenCalledWith( + 'https://maps.googleapis.com/maps/api/geocode/json?key=test_api_key&address=New+York' + ) expect(result).toEqual({ latitude: 40.7128, - longitude: -74.0060, + longitude: -74.006, formatted_address: 'New York, NY, USA', country: 'United States', postal_code: '10001', - }); - }); + }) + }) it('should perform reverse geocoding successfully', async () => { const mockResponse = { - data: { - status: 'OK', - results: [ - { - geometry: { - location: { lat: 48.8584, lng: 2.2945 }, - }, - formatted_address: 'Eiffel Tower, Paris, France', - address_components: [ - { types: ['country'], long_name: 'France' }, - { types: ['postal_code'], long_name: '75007' }, - ], + status: 'OK', + results: [ + { + geometry: { + location: { lat: 48.8584, lng: 2.2945 }, }, - ], - }, - }; - mockedAxios.get.mockResolvedValue(mockResponse); - - const result = await performGeocoding({ latitude: 48.8584, longitude: 2.2945 }, apiKey); - - expect(mockedAxios.get).toHaveBeenCalledWith( - 'https://maps.googleapis.com/maps/api/geocode/json', - { - params: { - latlng: '48.8584,2.2945', - key: apiKey, + formatted_address: 'Eiffel Tower, Paris, France', + address_components: [ + { types: ['country'], long_name: 'France' }, + { types: ['postal_code'], long_name: '75007' }, + ], }, - } - ); + ], + } + ;(fetch as jest.MockedFunction).mockResolvedValue({ + json: jest.fn().mockResolvedValue(mockResponse), + } as any) + + const result = await performGeocoding( + { latitude: 48.8584, longitude: 2.2945 }, + apiKey + ) + + expect(fetch).toHaveBeenCalledWith( + 'https://maps.googleapis.com/maps/api/geocode/json?key=test_api_key&latlng=48.8584%2C2.2945' + ) expect(result).toEqual({ latitude: 48.8584, longitude: 2.2945, formatted_address: 'Eiffel Tower, Paris, France', country: 'France', postal_code: '75007', - }); - }); + }) + }) it('should handle zero results', async () => { const mockResponse = { - data: { - status: 'ZERO_RESULTS', - results: [], - }, - }; - mockedAxios.get.mockResolvedValue(mockResponse); - - const result = await performGeocoding({ address: 'NonexistentPlace' }, apiKey); + status: 'ZERO_RESULTS', + results: [], + } + ;(fetch as jest.MockedFunction).mockResolvedValue({ + json: jest.fn().mockResolvedValue(mockResponse), + } as any) + + const result = await performGeocoding( + { address: 'NonexistentPlace' }, + apiKey + ) expect(result).toEqual({ message: 'No results found for the given input', field: 'address', - }); - }); + }) + }) it('should handle API errors', async () => { - mockedAxios.get.mockRejectedValue({ - response: { - status: 400, - data: { error_message: 'Invalid request' }, - }, - }); + ;(fetch as jest.MockedFunction).mockRejectedValue( + new Error('API error') + ) - const result = await performGeocoding({ address: 'New York' }, apiKey); + const result = await performGeocoding({ address: 'New York' }, apiKey) expect(result).toEqual({ message: 'An unexpected error occurred while geocoding', field: 'address', - }); - }); + }) + }) it('should handle unexpected errors', async () => { - mockedAxios.get.mockRejectedValue(new Error('Network error')); + ;(fetch as jest.MockedFunction).mockRejectedValue( + new Error('Network error') + ) - const result = await performGeocoding({ address: 'New York' }, apiKey); + const result = await performGeocoding({ address: 'New York' }, apiKey) expect(result).toEqual({ message: 'An unexpected error occurred while geocoding', field: 'address', - }); - }); + }) + }) it('should return an error when neither address nor coordinates are provided', async () => { - const result = await performGeocoding({}, apiKey); + const result = await performGeocoding({}, apiKey) expect(result).toEqual({ message: 'Either address or both latitude and longitude are required', field: 'input', - }); - }); -}); + }) + }) +}) diff --git a/enrich/geocode/src/enrich.geocode.plugin.ts b/enrich/geocode/src/enrich.geocode.plugin.ts index cf0f546d5..797b741a2 100644 --- a/enrich/geocode/src/enrich.geocode.plugin.ts +++ b/enrich/geocode/src/enrich.geocode.plugin.ts @@ -1,6 +1,5 @@ -import { FlatfileListener } from '@flatfile/listener' -import { recordHook } from '@flatfile/plugin-record-hook' -import axios from 'axios' +import { bulkRecordHook } from '@flatfile/plugin-record-hook' +import fetch from 'cross-fetch' interface GeocodingConfig { sheetSlug: string @@ -8,16 +7,6 @@ interface GeocodingConfig { latitudeField: string longitudeField: string autoGeocode: boolean - apiKey: string -} - -const defaultConfig: GeocodingConfig = { - sheetSlug: 'addresses', - addressField: 'address', - latitudeField: 'latitude', - longitudeField: 'longitude', - autoGeocode: true, - apiKey: 'YOUR_API_KEY_HERE', } interface GeocodingResult { @@ -39,28 +28,15 @@ export async function performGeocoding( ): Promise { try { let response + let url = 'https://maps.googleapis.com/maps/api/geocode/json' + let params: Record = { key: apiKey } + if (input.address) { // Forward geocoding - response = await axios.get( - 'https://maps.googleapis.com/maps/api/geocode/json', - { - params: { - address: input.address, - key: apiKey, - }, - } - ) + params.address = input.address } else if (input.latitude && input.longitude) { // Reverse geocoding - response = await axios.get( - 'https://maps.googleapis.com/maps/api/geocode/json', - { - params: { - latlng: `${input.latitude},${input.longitude}`, - key: apiKey, - }, - } - ) + params.latlng = `${input.latitude},${input.longitude}` } else { return { message: 'Either address or both latitude and longitude are required', @@ -68,8 +44,12 @@ export async function performGeocoding( } } - if (response.data.status === 'OK') { - const result = response.data.results[0] + const queryString = new URLSearchParams(params).toString() + response = await fetch(`${url}?${queryString}`) + const data = await response.json() + + if (data.status === 'OK') { + const result = data.results[0] const { lat, lng } = result.geometry.location const geocodingResult: GeocodingResult = { latitude: lat, @@ -88,24 +68,22 @@ export async function performGeocoding( } return geocodingResult - } else if (response.data.status === 'ZERO_RESULTS') { + } else if (data.status === 'ZERO_RESULTS') { return { message: 'No results found for the given input', field: input.address ? 'address' : 'coordinates', } } else { return { - message: `Geocoding failed: ${response.data.status}`, + message: `Geocoding failed: ${data.status}`, field: input.address ? 'address' : 'coordinates', } } } catch (error) { - // console.error('Error calling Google Maps Geocoding API:', error) - if (axios.isAxiosError(error) && error.response) { + console.error('Error calling Google Maps Geocoding API:', error) + if (error instanceof Error) { return { - message: `API error: ${error.response.status} - ${ - error.response.data.error_message || 'Unknown error' - }`, + message: `API error: ${error.message}`, field: input.address ? 'address' : 'coordinates', } } else { @@ -117,14 +95,12 @@ export async function performGeocoding( } } -export function enrichGeocode( - listener: FlatfileListener, - userConfig: Partial = {} -) { - const config: GeocodingConfig = { ...defaultConfig, ...userConfig } - - listener.use( - recordHook(config.sheetSlug, async (record, event) => { +export function enrichGeocode(config: GeocodingConfig) { + return bulkRecordHook(config.sheetSlug, async (records, event) => { + const googleMapsApiKey = + process.env.GOOGLE_MAPS_API_KEY || + (await event.secrets('GOOGLE_MAPS_API_KEY')) + for (const record of records) { const address = record.get(config.addressField) as string const latitude = record.get(config.latitudeField) as number const longitude = record.get(config.longitudeField) as number @@ -135,7 +111,7 @@ export function enrichGeocode( const result = await performGeocoding( { address, latitude, longitude }, - config.apiKey + googleMapsApiKey ) if ('message' in result) { @@ -147,12 +123,9 @@ export function enrichGeocode( if (result.country) record.set('country', result.country) if (result.postal_code) record.set('postal_code', result.postal_code) } - - return record - }) - ) - - return listener + } + return records + }) } export default enrichGeocode diff --git a/import/rss/README.MD b/import/rss/README.MD index 1df1fe135..407abaab4 100644 --- a/import/rss/README.MD +++ b/import/rss/README.MD @@ -36,40 +36,42 @@ npm install @flatfile/plugin-import-rss ### JavaScript ```javascript -import { FlatfileListener } from '@flatfile/listener'; import { rssImport } from "@flatfile/plugin-import-rss"; -const listener = new FlatfileListener(); - -listener.use( - rssImport({ - operation: "importRSSFeed", - feeds: [ - { - sheetSlug: "rss_feed_data", - rssFeedUrl: "https://example.com/rss-feed" - } - ] - }) -); - -listener.mount(); +export default function (listener) { + listener.use( + rssImport({ + operation: "importRSSFeed", + feeds: [ + { + sheetSlug: "rss_feed_data", + rssFeedUrl: "https://example.com/rss-feed" + } + ] + }) + ); +} ``` ### TypeScript ```typescript -import { rssImport, RSSImportConfig } from "@flatfile/plugin-import-rss"; - -listener.use(rssImport({ - operation: "importRSSFeed", - feeds: [ - { - sheetSlug: "rss_feed_data", - rssFeedUrl: "https://example.com/rss-feed" - } - ] -})); +import { FlatfileListener } from '@flatfile/listener'; +import { rssImport } from "@flatfile/plugin-import-rss"; + +export default function (listener: FlatfileListener) { + listener.use( + rssImport({ + operation: "importRSSFeed", + feeds: [ + { + sheetSlug: "rss_feed_data", + rssFeedUrl: "https://example.com/rss-feed" + } + ] + }) + ); +} ``` ## Example Usage diff --git a/package-lock.json b/package-lock.json index b71ca1335..5df264996 100644 --- a/package-lock.json +++ b/package-lock.json @@ -83,11 +83,12 @@ } }, "enrich/geocode": { - "version": "1.0.0", - "license": "MIT", + "name": "@flatfile/plugin-enrich-geocode", + "version": "0.0.0", + "license": "ISC", "dependencies": { "@flatfile/plugin-record-hook": "^1.6.1", - "axios": "^1.7.7" + "cross-fetch": "^4.0.0" }, "devDependencies": { "@flatfile/hooks": "^1.5.0", @@ -95,6 +96,9 @@ "@types/node": "^22.6.1", "typescript": "^5.6.2" }, + "engines": { + "node": ">= 16" + }, "peerDependencies": { "@flatfile/listener": "^1.0.5" } @@ -164,7 +168,6 @@ "flatfilers/playground": { "name": "@private/playground", "version": "0.0.0", - "extraneous": true, "license": "ISC", "dependencies": { "@flatfile/api": "^1.9.19", @@ -6283,6 +6286,10 @@ "node": ">=14" } }, + "node_modules/@private/playground": { + "resolved": "flatfilers/playground", + "link": true + }, "node_modules/@private/sandbox": { "resolved": "flatfilers/sandbox", "link": true From 8a2b3725931e02e168a7eff67555015141f524ad Mon Sep 17 00:00:00 2001 From: Carl Brugger Date: Thu, 10 Oct 2024 16:55:06 -0400 Subject: [PATCH 6/6] Update enrich.geocode.plugin.spec.ts --- enrich/geocode/src/enrich.geocode.plugin.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/enrich/geocode/src/enrich.geocode.plugin.spec.ts b/enrich/geocode/src/enrich.geocode.plugin.spec.ts index 9ef15dd52..cf4339b7d 100644 --- a/enrich/geocode/src/enrich.geocode.plugin.spec.ts +++ b/enrich/geocode/src/enrich.geocode.plugin.spec.ts @@ -109,7 +109,7 @@ describe('performGeocoding', () => { const result = await performGeocoding({ address: 'New York' }, apiKey) expect(result).toEqual({ - message: 'An unexpected error occurred while geocoding', + message: 'API error: API error', field: 'address', }) }) @@ -122,7 +122,7 @@ describe('performGeocoding', () => { const result = await performGeocoding({ address: 'New York' }, apiKey) expect(result).toEqual({ - message: 'An unexpected error occurred while geocoding', + message: 'API error: Network error', field: 'address', }) })