importer: rss feed

flatfilers/sandbox/src/index.ts

@@ -1,6 +1,117 @@
 import type { FlatfileListener } from '@flatfile/listener'
+import { rssImport } from '@flatfile/plugin-import-rss'
+import { configureSpace } from '@flatfile/plugin-space-configure'
 import { MarkdownExtractor } from '@flatfile/plugin-markdown-extractor'
 
 export default async function (listener: FlatfileListener) {
+  listener.use(
+    rssImport([
+      {
+        sheetSlug: 'rss-feed-1',
+        rssFeedUrl: 'http://rss.cnn.com/rss/money_topstories.rss',
+      },
+      {
+        sheetSlug: 'rss-feed-2',
+        rssFeedUrl: 'http://rss.cnn.com/rss/money_news_companies.rss',
+      },
+    ])
+  )
+  listener.use(
+    configureSpace({
+      workbooks: [
+        {
+          name: 'Sandbox',
+          sheets: [
+            {
+              name: 'RSS Feed One',
+              slug: 'rss-feed-1',
+              fields: [
+                {
+                  key: 'title',
+                  type: 'string',
+                  label: 'Title',
+                },
+                {
+                  key: 'link',
+                  type: 'string',
+                  label: 'Link',
+                },
+                {
+                  key: 'pubDate',
+                  type: 'string',
+                  label: 'Pub Date',
+                },
+                {
+                  key: 'content',
+                  type: 'string',
+                  label: 'Content',
+                },
+                {
+                  key: 'guid',
+                  type: 'string',
+                  label: 'GUID',
+                },
+              ],
+              actions: [
+                {
+                  operation: 'importRSSFeed',
+                  label: 'Import RSS Feed',
+                  description: 'Import data from an RSS feed into the workbook',
+                  primary: true,
+                  icon: 'rss_feed',
+                  tooltip: 'Click to import data from an RSS feed',
+                  mode: 'foreground',
+                },
+              ],
+            },
+            {
+              name: 'RSS Feed Two',
+              slug: 'rss-feed-2',
+              fields: [
+                {
+                  key: 'title',
+                  type: 'string',
+                  label: 'Title',
+                },
+                {
+                  key: 'link',
+                  type: 'string',
+                  label: 'Link',
+                },
+                {
+                  key: 'pubDate',
+                  type: 'string',
+                  label: 'Pub Date',
+                },
+                {
+                  key: 'content',
+                  type: 'string',
+                  label: 'Content',
+                },
+                {
+                  key: 'guid',
+                  type: 'string',
+                  label: 'GUID',
+                },
+              ],
+              actions: [
+                {
+                  operation: 'importRSSFeed',
+                  label: 'Import RSS Feed',
+                  description: 'Import data from an RSS feed into the workbook',
+                  primary: true,
+                  icon: 'rss_feed',
+                  tooltip: 'Click to import data from an RSS feed',
+                  mode: 'foreground',
+                },
+              ],
+            },
+          ],
+        },
+      ],
+    })
+  )
+
   listener.use(MarkdownExtractor())
+
 }

import/rss/README.MD

@@ -0,0 +1,122 @@
+<!-- START_INFOCARD -->
+
+The `@flatfile/plugin-import-rss` plugin automates the process of importing RSS feed data into Flatfile Sheets. It listens for and responds to job:ready events with the specified operation type, fetching RSS feed content and populating Flatfile sheets with the retrieved data. This plugin supports multiple RSS feeds and provides flexible configuration options.
+
+**Event Type:** 
+`job:ready`
+
+**Supported RSS feed items:** 
+title, link, pubDate, content, guid
+
+<!-- END_INFOCARD -->
+
+> When embedding Flatfile, this plugin should be deployed in a server-side listener. [Learn more](https://flatfile.com/docs/orchestration/listeners#listener-types)
+
+The `@flatfile/plugin-import-rss` plugin allows you to easily import RSS feed data into your Flatfile Sheets. It automates the process of fetching RSS content and populating Flatfile Sheets with the retrieved data.
+
+## Parameters
+
+The `rssImport` function accepts a configuration object with the following properties:
+
+- `operation` (string): The operation name to listen for in the job:ready event.
+- `feeds` (array): An array of feed configurations, each containing:
+  - `sheetSlug` (string): The slug of the sheet to import RSS data into.
+  - `rssFeedUrl` (string): The URL of the RSS feed to import data from.
+
+## Usage
+
+### Installation
+
+To install the plugin, run the following command:
+
+```bash
+npm install @flatfile/plugin-import-rss
+```
+
+### JavaScript
+
+```javascript
+import { FlatfileListener } from '@flatfile/listener';
+import { rssImport } from "@flatfile/plugin-import-rss";
+
+const listener = new FlatfileListener();
+
+listener.use(
+  rssImport({
+    operation: "importRSSFeed",
+    feeds: [
+      {
+        sheetSlug: "rss_feed_data",
+        rssFeedUrl: "https://example.com/rss-feed"
+      }
+    ]
+  })
+);
+
+listener.mount();
+```
+
+### TypeScript
+
+```typescript
+import { rssImport, RSSImportConfig } from "@flatfile/plugin-import-rss";
+
+listener.use(rssImport({
+  operation: "importRSSFeed",
+  feeds: [
+    {
+      sheetSlug: "rss_feed_data",
+      rssFeedUrl: "https://example.com/rss-feed"
+    }
+  ]
+}));
+```
+
+## Example Usage
+
+Here's an example of how to use the plugin with multiple RSS feeds:
+
+```typescript listener.ts
+import type { FlatfileListener } from '@flatfile/listener';
+import { rssImport } from "@flatfile/plugin-import-rss";
+
+export default function (listener: FlatfileListener) {
+  listener.use(
+    rssImport({
+      operation: "importRSSFeed",
+      feeds: [
+        {
+          sheetSlug: "tech_news",
+          rssFeedUrl: "https://techcrunch.com/feed/"
+        },
+        {
+          sheetSlug: "world_news",
+          rssFeedUrl: "https://rss.nytimes.com/services/xml/rss/nyt/World.xml"
+        }
+      ]
+    })
+  );
+}
+```
+
+## API Calls
+
+The plugin uses the following Flatfile API calls:
+
+- `api.sheets.get`
+- `api.jobs.ack`
+- `api.jobs.complete`
+- `api.jobs.fail`
+- `api.records.insert`
+
+## Contributing
+
+Contributions to this plugin are welcome! Please follow these steps:
+
+1. Fork the repository
+2. Create a new branch for your feature or bug fix
+3. Make your changes and commit them with clear, descriptive messages
+4. Push your changes to your fork
+5. Submit a pull request with a clear description of your changes
+
+Please ensure your code adheres to the existing style and includes appropriate tests.

import/rss/metadata.json

@@ -0,0 +1,84 @@
+{
+  "timestamp": "2024-09-25T16-37-16-776Z",
+  "task": "Create a RSS Feed Import Flatfile Action:\n      - Implement a custom action to import new entries from an RSS feed into a Flatfile Sheet\n      - Allow users to specify the RSS feed URL and update frequency\n      - Parse the RSS feed and map feed items to appropriate sheet columns\n      - Implement deduplication to avoid importing duplicate entries\n      - Provide options for handling media enclosures and attachments\n      - Allow users to configure filters for importing specific types of content\n      - Implement error handling for feed parsing and network issues\n      - Generate a report of newly imported entries and any import failures\n      - Optionally, allow scheduling of regular imports",
+  "summary": "This code implements a Flatfile Action for importing RSS feed data, including error handling, scheduling, and reporting. It uses the Flatfile Listener to handle events, parses RSS feeds, maps data to sheet columns, generates reports, and schedules regular imports.",
+  "steps": [
+    [
+      "First, let's create a basic structure for our RSS Feed Import Action using Flatfile's Action framework.\n",
+      "#E1",
+      "PineconeAssistant",
+      "Example of a Flatfile Action structure for RSS Feed Import",
+      "Plan: First, let's create a basic structure for our RSS Feed Import Action using Flatfile's Action framework.\n#E1 = PineconeAssistant[Example of a Flatfile Action structure for RSS Feed Import]"
+    ],
+    [
+      "Now, let's implement the main logic for parsing the RSS feed and mapping items to sheet columns.\n",
+      "#E2",
+      "PineconeAssistant",
+      "Example code for parsing RSS feed and mapping to Flatfile sheet columns",
+      "Plan: Now, let's implement the main logic for parsing the RSS feed and mapping items to sheet columns.\n#E2 = PineconeAssistant[Example code for parsing RSS feed and mapping to Flatfile sheet columns]"
+    ],
+    [
+      "We need to implement deduplication to avoid importing duplicate entries. Let's use Flatfile's Record Hook for this.\n",
+      "#E3",
+      "PineconeAssistant",
+      "Example of using Record Hook in Flatfile for deduplication",
+      "Plan: We need to implement deduplication to avoid importing duplicate entries. Let's use Flatfile's Record Hook for this.\n#E3 = PineconeAssistant[Example of using Record Hook in Flatfile for deduplication]"
+    ],
+    [
+      "Let's add functionality to handle media enclosures and attachments from the RSS feed.\n",
+      "#E4",
+      "PineconeAssistant",
+      "Example of handling media enclosures and attachments in Flatfile",
+      "Plan: Let's add functionality to handle media enclosures and attachments from the RSS feed.\n#E4 = PineconeAssistant[Example of handling media enclosures and attachments in Flatfile]"
+    ],
+    [
+      "Implement user-configurable filters for importing specific types of content.\n",
+      "#E5",
+      "PineconeAssistant",
+      "Example of implementing content filters in Flatfile Action",
+      "Plan: Implement user-configurable filters for importing specific types of content.\n#E5 = PineconeAssistant[Example of implementing content filters in Flatfile Action]"
+    ],
+    [
+      "Add error handling for feed parsing and network issues.\n",
+      "#E6",
+      "PineconeAssistant",
+      "Example of error handling in Flatfile Action for RSS feed parsing and network issues",
+      "Plan: Add error handling for feed parsing and network issues.\n#E6 = PineconeAssistant[Example of error handling in Flatfile Action for RSS feed parsing and network issues]"
+    ],
+    [
+      "Generate a report of newly imported entries and any import failures.\n",
+      "#E7",
+      "PineconeAssistant",
+      "Example of generating import report in Flatfile Action",
+      "Plan: Generate a report of newly imported entries and any import failures.\n#E7 = PineconeAssistant[Example of generating import report in Flatfile Action]"
+    ],
+    [
+      "Implement scheduling functionality for regular imports.\n",
+      "#E8",
+      "PineconeAssistant",
+      "Example of implementing scheduled imports in Flatfile",
+      "Plan: Implement scheduling functionality for regular imports.\n#E8 = PineconeAssistant[Example of implementing scheduled imports in Flatfile]"
+    ],
+    [
+      "Combine all the components into a complete RSS Feed Import Action.\n",
+      "#E9",
+      "LLM",
+      "Combine the following components into a complete Flatfile Action for RSS Feed Import: #E1, #E2, #E3, #E4, #E5, #E6, #E7, #E8. Ensure all imports are correctly included and the code is valid.",
+      "Plan: Combine all the components into a complete RSS Feed Import Action.\n#E9 = LLM[Combine the following components into a complete Flatfile Action for RSS Feed Import: #E1, #E2, #E3, #E4, #E5, #E6, #E7, #E8. Ensure all imports are correctly included and the code is valid.]"
+    ],
+    [
+      "Validate the final Action code, checking for unused imports and ensuring all Event Topics are valid.\n",
+      "#E10",
+      "PineconeAssistant",
+      "Validate the following Flatfile Action code, remove unused imports, and confirm all Event Topics are valid: #E9",
+      "Plan: Validate the final Action code, checking for unused imports and ensuring all Event Topics are valid.\n#E10 = PineconeAssistant[Validate the following Flatfile Action code, remove unused imports, and confirm all Event Topics are valid: #E9]"
+    ]
+  ],
+  "metrics": {
+    "tokens": {
+      "plan": 4763,
+      "state": 5988,
+      "total": 10751
+    }
+  }
+}