Add RequestQueue for fetching queue, retries and errors

CLAUDE.md

@@ -0,0 +1,50 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Commands
+
+### Development
+- `yarn dev` - Watch and compile code continuously
+- `yarn storybook` - Run Storybook on port 6001
+- `yarn tdd` - Run Jest tests in watch mode
+- `yarn start` - Run dev, storybook, and tdd in parallel
+- `yarn test` - Run all Jest tests
+- `yarn test:smoke` - Run smoke tests with open handles detection
+- `yarn update-schema` - Update GraphQL introspection schema
+- `yarn prepare` - Build the package using package-prepare
+- `npm publish` - Transpile and publish to NPM
+
+### Testing
+Tests are located in the `tests/` directory. To run a specific test file:
+```bash
+yarn test tests/[filename].test.js
+```
+
+## Architecture
+
+This package is a GraphQL content layer for fetching and processing conference content from GraphCMS. It:
+
+1. **Fetches data** from GraphCMS using GraphQL queries through multiple fetch modules (`fetch-*.js`)
+2. **Processes content** through a post-processing layer that merges talks, Q&A sessions, and populates speaker activities
+3. **Exposes content** via the `getContent` async function for consumption
+4. **Generates Storybook** for visualizing both CMS and content layers
+
+### Key Components
+
+- **Entry point**: `src/index.js` - Creates GraphQL client and orchestrates all content fetching
+- **Content fetchers**: `src/fetch-*.js` files - Each handles a specific content type (speakers, talks, sponsors, etc.)
+- **Post-processing**: `src/postprocess.js` - Merges and enriches content relationships
+- **Configuration**: Requires `CMS_ENDPOINT` and `CMS_TOKEN` environment variables for GraphCMS connection
+- **Conference settings**: Must be passed to `getContent()` with conference-specific data including `conferenceTitle`, `eventYear`, `tagColors`, and `speakerAvatar` dimensions
+
+### Content Flow
+1. Conference settings are passed to `getContent(conferenceSettings)`
+2. All fetch modules run in parallel via Promise.all
+3. Content pieces are merged with conflict resolution for duplicate keys
+4. Post-processing enriches the content (populates speaker talks, merges Q&A sessions)
+5. Schedule items are sorted chronologically
+6. Final processed content is returned
+
+### GraphQL Schema
+The GraphQL schema is stored in `schema.graphql` and can be updated using `yarn update-schema`. The schema endpoint is configured in `.graphqlconfig`.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@focus-reactive/graphql-content-layer",
-  "version": "3.2.6",
+  "version": "3.2.7",
   "private": false,
   "main": "dist/index.js",
   "scripts": {
@@ -94,4 +94,4 @@
     "gitnation",
     "conference"
   ]
-}
+}

src/index.js

@@ -1,6 +1,7 @@
 const { GraphQLClient } = require('graphql-request');
 
 const { credentials } = require('./config');
+const RequestQueue = require('./request-queue');
 const textContent = require('./fetch-texts');
 const pageContent = require('./fetch-pages');
 const brandContent = require('./fetch-brand');
@@ -42,40 +43,65 @@ const getQueriesData = (content, conferenceSettings) => {
 };
 
 const getContent = async conferenceSettings => {
-  const fetchAll = [
-    textContent,
-    pageContent,
-    brandContent,
-    speakerContent,
-    advisersContent,
-    performanceContent,
-    sponsorContent,
-    talksContent,
-    workshopContent,
-    mcContent,
-    faqContent,
-    extContent,
-    jobsContent,
-    committeeContent,
-    diversityContent,
-    latestLinksContent,
-  ].map(async content => {
-    try {
-      getQueriesData(content, conferenceSettings);
-      const getVarsFromSettings = content.selectSettings || (() => undefined);
-      const { conferenceTitle, eventYear } = conferenceSettings;
-      return await content.fetchData(client, {
-        conferenceTitle,
-        eventYear,
-        ...getVarsFromSettings(conferenceSettings),
-      });
-    } catch (err) {
-      console.error(err);
-      process.exit(1);
-    }
+  const queue = new RequestQueue({
+    concurrency: 5,
+    retryAttempts: 3,
+    retryDelay: 2000,
+    maxRetryDelay: 30000,
+    timeout: 60000,
+  });
+
+  const contentModules = [
+    { module: textContent, name: 'texts' },
+    { module: pageContent, name: 'pages' },
+    { module: brandContent, name: 'brand' },
+    { module: speakerContent, name: 'speakers' },
+    { module: advisersContent, name: 'advisers' },
+    { module: performanceContent, name: 'performance' },
+    { module: sponsorContent, name: 'sponsors' },
+    { module: talksContent, name: 'talks' },
+    { module: workshopContent, name: 'workshops' },
+    { module: mcContent, name: 'mc' },
+    { module: faqContent, name: 'faq' },
+    { module: extContent, name: 'extended' },
+    { module: jobsContent, name: 'jobs' },
+    { module: committeeContent, name: 'committee' },
+    { module: diversityContent, name: 'diversity' },
+    { module: latestLinksContent, name: 'landings' },
+  ];
+
+  const fetchPromises = contentModules.map(({ module: content, name }) => {
+    return queue.add(async () => {
+      try {
+        getQueriesData(content, conferenceSettings);
+        const getVarsFromSettings = content.selectSettings || (() => undefined);
+        const { conferenceTitle, eventYear } = conferenceSettings;
+
+        // eslint-disable-next-line no-console
+        console.log(`Fetching ${name} for ${conferenceTitle} ${eventYear}`);
+
+        const result = await content.fetchData(client, {
+          conferenceTitle,
+          eventYear,
+          ...getVarsFromSettings(conferenceSettings),
+        });
+
+        // eslint-disable-next-line no-console
+        console.log(
+          `Successfully fetched ${name} for ${conferenceTitle} ${eventYear}`,
+        );
+        return result;
+      } catch (err) {
+        console.error(
+          `Failed to fetch ${name} for ${conferenceSettings.conferenceTitle}:`,
+          err,
+        );
+        throw err;
+      }
+    }, name);
   });
 
-  const contentArray = await Promise.all(fetchAll);
+  const contentArray = await Promise.all(fetchPromises);
   const contentMap = contentArray.reduce((content, piece) => {
     try {
       const newKeys = Object.keys(piece);
@@ -86,6 +112,7 @@ const getContent = async conferenceSettings => {
         piece[k] = { ...content[k], ...piece[k] };
       });
     } catch (err) {
+      // eslint-disable-next-line no-console
       console.log('content, piece', piece);
       console.error(err);
     }

src/request-queue.js

@@ -0,0 +1,112 @@
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+class RequestQueue {
+  constructor(options = {}) {
+    this.concurrency = options.concurrency || 5;
+    this.retryAttempts = options.retryAttempts || 3;
+    this.retryDelay = options.retryDelay || 1000;
+    this.maxRetryDelay = options.maxRetryDelay || 30000;
+    this.timeout = options.timeout || 60000;
+
+    this.queue = [];
+    this.running = 0;
+    this.results = new Map();
+  }
+
+  async add(fn, id) {
+    return new Promise((resolve, reject) => {
+      this.queue.push({ fn, id, resolve, reject, attempts: 0 });
+      this.process();
+    });
+  }
+
+  async process() {
+    while (this.running < this.concurrency && this.queue.length > 0) {
+      const task = this.queue.shift();
+      this.running++;
+      this.executeTask(task);
+    }
+  }
+
+  async executeTask(task) {
+    const { fn, id, resolve, reject, attempts } = task;
+
+    try {
+      const timeoutPromise = new Promise((_, reject) =>
+        setTimeout(
+          () => reject(new Error(`Request timeout after ${this.timeout}ms`)),
+          this.timeout,
+        ),
+      );
+
+      const result = await Promise.race([fn(), timeoutPromise]);
+
+      resolve(result);
+      if (id) {
+        this.results.set(id, { success: true, data: result });
+      }
+    } catch (error) {
+      const isRetriableError = this.isRetriable(error);
+      const nextAttempt = attempts + 1;
+
+      if (isRetriableError && nextAttempt < this.retryAttempts) {
+        const delay = Math.min(
+          this.retryDelay * Math.pow(2, attempts),
+          this.maxRetryDelay,
+        );
+
+        console.warn(
+          `Request failed (attempt ${nextAttempt}/${this.retryAttempts}), retrying in ${delay}ms...`,
+          {
+            error: error.message || error,
+            id,
+          },
+        );
+
+        await sleep(delay);
+
+        task.attempts = nextAttempt;
+        this.queue.unshift(task);
+      } else {
+        console.error(`Request failed after ${nextAttempt} attempts`, {
+          error: error.message || error,
+          id,
+        });
+
+        if (id) {
+          this.results.set(id, { success: false, error });
+        }
+        reject(error);
+      }
+    } finally {
+      this.running--;
+      this.process();
+    }
+  }
+
+  isRetriable(error) {
+    const errorMessage = error.message || '';
+    const errorCode = error.code || '';
+    const statusCode = error.response && error.response.status;
+
+    if (statusCode === 429) return true;
+
+    if (errorCode === 'ETIMEDOUT' || errorCode === 'ECONNRESET') return true;
+
+    if (
+      errorMessage.includes('timeout') ||
+      errorMessage.includes('ETIMEDOUT') ||
+      errorMessage.includes('429') ||
+      errorMessage.includes('Too Many Requests') ||
+      errorMessage.includes('rate limit')
+    ) {
+      return true;
+    }
+
+    if (statusCode >= 500 && statusCode < 600) return true;
+
+    return false;
+  }
+}
+
+module.exports = RequestQueue;