From 1b815e33209eb7facc97b4365f64e4b6fecd1dbe Mon Sep 17 00:00:00 2001 From: hemant-fundwave Date: Mon, 11 Dec 2023 11:58:10 +0530 Subject: [PATCH 01/13] feat: pauseQueue config option + methods --- jest.config.js | 1 + package.json | 2 +- src/index.ts | 46 ++++++++++++++++++++++++++----------- src/interfaces/index.ts | 1 + tests/index.test.ts | 51 +++++++++++++++++++++++++++++++---------- 5 files changed, 75 insertions(+), 26 deletions(-) diff --git a/jest.config.js b/jest.config.js index bf20df0..b8dde5d 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,6 +1,7 @@ export default { preset: "ts-jest", testEnvironment: "node", + fakeTimers: { enableGlobally: true }, transform: { "^.+\\.(ts|tsx)$": "ts-jest", }, diff --git a/package.json b/package.json index 0faf0c5..ba67e87 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build:esm": "tsc --module NodeNext --outDir ./dist/esm", "build": "tsc --module commonjs --outDir ./dist/cjs", "prepare": "npm run build && npm run build:esm", - "test": "jest", + "test": "jest --detectOpenHandles", "lint": "npx eslint src/index.ts", "docs": "npx jsdoc-to-markdown ./src/index.ts --configure ./jsdoc2md.json > DOCUMENTATION.md" }, diff --git a/src/index.ts b/src/index.ts index 222ff2c..985ff45 100644 --- a/src/index.ts +++ b/src/index.ts @@ -35,6 +35,11 @@ export class FetchQueue { */ private _queue: Array<() => void>; + /** + * If true, Disables task executions but {@link _queue} gets populated. + */ + private _pauseQueue: boolean; + /** * Initializes a new instance of the FetchQueue class with an optional FetchQueueConfig object. * If no options are provided, the default concurrent value is set to 3. @@ -47,6 +52,7 @@ export class FetchQueue { this._queue = []; this._urlsQueued = []; this._urlsExecuting = []; + this._pauseQueue = options?.pauseQueueOnInit || false; if (typeof this._concurrent !== "number" || this._concurrent <= 0) { throw new Error("Concurrent should be a number greater than zero."); @@ -61,10 +67,7 @@ export class FetchQueue { * @param options - The options for the fetch request. * @returns A Promise that resolves to the fetch response. */ - private _run = async ( - url: URL | RequestInfo, - options?: RequestInit - ): Promise => { + private _run = async (url: URL | RequestInfo, options?: RequestInit): Promise => { this._activeRequests++; try { if (this._debug) { @@ -93,7 +96,7 @@ export class FetchQueue { this._urlsQueued.shift(); } } - if (this._queue.length <= 0) return; + if (this._queue.length <= 0 || this._pauseQueue) return; const nextTask = this._queue.shift(); nextTask!(); }; @@ -136,6 +139,28 @@ export class FetchQueue { this._debug = debug; } + /** + * Disables the queuing of fetch requests in the FetchQueue. + * Sets the `_disableQueue` property to true and the `_activeRequests` property to 0. + * + * @returns {void} + */ + public pauseQueue(): void { + this._pauseQueue = true; + this._activeRequests = 0; + } + + /** + * Enables the queuing of fetch requests in the FetchQueue. + * Sets the `_disableQueue` property to false and calls the `_emitRequestCompletedEvent` method. + * + * @returns {void} + */ + public startQueue(): void { + this._pauseQueue = false; + this._emitRequestCompletedEvent(); + } + /** * @returns Length of queue */ @@ -147,13 +172,10 @@ export class FetchQueue { * The internal fetch implementation that handles queuing of fetch requests. */ private _f_fetch = (() => { - return ( - url: RequestInfo | URL, - options?: RequestInit - ): Promise => { + return (url: RequestInfo | URL, options?: RequestInit): Promise => { const task = () => this._run(url, options); - if (this._activeRequests < this._concurrent) { + if (this._activeRequests < this._concurrent && !this._pauseQueue) { return task(); } else { return new Promise((resolve, reject) => { @@ -162,9 +184,7 @@ export class FetchQueue { }; this._queue.push(queueTask); if (this._debug) { - this._urlsQueued.push( - url.toString().split("/").slice(-3).join("/") - ); + this._urlsQueued.push(url.toString().split("/").slice(-3).join("/")); } }); } diff --git a/src/interfaces/index.ts b/src/interfaces/index.ts index a99a293..d44c6fb 100644 --- a/src/interfaces/index.ts +++ b/src/interfaces/index.ts @@ -1,4 +1,5 @@ export type FetchQueueConfig = { concurrent: number; + pauseQueueOnInit?: boolean debug?: boolean; }; diff --git a/tests/index.test.ts b/tests/index.test.ts index 590c9c9..cbf6e7f 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,11 +1,6 @@ import { FetchQueue } from "../src/index"; -const urls = [ - "https://example.com/", - "https://github.com/", - "https://example.com/3", - "https://google.com/", -]; +const urls = ["https://example.com/", "https://github.com/", "https://example.com/3", "https://google.com/"]; describe("FetchQueue", () => { // test @@ -15,21 +10,20 @@ describe("FetchQueue", () => { // test it("should execute multiple fetch requests with expected queue lengths", async () => { - jest.useFakeTimers(); - const fetchQueue = new FetchQueue({ concurrent: 2 }); + const fetchQueue = new FetchQueue({ concurrent: 1 }); const fetch = fetchQueue.getFetchMethod(); const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { jest.advanceTimersByTime(5000); switch (urlIndex) { + case 0: case 1: - case 2: expect(fetchQueue.getQueueLength()).toBe(0); break; - case 3: + case 2: expect(fetchQueue.getQueueLength()).toBe(1); break; - case 4: + case 3: expect(fetchQueue.getQueueLength()).toBe(2); break; } @@ -101,7 +95,40 @@ describe("FetchQueue", () => { }, 60000); afterEach(() => { - jest.useRealTimers(); jest.clearAllMocks(); }); }); + +describe("FetchQueue with configuration", () => { + //test + it("should not execute fetch request with disableQueue true in config", async () => { + const fetchQueue = new FetchQueue({ concurrent: 2, pauseQueueOnInit: true }); + const fetch = fetchQueue.getFetchMethod(); + + const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { + fetch(url); + jest.advanceTimersByTime(5000); + switch (urlIndex) { + case 0: + case 1: + expect(fetchQueue["_activeRequests"]).toBe(0); + break; + case 2: + expect(fetchQueue["_activeRequests"]).toBe(0); + expect(fetchQueue.getQueueLength()).toBe(3); + break; + case 3: + expect(fetchQueue["_activeRequests"]).toBe(0); + expect(fetchQueue.getQueueLength()).toBe(4); + break; + } + return Promise.resolve(true); + }); + + const promises = urls.map((url) => mockFetch(url, urls.indexOf(url))); + await Promise.all(promises); + + fetchQueue.startQueue(); + expect(fetchQueue.getQueueLength()).toBe(3); + }); +}); From cae7a50d42aec56f1f5380ab7d0bbb908a38903c Mon Sep 17 00:00:00 2001 From: hemant-fundwave Date: Mon, 11 Dec 2023 12:22:37 +0530 Subject: [PATCH 02/13] update: readme doc --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 6086edb..8d558ed 100644 --- a/README.md +++ b/README.md @@ -56,3 +56,15 @@ console.log(fetchedQueue.getConcurrent()); // output: 5 fetchQueue.setDebug(true); console.log(fetchedQueue.getDebug()); // output: true ``` + +```js +// start and pause queue +const fetchQueue = new FetchQueue(concurrent: 3, pauseQueueOnInit: true); +const customFetch = fetchQueue.getFetchMethod(); + +// ...some calls +fetchQueue.startQueue(); + +// ...some calls +fetcheQueue.pauseQueue(); +``` \ No newline at end of file From c7bc2aa29bf7a977f5333626cf782f349b05bcd4 Mon Sep 17 00:00:00 2001 From: fundabot Date: Mon, 11 Dec 2023 06:53:43 +0000 Subject: [PATCH 03/13] CI: bumps @fundwave/fetch-queue to 1.1.2-disable-queue.0 [skip ci] --- DOCUMENTATION.md | 16 ++++++++++++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 109df6e..2f6f706 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -13,6 +13,8 @@ It ensures that the number of active requests does not exceed a specified limit, * [.setConcurrent(concurrent)](#FetchQueue+setConcurrent) * [.getDebug()](#FetchQueue+getDebug) ⇒ * [.setDebug(debug)](#FetchQueue+setDebug) + * [.pauseQueue()](#FetchQueue+pauseQueue) ⇒ void + * [.startQueue()](#FetchQueue+startQueue) ⇒ void * [.getQueueLength()](#FetchQueue+getQueueLength) ⇒ @@ -65,6 +67,20 @@ If no options are provided, the default concurrent value is set to 3.

| --- | --- | | debug | boolean | + + +### fetchQueue.pauseQueue() ⇒ void +

Disables the queuing of fetch requests in the FetchQueue. +Sets the _disableQueue property to true and the _activeRequests property to 0.

+ +**Kind**: instance method of [FetchQueue](#FetchQueue) + + +### fetchQueue.startQueue() ⇒ void +

Enables the queuing of fetch requests in the FetchQueue. +Sets the _disableQueue property to false and calls the _emitRequestCompletedEvent method.

+ +**Kind**: instance method of [FetchQueue](#FetchQueue) ### fetchQueue.getQueueLength() ⇒ diff --git a/package-lock.json b/package-lock.json index e0f624e..85370f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fundwave/fetchq", - "version": "1.1.1", + "version": "1.1.2-disable-queue.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fundwave/fetchq", - "version": "1.1.1", + "version": "1.1.2-disable-queue.0", "license": "MIT", "dependencies": { "node-fetch": "^3.3.2", diff --git a/package.json b/package.json index ba67e87..0e312fe 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fundwave/fetchq", - "version": "1.1.1", + "version": "1.1.2-disable-queue.0", "description": "Queue for fetch requests", "main": "dist/esm/index.js", "types": "dist/esm/index.d.ts", From 879b792fdc3eb588fb976af040eebbccbb8ed890 Mon Sep 17 00:00:00 2001 From: hemant-fundwave Date: Tue, 9 Jan 2024 13:56:43 +0530 Subject: [PATCH 04/13] chore: refactor private fields to ECMAScript hash + additonal test case --- src/index.ts | 113 +++++++++++++++++++++++--------------------- tests/index.test.ts | 49 ++++++++++++++++--- 2 files changed, 101 insertions(+), 61 deletions(-) diff --git a/src/index.ts b/src/index.ts index 985ff45..8e2818b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,36 +9,36 @@ export class FetchQueue { /** * The maximum number of concurrent fetch requests allowed. */ - private _concurrent: number; + #concurrent: number; /** * Indicates whether debugging is enabled or not. */ - private _debug: boolean; + #debug: boolean; /** * An array of strings representing the URLs in the queue. */ - private _urlsQueued: Array; + #urlsQueued: Array; /** * An array of strings representing the URLs executing. */ - private _urlsExecuting: Array; + #urlsExecuting: Array; /** * The current number of active fetch requests. */ - private _activeRequests: number; + #activeRequests: number; /** * A queue of tasks to be executed when a slot becomes available for a new fetch request. */ - private _queue: Array<() => void>; + #queue: Array<() => void>; /** - * If true, Disables task executions but {@link _queue} gets populated. + * If true, Disables task executions but {@link #queue} gets populated. */ - private _pauseQueue: boolean; + #pauseQueue: boolean; /** * Initializes a new instance of the FetchQueue class with an optional FetchQueueConfig object. @@ -46,15 +46,15 @@ export class FetchQueue { * @param {FetchQueueConfig} options - The FetchQueueConfig object containing the concurrent value. */ constructor(options?: FetchQueueConfig) { - this._concurrent = options?.concurrent || 3; - this._debug = options?.debug || false; - this._activeRequests = 0; - this._queue = []; - this._urlsQueued = []; - this._urlsExecuting = []; - this._pauseQueue = options?.pauseQueueOnInit || false; - - if (typeof this._concurrent !== "number" || this._concurrent <= 0) { + this.#concurrent = options?.concurrent || 3; + this.#debug = options?.debug || false; + this.#activeRequests = 0; + this.#queue = []; + this.#urlsQueued = []; + this.#urlsExecuting = []; + this.#pauseQueue = options?.pauseQueueOnInit || false; + + if (typeof this.#concurrent !== "number" || this.#concurrent <= 0) { throw new Error("Concurrent should be a number greater than zero."); } } @@ -67,37 +67,37 @@ export class FetchQueue { * @param options - The options for the fetch request. * @returns A Promise that resolves to the fetch response. */ - private _run = async (url: URL | RequestInfo, options?: RequestInit): Promise => { - this._activeRequests++; + #run = async (url: URL | RequestInfo, options?: RequestInit): Promise => { + this.#activeRequests++; try { - if (this._debug) { - this._urlsExecuting.push(url.toString()); - console.log("executing", this._urlsExecuting); + if (this.#debug) { + this.#urlsExecuting.push(url.toString()); + console.log("executing", this.#urlsExecuting); } const response: Response = await fetch(url, options); - if (this._debug) { - const index = this._urlsExecuting.indexOf(url.toString()); - this._urlsExecuting.splice(index, 1); + if (this.#debug) { + const index = this.#urlsExecuting.indexOf(url.toString()); + this.#urlsExecuting.splice(index, 1); } return response; } finally { - this._activeRequests--; - this._emitRequestCompletedEvent(); + this.#activeRequests--; + this.#emitRequestCompletedEvent(); } }; /** * Executes the next task in the queue when a fetch request is completed. */ - private _emitRequestCompletedEvent = (): void => { - if (this._debug) { - if (this._urlsQueued.length > 0) { - console.log("queue", this._urlsQueued); - this._urlsQueued.shift(); + #emitRequestCompletedEvent = (): void => { + if (this.#debug) { + if (this.#urlsQueued.length > 0) { + console.log("queue", this.#urlsQueued); + this.#urlsQueued.shift(); } } - if (this._queue.length <= 0 || this._pauseQueue) return; - const nextTask = this._queue.shift(); + if (this.#queue.length <= 0 || this.#pauseQueue) return; + const nextTask = this.#queue.shift(); nextTask!(); }; @@ -106,14 +106,14 @@ export class FetchQueue { * @returns The custom fetch function. */ public getFetchMethod() { - return this._f_fetch; + return this.#f_fetch; } /** * @returns value of concurrent property */ public getConcurrent() { - return this._concurrent; + return this.#concurrent; } /** @@ -121,14 +121,14 @@ export class FetchQueue { * @param {number} concurrent */ public setConcurrent(concurrent: number) { - this._concurrent = concurrent; + this.#concurrent = concurrent; } /** * @returns value of debug property */ public getDebug() { - return this._debug; + return this.#debug; } /** @@ -136,55 +136,58 @@ export class FetchQueue { * @param {boolean} debug */ public setDebug(debug: boolean) { - this._debug = debug; + this.#debug = debug; } /** * Disables the queuing of fetch requests in the FetchQueue. - * Sets the `_disableQueue` property to true and the `_activeRequests` property to 0. - * * @returns {void} */ public pauseQueue(): void { - this._pauseQueue = true; - this._activeRequests = 0; + this.#pauseQueue = true; + this.#activeRequests = 0; } /** * Enables the queuing of fetch requests in the FetchQueue. - * Sets the `_disableQueue` property to false and calls the `_emitRequestCompletedEvent` method. - * * @returns {void} */ public startQueue(): void { - this._pauseQueue = false; - this._emitRequestCompletedEvent(); + this.#pauseQueue = false; + this.#emitRequestCompletedEvent(); } /** * @returns Length of queue */ - public getQueueLength() { - return this._queue.length; + public getQueueLength(): number { + return this.#queue.length; + } + + /** + * @returns Number of active requests + */ + public getActiveRequests(): number { + return this.#activeRequests; } /** * The internal fetch implementation that handles queuing of fetch requests. */ - private _f_fetch = (() => { + #f_fetch = (() => { return (url: RequestInfo | URL, options?: RequestInit): Promise => { - const task = () => this._run(url, options); + const task = () => this.#run(url, options); - if (this._activeRequests < this._concurrent && !this._pauseQueue) { + if (this.#activeRequests < this.#concurrent && !this.#pauseQueue) { return task(); } else { return new Promise((resolve, reject) => { const queueTask = () => { task().then(resolve).catch(reject); }; - this._queue.push(queueTask); - if (this._debug) { - this._urlsQueued.push(url.toString().split("/").slice(-3).join("/")); + this.#queue.push(queueTask); + if (this.#debug) { + this.#urlsQueued.push(url.toString().split("/").slice(-3).join("/")); } }); } diff --git a/tests/index.test.ts b/tests/index.test.ts index cbf6e7f..16b5e4a 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -30,7 +30,7 @@ describe("FetchQueue", () => { return Promise.resolve(await fetch(url)); }); - const promises = urls.map((url) => mockFetch(url, urls.indexOf(url))); + const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); await Promise.all(promises); }); @@ -99,7 +99,7 @@ describe("FetchQueue", () => { }); }); -describe("FetchQueue with configuration", () => { +describe("test case with start and pause queue", () => { //test it("should not execute fetch request with disableQueue true in config", async () => { const fetchQueue = new FetchQueue({ concurrent: 2, pauseQueueOnInit: true }); @@ -111,24 +111,61 @@ describe("FetchQueue with configuration", () => { switch (urlIndex) { case 0: case 1: - expect(fetchQueue["_activeRequests"]).toBe(0); + expect(fetchQueue.getActiveRequests()).toBe(0); break; case 2: - expect(fetchQueue["_activeRequests"]).toBe(0); + expect(fetchQueue.getActiveRequests()).toBe(0); expect(fetchQueue.getQueueLength()).toBe(3); break; case 3: - expect(fetchQueue["_activeRequests"]).toBe(0); + expect(fetchQueue.getActiveRequests()).toBe(0); expect(fetchQueue.getQueueLength()).toBe(4); break; } return Promise.resolve(true); }); - const promises = urls.map((url) => mockFetch(url, urls.indexOf(url))); + const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); await Promise.all(promises); fetchQueue.startQueue(); expect(fetchQueue.getQueueLength()).toBe(3); }); + + it("execute a single fetch queue and pause others on completion", async () => { + const fetchQueue = new FetchQueue({ concurrent: 2 }); + const fetch = fetchQueue.getFetchMethod(); + + const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { + jest.advanceTimersByTime(5000); + switch (urlIndex) { + case 0: + expect(fetchQueue.getActiveRequests()).toBe(0); + expect(fetchQueue.getQueueLength()).toBe(0); + break; + case 1: + expect(fetchQueue.getActiveRequests()).toBe(1); + expect(fetchQueue.getQueueLength()).toBe(0); + fetchQueue.pauseQueue(); + expect(fetchQueue.getQueueLength()).toBe(0); + expect(fetchQueue.getActiveRequests()).toBe(0); + break; + case 2: + expect(fetchQueue.getQueueLength()).toBe(1); + expect(fetchQueue.getActiveRequests()).toBe(0); + break; + case 3: + fetchQueue.startQueue(); + expect(fetchQueue.getQueueLength()).toBe(1); + expect(fetchQueue.getActiveRequests()).toBe(1); + break; + } + return Promise.resolve(fetch(url)); + }); + + const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); + await Promise.all(promises); + + expect(fetchQueue.getQueueLength()).toBe(0); + }); }); From 363a68264e6ac47a0869851b570f5376f7e17d05 Mon Sep 17 00:00:00 2001 From: fundabot Date: Tue, 9 Jan 2024 08:30:58 +0000 Subject: [PATCH 05/13] CI: bumps @fundwave/fetch-queue to 1.1.2-disable-queue.1 [skip ci] --- DOCUMENTATION.md | 12 ++++++++---- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 2f6f706..4e1aa66 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -16,6 +16,7 @@ It ensures that the number of active requests does not exceed a specified limit, * [.pauseQueue()](#FetchQueue+pauseQueue) ⇒ void * [.startQueue()](#FetchQueue+startQueue) ⇒ void * [.getQueueLength()](#FetchQueue+getQueueLength) ⇒ + * [.getActiveRequests()](#FetchQueue+getActiveRequests) ⇒ @@ -70,15 +71,13 @@ If no options are provided, the default concurrent value is set to 3.

### fetchQueue.pauseQueue() ⇒ void -

Disables the queuing of fetch requests in the FetchQueue. -Sets the _disableQueue property to true and the _activeRequests property to 0.

+

Disables the queuing of fetch requests in the FetchQueue.

**Kind**: instance method of [FetchQueue](#FetchQueue) ### fetchQueue.startQueue() ⇒ void -

Enables the queuing of fetch requests in the FetchQueue. -Sets the _disableQueue property to false and calls the _emitRequestCompletedEvent method.

+

Enables the queuing of fetch requests in the FetchQueue.

**Kind**: instance method of [FetchQueue](#FetchQueue) @@ -86,3 +85,8 @@ Sets the _disableQueue property to false and calls the _emitR ### fetchQueue.getQueueLength() ⇒ **Kind**: instance method of [FetchQueue](#FetchQueue) **Returns**:

Length of queue

+ + +### fetchQueue.getActiveRequests() ⇒ +**Kind**: instance method of [FetchQueue](#FetchQueue) +**Returns**:

Number of active requests

diff --git a/package-lock.json b/package-lock.json index 85370f0..342d052 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.0", + "version": "1.1.2-disable-queue.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.0", + "version": "1.1.2-disable-queue.1", "license": "MIT", "dependencies": { "node-fetch": "^3.3.2", diff --git a/package.json b/package.json index 0e312fe..67a60ae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.0", + "version": "1.1.2-disable-queue.1", "description": "Queue for fetch requests", "main": "dist/esm/index.js", "types": "dist/esm/index.d.ts", From 1e29732c3a7f32afeb6d9e9f42c13392055466a2 Mon Sep 17 00:00:00 2001 From: hemant-fundwave Date: Thu, 4 Apr 2024 18:46:18 +0530 Subject: [PATCH 06/13] update: add emprtyQueue option while starting queue --- src/index.ts | 25 ++++++++++++++----------- tests/index.test.ts | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/index.ts b/src/index.ts index 8e2818b..a9a2403 100644 --- a/src/index.ts +++ b/src/index.ts @@ -150,9 +150,13 @@ export class FetchQueue { /** * Enables the queuing of fetch requests in the FetchQueue. + * @param {boolean} [emptyQueue] If true, empties the queue before starting. * @returns {void} */ - public startQueue(): void { + public startQueue(emptyQueue?: boolean): void { + if (emptyQueue) this.#queue = []; + if (this.#debug && emptyQueue) this.#urlsQueued = []; + this.#pauseQueue = false; this.#emitRequestCompletedEvent(); } @@ -180,17 +184,16 @@ export class FetchQueue { if (this.#activeRequests < this.#concurrent && !this.#pauseQueue) { return task(); - } else { - return new Promise((resolve, reject) => { - const queueTask = () => { - task().then(resolve).catch(reject); - }; - this.#queue.push(queueTask); - if (this.#debug) { - this.#urlsQueued.push(url.toString().split("/").slice(-3).join("/")); - } - }); } + return new Promise((resolve, reject) => { + const queueTask = () => { + task().then(resolve).catch(reject); + }; + this.#queue.push(queueTask); + if (this.#debug) { + this.#urlsQueued.push(url.toString().split("/").slice(-3).join("/")); + } + }); }; })(); } diff --git a/tests/index.test.ts b/tests/index.test.ts index 16b5e4a..c30ed4e 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -168,4 +168,41 @@ describe("test case with start and pause queue", () => { expect(fetchQueue.getQueueLength()).toBe(0); }); + + // it("start fetchQueue with emptyQueue set to true", async () => { + // const fetchQueue = new FetchQueue({ concurrent: 1 }); + // const fetch = fetchQueue.getFetchMethod(); + + // const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { + // switch (urlIndex) { + // case 0: + // expect(fetchQueue.getActiveRequests()).toBe(0); + // expect(fetchQueue.getQueueLength()).toBe(0); + // fetchQueue.pauseQueue(); + // break; + // case 1: + // expect(fetchQueue.getQueueLength()).toBe(1); + // expect(fetchQueue.getActiveRequests()).toBe(0); + // break; + // case 2: + // expect(fetchQueue.getQueueLength()).toBe(2); + // expect(fetchQueue.getActiveRequests()).toBe(0); + // fetchQueue.startQueue(true); + // break; + // case 3: + // expect(fetchQueue.getQueueLength()).toBe(0); + // expect(fetchQueue.getActiveRequests()).toBe(1); + // break; + // } + + // await fetch(url); + // jest.advanceTimersByTime(5000); + // return; + // }); + + // const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); + // await Promise.all(promises); + + // expect(fetchQueue.getQueueLength()).toBe(0); + // }); }); From 9066381126fe10f85e2386a25e2b464ab3e399aa Mon Sep 17 00:00:00 2001 From: fundabot Date: Thu, 4 Apr 2024 13:17:33 +0000 Subject: [PATCH 07/13] CI: bumps @fundwave/fetch-queue to 1.1.2-disable-queue.2 [skip ci] --- DOCUMENTATION.md | 9 +++++++-- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index 4e1aa66..fea9758 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -14,7 +14,7 @@ It ensures that the number of active requests does not exceed a specified limit, * [.getDebug()](#FetchQueue+getDebug) ⇒ * [.setDebug(debug)](#FetchQueue+setDebug) * [.pauseQueue()](#FetchQueue+pauseQueue) ⇒ void - * [.startQueue()](#FetchQueue+startQueue) ⇒ void + * [.startQueue([emptyQueue])](#FetchQueue+startQueue) ⇒ void * [.getQueueLength()](#FetchQueue+getQueueLength) ⇒ * [.getActiveRequests()](#FetchQueue+getActiveRequests) ⇒ @@ -76,10 +76,15 @@ If no options are provided, the default concurrent value is set to 3.

**Kind**: instance method of [FetchQueue](#FetchQueue) -### fetchQueue.startQueue() ⇒ void +### fetchQueue.startQueue([emptyQueue]) ⇒ void

Enables the queuing of fetch requests in the FetchQueue.

**Kind**: instance method of [FetchQueue](#FetchQueue) + +| Param | Type | Description | +| --- | --- | --- | +| [emptyQueue] | boolean |

If true, empties the queue before starting.

| + ### fetchQueue.getQueueLength() ⇒ diff --git a/package-lock.json b/package-lock.json index 342d052..f68074c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.1", + "version": "1.1.2-disable-queue.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.1", + "version": "1.1.2-disable-queue.2", "license": "MIT", "dependencies": { "node-fetch": "^3.3.2", diff --git a/package.json b/package.json index 67a60ae..2f1a4a9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.1", + "version": "1.1.2-disable-queue.2", "description": "Queue for fetch requests", "main": "dist/esm/index.js", "types": "dist/esm/index.d.ts", From ab370efdfe91ce57a4c72c311bd714b56e762b5c Mon Sep 17 00:00:00 2001 From: hemant-fundwave Date: Fri, 5 Apr 2024 12:22:05 +0530 Subject: [PATCH 08/13] update: use abort controller to resolve emptied requests [skip ci] --- src/index.ts | 36 +++++++++++++------- tests/index.test.ts | 80 +++++++++++++++++++++++++-------------------- 2 files changed, 68 insertions(+), 48 deletions(-) diff --git a/src/index.ts b/src/index.ts index a9a2403..afe70de 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,4 @@ -import fetch from "node-fetch"; -import { RequestInfo, RequestInit, Response } from "node-fetch"; +import fetch, { RequestInfo, RequestInit, Response } from "node-fetch"; import { FetchQueueConfig } from "./interfaces/index.js"; /** * The `FetchQueue` class is a utility class that allows for managing and controlling concurrent fetch requests. @@ -19,7 +18,7 @@ export class FetchQueue { /** * An array of strings representing the URLs in the queue. */ - #urlsQueued: Array; + #urlsQueued: Array<{ url: string; controller: AbortController }>; /** * An array of strings representing the URLs executing. @@ -67,18 +66,26 @@ export class FetchQueue { * @param options - The options for the fetch request. * @returns A Promise that resolves to the fetch response. */ - #run = async (url: URL | RequestInfo, options?: RequestInit): Promise => { + #run = async (url: URL | RequestInfo, options?: RequestInit, controller?: AbortController): Promise => { this.#activeRequests++; try { if (this.#debug) { this.#urlsExecuting.push(url.toString()); console.log("executing", this.#urlsExecuting); } + + if (!!controller && controller.signal.aborted) { + console.log({ signal: controller.signal.aborted }); + return new Response(); + } + const response: Response = await fetch(url, options); + if (this.#debug) { const index = this.#urlsExecuting.indexOf(url.toString()); this.#urlsExecuting.splice(index, 1); } + return response; } finally { this.#activeRequests--; @@ -154,8 +161,14 @@ export class FetchQueue { * @returns {void} */ public startQueue(emptyQueue?: boolean): void { - if (emptyQueue) this.#queue = []; - if (this.#debug && emptyQueue) this.#urlsQueued = []; + if (emptyQueue) { + this.#urlsQueued.forEach((request) => { + console.log(request); + request.controller.abort(); + console.log({ abort1: request.controller.signal.aborted }); + }); + this.#urlsQueued = []; + } this.#pauseQueue = false; this.#emitRequestCompletedEvent(); @@ -180,19 +193,18 @@ export class FetchQueue { */ #f_fetch = (() => { return (url: RequestInfo | URL, options?: RequestInit): Promise => { - const task = () => this.#run(url, options); + const controller = new AbortController(); + const executeFetchRequest = (controller: AbortController) => this.#run(url, options, controller); if (this.#activeRequests < this.#concurrent && !this.#pauseQueue) { - return task(); + return executeFetchRequest(controller); } return new Promise((resolve, reject) => { const queueTask = () => { - task().then(resolve).catch(reject); + executeFetchRequest(controller).then(resolve).catch(reject); }; this.#queue.push(queueTask); - if (this.#debug) { - this.#urlsQueued.push(url.toString().split("/").slice(-3).join("/")); - } + this.#urlsQueued.push({ url: url.toString().split("/").slice(-3).join("/"), controller }); }); }; })(); diff --git a/tests/index.test.ts b/tests/index.test.ts index c30ed4e..85dca7c 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -2,6 +2,14 @@ import { FetchQueue } from "../src/index"; const urls = ["https://example.com/", "https://github.com/", "https://example.com/3", "https://google.com/"]; +async function wait(time: number = 1200) { + return new Promise((resolve) => { + setTimeout(function () { + return resolve(true); + }, time); + }); +} + describe("FetchQueue", () => { // test it("should not initialize FetchQueue with negative concurrent value", () => { @@ -169,40 +177,40 @@ describe("test case with start and pause queue", () => { expect(fetchQueue.getQueueLength()).toBe(0); }); - // it("start fetchQueue with emptyQueue set to true", async () => { - // const fetchQueue = new FetchQueue({ concurrent: 1 }); - // const fetch = fetchQueue.getFetchMethod(); - - // const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { - // switch (urlIndex) { - // case 0: - // expect(fetchQueue.getActiveRequests()).toBe(0); - // expect(fetchQueue.getQueueLength()).toBe(0); - // fetchQueue.pauseQueue(); - // break; - // case 1: - // expect(fetchQueue.getQueueLength()).toBe(1); - // expect(fetchQueue.getActiveRequests()).toBe(0); - // break; - // case 2: - // expect(fetchQueue.getQueueLength()).toBe(2); - // expect(fetchQueue.getActiveRequests()).toBe(0); - // fetchQueue.startQueue(true); - // break; - // case 3: - // expect(fetchQueue.getQueueLength()).toBe(0); - // expect(fetchQueue.getActiveRequests()).toBe(1); - // break; - // } - - // await fetch(url); - // jest.advanceTimersByTime(5000); - // return; - // }); - - // const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); - // await Promise.all(promises); - - // expect(fetchQueue.getQueueLength()).toBe(0); - // }); + it("start fetchQueue with emptyQueue set to true", async () => { + const fetchQueue = new FetchQueue({ concurrent: 1, debug: true }); + const fetch = fetchQueue.getFetchMethod(); + + const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { + // await wait(500); + console.log(Date.now()); + switch (urlIndex) { + case 0: + fetchQueue.pauseQueue(); + expect(fetchQueue.getQueueLength()).toBe(0); + expect(fetchQueue.getActiveRequests()).toBe(0); + break; + case 1: + expect(fetchQueue.getQueueLength()).toBe(1); + expect(fetchQueue.getActiveRequests()).toBe(0); + break; + case 2: + expect(fetchQueue.getQueueLength()).toBe(2); + expect(fetchQueue.getActiveRequests()).toBe(0); + fetchQueue.startQueue(true); + break; + case 3: + expect(fetchQueue.getQueueLength()).toBe(0); + expect(fetchQueue.getActiveRequests()).toBe(1); + break; + } + + return await fetch(url); + }); + + const promises = urls.map(async (url, urlIndex) => await mockFetch(url, urlIndex)); + await Promise.all(promises); + + expect(fetchQueue.getQueueLength()).toBe(0); + }); }); From 30e7b3e6e769e4f89ee619b3d0c6f2fb0707a687 Mon Sep 17 00:00:00 2001 From: hemant-fundwave Date: Fri, 5 Apr 2024 15:44:16 +0530 Subject: [PATCH 09/13] feat: new emptyQueue method - which accepts regexp to include urls to be emptied --- README.md | 7 ++++-- src/index.ts | 60 ++++++++++++++++++++++++--------------------- tests/index.test.ts | 44 +++++++++++++++++++-------------- 3 files changed, 63 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 8d558ed..e96d9d0 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,11 @@ const fetchQueue = new FetchQueue(concurrent: 3, pauseQueueOnInit: true); const customFetch = fetchQueue.getFetchMethod(); // ...some calls +fetchQueue.emptyQueue(); fetchQueue.startQueue(); // ...some calls -fetcheQueue.pauseQueue(); -``` \ No newline at end of file +fetchQueue.pauseQueue(); +``` + +Note: See DOCUMENTATION.md for more information on methods. \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index afe70de..2207979 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,7 +23,7 @@ export class FetchQueue { /** * An array of strings representing the URLs executing. */ - #urlsExecuting: Array; + #urlsExecuting: Set; /** * The current number of active fetch requests. @@ -50,7 +50,7 @@ export class FetchQueue { this.#activeRequests = 0; this.#queue = []; this.#urlsQueued = []; - this.#urlsExecuting = []; + this.#urlsExecuting = new Set(); this.#pauseQueue = options?.pauseQueueOnInit || false; if (typeof this.#concurrent !== "number" || this.#concurrent <= 0) { @@ -67,24 +67,21 @@ export class FetchQueue { * @returns A Promise that resolves to the fetch response. */ #run = async (url: URL | RequestInfo, options?: RequestInit, controller?: AbortController): Promise => { - this.#activeRequests++; try { if (this.#debug) { - this.#urlsExecuting.push(url.toString()); + this.#urlsExecuting.add(url.toString()); console.log("executing", this.#urlsExecuting); } if (!!controller && controller.signal.aborted) { - console.log({ signal: controller.signal.aborted }); - return new Response(); + if (this.#debug) this.#urlsExecuting.delete(url.toString()); + throw new Error("Aborted"); } + this.#activeRequests++; const response: Response = await fetch(url, options); - if (this.#debug) { - const index = this.#urlsExecuting.indexOf(url.toString()); - this.#urlsExecuting.splice(index, 1); - } + if (this.#debug) this.#urlsExecuting.delete(url.toString()); return response; } finally { @@ -97,13 +94,11 @@ export class FetchQueue { * Executes the next task in the queue when a fetch request is completed. */ #emitRequestCompletedEvent = (): void => { - if (this.#debug) { - if (this.#urlsQueued.length > 0) { - console.log("queue", this.#urlsQueued); - this.#urlsQueued.shift(); - } - } + if (this.#debug) console.log("queue", this.#urlsQueued); + if (this.#queue.length <= 0 || this.#pauseQueue) return; + + this.#urlsQueued.shift(); const nextTask = this.#queue.shift(); nextTask!(); }; @@ -146,6 +141,25 @@ export class FetchQueue { this.#debug = debug; } + /** + * Empties the queue of fetch requests. + * + * @param urlPattern - Optional regular expression to match against the URLs in the queue. + * If provided, only the requests with URLs that match the pattern will be aborted. + * If not provided, all requests in the queue will be aborted. + */ + public emptyQueue(urlPattern?: RegExp) { + this.#urlsQueued.forEach((request) => { + if (!urlPattern) request.controller.abort(); + else if (!!urlPattern && request.url.match(urlPattern)) request.controller.abort(); + }); + + this.#queue.forEach((task) => task()); + + this.#urlsQueued = []; + this.#queue = []; + } + /** * Disables the queuing of fetch requests in the FetchQueue. * @returns {void} @@ -157,19 +171,9 @@ export class FetchQueue { /** * Enables the queuing of fetch requests in the FetchQueue. - * @param {boolean} [emptyQueue] If true, empties the queue before starting. * @returns {void} */ - public startQueue(emptyQueue?: boolean): void { - if (emptyQueue) { - this.#urlsQueued.forEach((request) => { - console.log(request); - request.controller.abort(); - console.log({ abort1: request.controller.signal.aborted }); - }); - this.#urlsQueued = []; - } - + public startQueue(): void { this.#pauseQueue = false; this.#emitRequestCompletedEvent(); } @@ -204,7 +208,7 @@ export class FetchQueue { executeFetchRequest(controller).then(resolve).catch(reject); }; this.#queue.push(queueTask); - this.#urlsQueued.push({ url: url.toString().split("/").slice(-3).join("/"), controller }); + this.#urlsQueued.push({ url: url.toString(), controller }); }); }; })(); diff --git a/tests/index.test.ts b/tests/index.test.ts index 85dca7c..eeb0f99 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,6 +1,6 @@ import { FetchQueue } from "../src/index"; -const urls = ["https://example.com/", "https://github.com/", "https://example.com/3", "https://google.com/"]; +const urls = ["https://dummy.restapiexample.com/api/v1/employees", "https://dummyjson.com/products/1", "https://dummyjson.com/products/2", "https://dummyjson.com/products/3"]; async function wait(time: number = 1200) { return new Promise((resolve) => { @@ -14,7 +14,7 @@ describe("FetchQueue", () => { // test it("should not initialize FetchQueue with negative concurrent value", () => { expect(() => new FetchQueue({ concurrent: -1 })).toThrow(); - }); + }, 10000); // test it("should execute multiple fetch requests with expected queue lengths", async () => { @@ -35,12 +35,12 @@ describe("FetchQueue", () => { expect(fetchQueue.getQueueLength()).toBe(2); break; } - return Promise.resolve(await fetch(url)); + return await fetch(url); }); const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); await Promise.all(promises); - }); + }, 10000); // test it("should execute multiple fetch requests successfully except one", async () => { @@ -51,8 +51,8 @@ describe("FetchQueue", () => { const responses = await Promise.all(promises); expect(responses.filter((r) => r.status === 200)).toHaveLength(3); - expect(responses.find((r) => r.status === 404)).toBeDefined(); - }); + expect(responses.find((r) => r.status !== 200)).toBeDefined(); + }, 10000); // test it("should execute fetch requests with concurrency and after destroyQueue fetch requests parallel", async () => { @@ -65,9 +65,9 @@ describe("FetchQueue", () => { const run = async (url: URL | string): Promise => { activeRequests++; try { - startTime.push(new Date().getTime()); + startTime.push(Date.now()); const response = await require("node-fetch")(url); - endTime.push(new Date().getTime()); + endTime.push(Date.now()); return response; } catch (e) { throw e; @@ -116,17 +116,18 @@ describe("test case with start and pause queue", () => { const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { fetch(url); jest.advanceTimersByTime(5000); + expect(fetchQueue.getActiveRequests()).toBe(0); switch (urlIndex) { case 0: + expect(fetchQueue.getQueueLength()).toBe(1); + break; case 1: - expect(fetchQueue.getActiveRequests()).toBe(0); + expect(fetchQueue.getQueueLength()).toBe(2); break; case 2: - expect(fetchQueue.getActiveRequests()).toBe(0); expect(fetchQueue.getQueueLength()).toBe(3); break; case 3: - expect(fetchQueue.getActiveRequests()).toBe(0); expect(fetchQueue.getQueueLength()).toBe(4); break; } @@ -140,6 +141,7 @@ describe("test case with start and pause queue", () => { expect(fetchQueue.getQueueLength()).toBe(3); }); + // test it("execute a single fetch queue and pause others on completion", async () => { const fetchQueue = new FetchQueue({ concurrent: 2 }); const fetch = fetchQueue.getFetchMethod(); @@ -177,13 +179,12 @@ describe("test case with start and pause queue", () => { expect(fetchQueue.getQueueLength()).toBe(0); }); + //test it("start fetchQueue with emptyQueue set to true", async () => { - const fetchQueue = new FetchQueue({ concurrent: 1, debug: true }); + const fetchQueue = new FetchQueue({ concurrent: 1 }); const fetch = fetchQueue.getFetchMethod(); const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { - // await wait(500); - console.log(Date.now()); switch (urlIndex) { case 0: fetchQueue.pauseQueue(); @@ -195,9 +196,11 @@ describe("test case with start and pause queue", () => { expect(fetchQueue.getActiveRequests()).toBe(0); break; case 2: - expect(fetchQueue.getQueueLength()).toBe(2); + const regExp = /https:\/\/dummyjson\.com.*/g; + fetchQueue.emptyQueue(regExp); + expect(fetchQueue.getQueueLength()).toBe(0); expect(fetchQueue.getActiveRequests()).toBe(0); - fetchQueue.startQueue(true); + fetchQueue.startQueue(); break; case 3: expect(fetchQueue.getQueueLength()).toBe(0); @@ -205,12 +208,17 @@ describe("test case with start and pause queue", () => { break; } - return await fetch(url); + try { + await fetch(url).then((data) => data.json()); + } catch (error) { + } finally { + return; + } }); const promises = urls.map(async (url, urlIndex) => await mockFetch(url, urlIndex)); await Promise.all(promises); expect(fetchQueue.getQueueLength()).toBe(0); - }); + }, 20000); }); From 7fb4a46d5cbfa3ebf5b6e390391569449bbd192a Mon Sep 17 00:00:00 2001 From: fundabot Date: Fri, 5 Apr 2024 10:15:23 +0000 Subject: [PATCH 10/13] CI: bumps @fundwave/fetch-queue to 1.1.2-disable-queue.3 [skip ci] --- DOCUMENTATION.md | 21 ++++++++++++++------- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index fea9758..01de91a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -13,8 +13,9 @@ It ensures that the number of active requests does not exceed a specified limit, * [.setConcurrent(concurrent)](#FetchQueue+setConcurrent) * [.getDebug()](#FetchQueue+getDebug) ⇒ * [.setDebug(debug)](#FetchQueue+setDebug) + * [.emptyQueue(urlPattern)](#FetchQueue+emptyQueue) * [.pauseQueue()](#FetchQueue+pauseQueue) ⇒ void - * [.startQueue([emptyQueue])](#FetchQueue+startQueue) ⇒ void + * [.startQueue()](#FetchQueue+startQueue) ⇒ void * [.getQueueLength()](#FetchQueue+getQueueLength) ⇒ * [.getActiveRequests()](#FetchQueue+getActiveRequests) ⇒ @@ -68,6 +69,17 @@ If no options are provided, the default concurrent value is set to 3.

| --- | --- | | debug | boolean | + + +### fetchQueue.emptyQueue(urlPattern) +

Empties the queue of fetch requests.

+ +**Kind**: instance method of [FetchQueue](#FetchQueue) + +| Param | Description | +| --- | --- | +| urlPattern |

Optional regular expression to match against the URLs in the queue. If provided, only the requests with URLs that match the pattern will be aborted. If not provided, all requests in the queue will be aborted.

| + ### fetchQueue.pauseQueue() ⇒ void @@ -76,15 +88,10 @@ If no options are provided, the default concurrent value is set to 3.

**Kind**: instance method of [FetchQueue](#FetchQueue) -### fetchQueue.startQueue([emptyQueue]) ⇒ void +### fetchQueue.startQueue() ⇒ void

Enables the queuing of fetch requests in the FetchQueue.

**Kind**: instance method of [FetchQueue](#FetchQueue) - -| Param | Type | Description | -| --- | --- | --- | -| [emptyQueue] | boolean |

If true, empties the queue before starting.

| - ### fetchQueue.getQueueLength() ⇒ diff --git a/package-lock.json b/package-lock.json index f68074c..014c1d0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.2", + "version": "1.1.2-disable-queue.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.2", + "version": "1.1.2-disable-queue.3", "license": "MIT", "dependencies": { "node-fetch": "^3.3.2", diff --git a/package.json b/package.json index 2f1a4a9..45f71ca 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.2", + "version": "1.1.2-disable-queue.3", "description": "Queue for fetch requests", "main": "dist/esm/index.js", "types": "dist/esm/index.d.ts", From 277467b5eaa158c3b63393d75086b4e33e18c6c7 Mon Sep 17 00:00:00 2001 From: paras-verma Date: Fri, 5 Apr 2024 17:56:57 +0530 Subject: [PATCH 11/13] fix: use real-times @ tests#emptyQueue --- tests/index.test.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/index.test.ts b/tests/index.test.ts index eeb0f99..0ed61fa 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -180,11 +180,13 @@ describe("test case with start and pause queue", () => { }); //test - it("start fetchQueue with emptyQueue set to true", async () => { + it("use emptyQueue", async () => { + jest.useRealTimers(); const fetchQueue = new FetchQueue({ concurrent: 1 }); const fetch = fetchQueue.getFetchMethod(); const mockFetch = jest.fn().mockImplementation(async (url, urlIndex) => { + await wait(); switch (urlIndex) { case 0: fetchQueue.pauseQueue(); @@ -208,12 +210,7 @@ describe("test case with start and pause queue", () => { break; } - try { - await fetch(url).then((data) => data.json()); - } catch (error) { - } finally { - return; - } + await fetch(url).catch(() => {}); }); const promises = urls.map(async (url, urlIndex) => await mockFetch(url, urlIndex)); From c66dabc771d4cbf3ee3e66e304aa6d785a4cd245 Mon Sep 17 00:00:00 2001 From: paras-verma Date: Fri, 5 Apr 2024 17:58:50 +0530 Subject: [PATCH 12/13] update: corrputed url @ tests#urls chore: use 10s `TEST_TIMEOUT` @ tests --- package.json | 1 + tests/index.test.ts | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 45f71ca..f118e29 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "build:esm": "tsc --module NodeNext --outDir ./dist/esm", "build": "tsc --module commonjs --outDir ./dist/cjs", "prepare": "npm run build && npm run build:esm", + "pretest": "jest --clearCache", "test": "jest --detectOpenHandles", "lint": "npx eslint src/index.ts", "docs": "npx jsdoc-to-markdown ./src/index.ts --configure ./jsdoc2md.json > DOCUMENTATION.md" diff --git a/tests/index.test.ts b/tests/index.test.ts index 0ed61fa..b01c872 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -1,6 +1,6 @@ import { FetchQueue } from "../src/index"; -const urls = ["https://dummy.restapiexample.com/api/v1/employees", "https://dummyjson.com/products/1", "https://dummyjson.com/products/2", "https://dummyjson.com/products/3"]; +const urls = ["https://dummy.restapiexample.com/api/v1/fail", "https://dummyjson.com/products/1", "https://dummyjson.com/products/2", "https://dummyjson.com/products/3"]; async function wait(time: number = 1200) { return new Promise((resolve) => { @@ -10,11 +10,13 @@ async function wait(time: number = 1200) { }); } +const TEST_TIMEOUT = 10000; + describe("FetchQueue", () => { // test it("should not initialize FetchQueue with negative concurrent value", () => { expect(() => new FetchQueue({ concurrent: -1 })).toThrow(); - }, 10000); + }, TEST_TIMEOUT); // test it("should execute multiple fetch requests with expected queue lengths", async () => { @@ -40,7 +42,7 @@ describe("FetchQueue", () => { const promises = urls.map((url, urlIndex) => mockFetch(url, urlIndex)); await Promise.all(promises); - }, 10000); + }, TEST_TIMEOUT); // test it("should execute multiple fetch requests successfully except one", async () => { @@ -52,7 +54,7 @@ describe("FetchQueue", () => { expect(responses.filter((r) => r.status === 200)).toHaveLength(3); expect(responses.find((r) => r.status !== 200)).toBeDefined(); - }, 10000); + }, TEST_TIMEOUT); // test it("should execute fetch requests with concurrency and after destroyQueue fetch requests parallel", async () => { @@ -177,7 +179,7 @@ describe("test case with start and pause queue", () => { await Promise.all(promises); expect(fetchQueue.getQueueLength()).toBe(0); - }); + }, TEST_TIMEOUT); //test it("use emptyQueue", async () => { @@ -217,5 +219,5 @@ describe("test case with start and pause queue", () => { await Promise.all(promises); expect(fetchQueue.getQueueLength()).toBe(0); - }, 20000); + }, TEST_TIMEOUT); }); From 250894edb67964bc6ce640e3bdd86ea1cc6e0aed Mon Sep 17 00:00:00 2001 From: fundabot Date: Fri, 5 Apr 2024 12:31:53 +0000 Subject: [PATCH 13/13] CI: bumps @fundwave/fetch-queue to 1.1.2-disable-queue.4 [skip ci] --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 014c1d0..e570cac 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.3", + "version": "1.1.2-disable-queue.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.3", + "version": "1.1.2-disable-queue.4", "license": "MIT", "dependencies": { "node-fetch": "^3.3.2", diff --git a/package.json b/package.json index f118e29..b6706a2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@fundwave/fetchq", - "version": "1.1.2-disable-queue.3", + "version": "1.1.2-disable-queue.4", "description": "Queue for fetch requests", "main": "dist/esm/index.js", "types": "dist/esm/index.d.ts",