From 84d65bcd3e2b04610338ce50ddf55be83904194b Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Thu, 13 Feb 2025 22:48:51 +0800 Subject: [PATCH 1/2] fix: allow set rejectUnauthorized = true on urllib.request options closes https://github.com/node-modules/urllib/issues/531 pick from https://github.com/node-modules/urllib/pull/561 --- package.json | 2 +- src/HttpClient.ts | 5 +- src/index.ts | 54 ++++++++++++-- test/urllib.options.allowH2.test.ts | 19 +++++ ...b.options.rejectUnauthorized-false.test.ts | 71 +++++++++++++++++++ 5 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 test/urllib.options.allowH2.test.ts create mode 100644 test/urllib.options.rejectUnauthorized-false.test.ts diff --git a/package.json b/package.json index 4cae5cce..928f8336 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "git-contributor": "^2.0.0", "iconv-lite": "^0.6.3", "proxy": "^1.0.2", - "selfsigned": "^2.0.1", + "selfsigned": "^2.4.1", "tar-stream": "^2.2.0", "tshy": "^1.0.0", "tshy-after": "^1.0.0", diff --git a/src/HttpClient.ts b/src/HttpClient.ts index bd8b2d8a..03aed743 100644 --- a/src/HttpClient.ts +++ b/src/HttpClient.ts @@ -90,8 +90,9 @@ export type ClientOptions = { */ cert?: string | Buffer; /** - * If true, the server certificate is verified against the list of supplied CAs. - * An 'error' event is emitted if verification fails.Default: true. + * If `true`, the server certificate is verified against the list of supplied CAs. + * An 'error' event is emitted if verification fails. + * Default: `true` */ rejectUnauthorized?: boolean; /** diff --git a/src/index.ts b/src/index.ts index cc3cec4d..cc84b29a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,16 +3,62 @@ import { HttpClient, HEADER_USER_AGENT } from './HttpClient.js'; import { RequestOptions, RequestURL } from './Request.js'; let httpClient: HttpClient; +let allowH2HttpClient: HttpClient; +let allowUnauthorizedHttpClient: HttpClient; +let allowH2AndUnauthorizedHttpClient: HttpClient; const domainSocketHttpClients = new LRU(50); -export function getDefaultHttpClient(): HttpClient { +export function getDefaultHttpClient(rejectUnauthorized?: boolean, allowH2?: boolean): HttpClient { + if (rejectUnauthorized === false) { + if (allowH2) { + if (!allowH2AndUnauthorizedHttpClient) { + allowH2AndUnauthorizedHttpClient = new HttpClient({ + allowH2, + connect: { + rejectUnauthorized, + }, + }); + } + return allowH2AndUnauthorizedHttpClient; + } + + if (!allowUnauthorizedHttpClient) { + allowUnauthorizedHttpClient = new HttpClient({ + connect: { + rejectUnauthorized, + }, + }); + } + return allowUnauthorizedHttpClient; + } + + if (allowH2) { + if (!allowH2HttpClient) { + allowH2HttpClient = new HttpClient({ + allowH2, + }); + } + return allowH2HttpClient; + } + if (!httpClient) { httpClient = new HttpClient(); } return httpClient; } -export async function request(url: RequestURL, options?: RequestOptions) { +export interface UrllibRequestOptions extends RequestOptions { + /** + * If `true`, the server certificate is verified against the list of supplied CAs. + * An 'error' event is emitted if verification fails. + * Default: `true` + */ + rejectUnauthorized?: boolean; + /** Allow to use HTTP2 first. Default is `false` */ + allowH2?: boolean; +} + +export async function request(url: RequestURL, options?: UrllibRequestOptions) { if (options?.socketPath) { let domainSocketHttpclient = domainSocketHttpClients.get(options.socketPath); if (!domainSocketHttpclient) { @@ -24,7 +70,7 @@ export async function request(url: RequestURL, options?: RequestOptions return await domainSocketHttpclient.request(url, options); } - return await getDefaultHttpClient().request(url, options); + return await getDefaultHttpClient(options?.rejectUnauthorized, options?.allowH2).request(url, options); } // export curl method is keep compatible with urllib.curl() @@ -32,7 +78,7 @@ export async function request(url: RequestURL, options?: RequestOptions // import * as urllib from 'urllib'; // urllib.curl(url); // ``` -export async function curl(url: RequestURL, options?: RequestOptions) { +export async function curl(url: RequestURL, options?: UrllibRequestOptions) { return await request(url, options); } diff --git a/test/urllib.options.allowH2.test.ts b/test/urllib.options.allowH2.test.ts new file mode 100644 index 00000000..1dd78ddb --- /dev/null +++ b/test/urllib.options.allowH2.test.ts @@ -0,0 +1,19 @@ +import { strict as assert } from 'node:assert'; +import { describe, it } from 'vitest'; +import urllib from '../src/index.js'; + +describe('urllib.options.allowH2.test.ts', () => { + it('should 200 on options.allowH2 = true', async () => { + let response = await urllib.request('https://registry.npmmirror.com', { + allowH2: true, + dataType: 'json', + }); + assert.equal(response.status, 200); + + response = await urllib.curl('https://registry.npmmirror.com', { + allowH2: true, + dataType: 'json', + }); + assert.equal(response.status, 200); + }); +}); diff --git a/test/urllib.options.rejectUnauthorized-false.test.ts b/test/urllib.options.rejectUnauthorized-false.test.ts new file mode 100644 index 00000000..e5512403 --- /dev/null +++ b/test/urllib.options.rejectUnauthorized-false.test.ts @@ -0,0 +1,71 @@ +import { strict as assert } from 'node:assert'; +import { once } from 'node:events'; +import { createSecureServer } from 'node:http2'; +import { describe, it, beforeAll, afterAll } from 'vitest'; +import selfsigned from 'selfsigned'; +import urllib, { HttpClient } from '../src/index.js'; +import { startServer } from './fixtures/server.js'; + +describe('urllib.options.rejectUnauthorized-false.test.ts', () => { + let close: any; + let _url: string; + beforeAll(async () => { + const { closeServer, url } = await startServer({ https: true }); + close = closeServer; + _url = url; + }); + + afterAll(async () => { + await close(); + }); + + it('should 200 on options.rejectUnauthorized = false', async () => { + const response = await urllib.request(_url, { + rejectUnauthorized: false, + dataType: 'json', + }); + assert.equal(response.status, 200); + assert.equal(response.data.method, 'GET'); + }); + + it('should 200 with H2 on options.rejectUnauthorized = false', async () => { + const pem = selfsigned.generate(); + const server = createSecureServer({ + key: pem.private, + cert: pem.cert, + }); + + server.on('stream', (stream, headers) => { + assert.equal(headers[':method'], 'GET'); + stream.respond({ + 'content-type': 'text/plain; charset=utf-8', + 'x-custom-h2': 'hello', + ':status': 200, + }); + stream.end('hello h2!'); + }); + + server.listen(0); + await once(server, 'listening'); + + const httpClient = new HttpClient({ + allowH2: true, + connect: { + rejectUnauthorized: false, + }, + }); + + const url = `https://localhost:${server.address()!.port}`; + const response1 = await httpClient.request(url, {}); + assert.equal(response1.status, 200); + assert.equal(response1.data.toString(), 'hello h2!'); + + const response2 = await urllib.request(url, { + rejectUnauthorized: false, + allowH2: true, + dataType: 'text', + }); + assert.equal(response2.status, 200); + assert.equal(response2.data, 'hello h2!'); + }); +}); From d5698923099528eb08dd6a3e5c8e9aa80ff4fd9e Mon Sep 17 00:00:00 2001 From: fengmk2 Date: Thu, 13 Feb 2025 22:54:45 +0800 Subject: [PATCH 2/2] f --- .github/workflows/nodejs-3.x.yml | 3 ++- test/options.compressed.test.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nodejs-3.x.yml b/.github/workflows/nodejs-3.x.yml index 9628df73..387effc5 100644 --- a/.github/workflows/nodejs-3.x.yml +++ b/.github/workflows/nodejs-3.x.yml @@ -12,6 +12,7 @@ jobs: uses: node-modules/github-actions/.github/workflows/node-test.yml@master with: os: 'ubuntu-latest, macos-latest, windows-latest' - version: '14.19.3, 14, 16, 18, 20, 22' + # skip 16: TypeError: crypto$2.getRandomValues is not a function + version: '14.19.3, 14, 18, 20, 22' secrets: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/test/options.compressed.test.ts b/test/options.compressed.test.ts index 1a17ad3b..0c471936 100644 --- a/test/options.compressed.test.ts +++ b/test/options.compressed.test.ts @@ -193,7 +193,7 @@ describe('options.compressed.test.ts', () => { // console.error(err); assert.equal(err.name, 'UnzipError'); assert.equal(err.message, 'Decompression failed'); - if (nodeMajorVersion() >= 20) { + if (nodeMajorVersion() >= 18) { assert.equal(err.code, 'ERR__ERROR_FORMAT_PADDING_1'); } else { assert.equal(err.code, 'ERR_PADDING_1');