From f3077f8d28e916adf4eb51349cdc89ec71464ec3 Mon Sep 17 00:00:00 2001 From: shangxin Date: Fri, 17 Apr 2026 00:32:21 +0800 Subject: [PATCH] fix: avoid DNS lookup after proxied release CONNECT --- cli/release-staging/http.js | 176 +++++++++++++ cli/release-staging/index.js | 125 +-------- cli/release-staging/package.json | 1 + cli/release/http.js | 176 +++++++++++++ cli/release/index.js | 125 +-------- cli/release/package.json | 1 + .../__tests__/release/proxy-http-get.test.ts | 237 ++++++++++++++++++ freebuff/cli/release/http.js | 176 +++++++++++++ freebuff/cli/release/index.js | 125 +-------- freebuff/cli/release/package.json | 1 + 10 files changed, 786 insertions(+), 357 deletions(-) create mode 100644 cli/release-staging/http.js create mode 100644 cli/release/http.js create mode 100644 cli/src/__tests__/release/proxy-http-get.test.ts create mode 100644 freebuff/cli/release/http.js diff --git a/cli/release-staging/http.js b/cli/release-staging/http.js new file mode 100644 index 0000000000..3419e80ca3 --- /dev/null +++ b/cli/release-staging/http.js @@ -0,0 +1,176 @@ +const http = require('http') +const https = require('https') +const tls = require('tls') + +function createReleaseHttpClient({ + env = process.env, + userAgent, + requestTimeout, + httpModule = http, + httpsModule = https, + tlsModule = tls, +}) { + function getProxyUrl() { + return ( + env.HTTPS_PROXY || + env.https_proxy || + env.HTTP_PROXY || + env.http_proxy || + null + ) + } + + function shouldBypassProxy(hostname) { + const noProxy = env.NO_PROXY || env.no_proxy || '' + if (!noProxy) return false + + const domains = noProxy + .split(',') + .map((domain) => domain.trim().toLowerCase().replace(/:\d+$/, '')) + const host = hostname.toLowerCase() + + return domains.some((domain) => { + if (domain === '*') return true + if (domain.startsWith('.')) { + return host.endsWith(domain) || host === domain.slice(1) + } + return host === domain || host.endsWith(`.${domain}`) + }) + } + + function connectThroughProxy(proxyUrl, targetHost, targetPort) { + return new Promise((resolve, reject) => { + const proxy = new URL(proxyUrl) + const isHttpsProxy = proxy.protocol === 'https:' + const connectOptions = { + hostname: proxy.hostname, + port: proxy.port || (isHttpsProxy ? 443 : 80), + method: 'CONNECT', + path: `${targetHost}:${targetPort}`, + headers: { + Host: `${targetHost}:${targetPort}`, + }, + } + + if (proxy.username || proxy.password) { + const auth = Buffer.from( + `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent( + proxy.password || '', + )}`, + ).toString('base64') + connectOptions.headers['Proxy-Authorization'] = `Basic ${auth}` + } + + const transport = isHttpsProxy ? httpsModule : httpModule + const req = transport.request(connectOptions) + + req.on('connect', (res, socket) => { + if (res.statusCode === 200) { + resolve(socket) + return + } + + socket.destroy() + reject(new Error(`Proxy CONNECT failed with status ${res.statusCode}`)) + }) + + req.on('error', (error) => { + reject(new Error(`Proxy connection failed: ${error.message}`)) + }) + + req.setTimeout(requestTimeout, () => { + req.destroy() + reject(new Error('Proxy connection timeout.')) + }) + + req.end() + }) + } + + async function buildRequestOptions(url, options = {}) { + const parsedUrl = new URL(url) + const reqOptions = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || 443, + path: parsedUrl.pathname + parsedUrl.search, + headers: { + 'User-Agent': userAgent, + ...options.headers, + }, + } + + const proxyUrl = getProxyUrl() + if (!proxyUrl || shouldBypassProxy(parsedUrl.hostname)) { + return reqOptions + } + + const tunnelSocket = await connectThroughProxy( + proxyUrl, + parsedUrl.hostname, + parsedUrl.port || 443, + ) + + class TunnelAgent extends httpsModule.Agent { + createConnection(_options, callback) { + const secureSocket = tlsModule.connect({ + socket: tunnelSocket, + servername: parsedUrl.hostname, + }) + + if (typeof callback === 'function') { + if (typeof secureSocket.once === 'function') { + let settled = false + const finish = (error) => { + if (settled) return + settled = true + callback(error || null, error ? undefined : secureSocket) + } + + secureSocket.once('secureConnect', () => finish(null)) + secureSocket.once('error', (error) => finish(error)) + } else { + callback(null, secureSocket) + } + } + + return secureSocket + } + } + + reqOptions.agent = new TunnelAgent({ keepAlive: false }) + return reqOptions + } + + async function httpGet(url, options = {}) { + const reqOptions = await buildRequestOptions(url, options) + + return new Promise((resolve, reject) => { + const req = httpsModule.get(reqOptions, (res) => { + if (res.statusCode === 301 || res.statusCode === 302) { + res.resume() + httpGet(new URL(res.headers.location, url).href, options) + .then(resolve) + .catch(reject) + return + } + + resolve(res) + }) + + req.on('error', reject) + req.setTimeout(options.timeout || requestTimeout, () => { + req.destroy() + reject(new Error('Request timeout.')) + }) + }) + } + + return { + getProxyUrl, + httpGet, + } +} + +module.exports = { + createReleaseHttpClient, +} diff --git a/cli/release-staging/index.js b/cli/release-staging/index.js index 14f229fb4c..083e8879a9 100644 --- a/cli/release-staging/index.js +++ b/cli/release-staging/index.js @@ -6,10 +6,10 @@ const http = require('http') const https = require('https') const os = require('os') const path = require('path') -const tls = require('tls') const zlib = require('zlib') const tar = require('tar') +const { createReleaseHttpClient } = require('./http') const packageName = 'codecane' @@ -66,6 +66,11 @@ function createConfig(packageName) { } const CONFIG = createConfig(packageName) +const { getProxyUrl, httpGet } = createReleaseHttpClient({ + env: process.env, + userAgent: CONFIG.userAgent, + requestTimeout: CONFIG.requestTimeout, +}) function getPostHogConfig() { const apiKey = @@ -131,76 +136,6 @@ function trackUpdateFailed(errorMessage, version, context = {}) { } } -function getProxyUrl() { - return ( - process.env.HTTPS_PROXY || - process.env.https_proxy || - process.env.HTTP_PROXY || - process.env.http_proxy || - null - ) -} - -function shouldBypassProxy(hostname) { - const noProxy = process.env.NO_PROXY || process.env.no_proxy || '' - if (!noProxy) return false - const domains = noProxy.split(',').map((d) => d.trim().toLowerCase().replace(/:\d+$/, '')) - const host = hostname.toLowerCase() - return domains.some((d) => { - if (d === '*') return true - if (d.startsWith('.')) return host.endsWith(d) || host === d.slice(1) - return host === d || host.endsWith('.' + d) - }) -} - -function connectThroughProxy(proxyUrl, targetHost, targetPort) { - return new Promise((resolve, reject) => { - const proxy = new URL(proxyUrl) - const isHttpsProxy = proxy.protocol === 'https:' - const connectOptions = { - hostname: proxy.hostname, - port: proxy.port || (isHttpsProxy ? 443 : 80), - method: 'CONNECT', - path: `${targetHost}:${targetPort}`, - headers: { - Host: `${targetHost}:${targetPort}`, - }, - } - - if (proxy.username || proxy.password) { - const auth = Buffer.from( - `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent(proxy.password || '')}`, - ).toString('base64') - connectOptions.headers['Proxy-Authorization'] = `Basic ${auth}` - } - - const transport = isHttpsProxy ? https : http - const req = transport.request(connectOptions) - - req.on('connect', (res, socket) => { - if (res.statusCode === 200) { - resolve(socket) - } else { - socket.destroy() - reject( - new Error(`Proxy CONNECT failed with status ${res.statusCode}`), - ) - } - }) - - req.on('error', (err) => { - reject(new Error(`Proxy connection failed: ${err.message}`)) - }) - - req.setTimeout(CONFIG.requestTimeout, () => { - req.destroy() - reject(new Error('Proxy connection timeout.')) - }) - - req.end() - }) -} - const PLATFORM_TARGETS = { 'linux-x64': `${packageName}-linux-x64.tar.gz`, 'linux-arm64': `${packageName}-linux-arm64.tar.gz`, @@ -225,54 +160,6 @@ const term = { }, } -async function httpGet(url, options = {}) { - const parsedUrl = new URL(url) - const proxyUrl = getProxyUrl() - - const reqOptions = { - hostname: parsedUrl.hostname, - path: parsedUrl.pathname + parsedUrl.search, - headers: { - 'User-Agent': CONFIG.userAgent, - ...options.headers, - }, - } - - if (proxyUrl && !shouldBypassProxy(parsedUrl.hostname)) { - const tunnelSocket = await connectThroughProxy( - proxyUrl, - parsedUrl.hostname, - parsedUrl.port || 443, - ) - reqOptions.agent = false - reqOptions.createConnection = () => - tls.connect({ - socket: tunnelSocket, - servername: parsedUrl.hostname, - }) - } - - return new Promise((resolve, reject) => { - const req = https.get(reqOptions, (res) => { - if (res.statusCode === 302 || res.statusCode === 301) { - res.resume() - return httpGet(new URL(res.headers.location, url).href, options) - .then(resolve) - .catch(reject) - } - resolve(res) - }) - - req.on('error', reject) - - const timeout = options.timeout || CONFIG.requestTimeout - req.setTimeout(timeout, () => { - req.destroy() - reject(new Error('Request timeout.')) - }) - }) -} - async function getLatestVersion() { try { const res = await httpGet( diff --git a/cli/release-staging/package.json b/cli/release-staging/package.json index 23ae8cac37..f84bff8721 100644 --- a/cli/release-staging/package.json +++ b/cli/release-staging/package.json @@ -12,6 +12,7 @@ }, "files": [ "index.js", + "http.js", "postinstall.js", "README.md" ], diff --git a/cli/release/http.js b/cli/release/http.js new file mode 100644 index 0000000000..3419e80ca3 --- /dev/null +++ b/cli/release/http.js @@ -0,0 +1,176 @@ +const http = require('http') +const https = require('https') +const tls = require('tls') + +function createReleaseHttpClient({ + env = process.env, + userAgent, + requestTimeout, + httpModule = http, + httpsModule = https, + tlsModule = tls, +}) { + function getProxyUrl() { + return ( + env.HTTPS_PROXY || + env.https_proxy || + env.HTTP_PROXY || + env.http_proxy || + null + ) + } + + function shouldBypassProxy(hostname) { + const noProxy = env.NO_PROXY || env.no_proxy || '' + if (!noProxy) return false + + const domains = noProxy + .split(',') + .map((domain) => domain.trim().toLowerCase().replace(/:\d+$/, '')) + const host = hostname.toLowerCase() + + return domains.some((domain) => { + if (domain === '*') return true + if (domain.startsWith('.')) { + return host.endsWith(domain) || host === domain.slice(1) + } + return host === domain || host.endsWith(`.${domain}`) + }) + } + + function connectThroughProxy(proxyUrl, targetHost, targetPort) { + return new Promise((resolve, reject) => { + const proxy = new URL(proxyUrl) + const isHttpsProxy = proxy.protocol === 'https:' + const connectOptions = { + hostname: proxy.hostname, + port: proxy.port || (isHttpsProxy ? 443 : 80), + method: 'CONNECT', + path: `${targetHost}:${targetPort}`, + headers: { + Host: `${targetHost}:${targetPort}`, + }, + } + + if (proxy.username || proxy.password) { + const auth = Buffer.from( + `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent( + proxy.password || '', + )}`, + ).toString('base64') + connectOptions.headers['Proxy-Authorization'] = `Basic ${auth}` + } + + const transport = isHttpsProxy ? httpsModule : httpModule + const req = transport.request(connectOptions) + + req.on('connect', (res, socket) => { + if (res.statusCode === 200) { + resolve(socket) + return + } + + socket.destroy() + reject(new Error(`Proxy CONNECT failed with status ${res.statusCode}`)) + }) + + req.on('error', (error) => { + reject(new Error(`Proxy connection failed: ${error.message}`)) + }) + + req.setTimeout(requestTimeout, () => { + req.destroy() + reject(new Error('Proxy connection timeout.')) + }) + + req.end() + }) + } + + async function buildRequestOptions(url, options = {}) { + const parsedUrl = new URL(url) + const reqOptions = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || 443, + path: parsedUrl.pathname + parsedUrl.search, + headers: { + 'User-Agent': userAgent, + ...options.headers, + }, + } + + const proxyUrl = getProxyUrl() + if (!proxyUrl || shouldBypassProxy(parsedUrl.hostname)) { + return reqOptions + } + + const tunnelSocket = await connectThroughProxy( + proxyUrl, + parsedUrl.hostname, + parsedUrl.port || 443, + ) + + class TunnelAgent extends httpsModule.Agent { + createConnection(_options, callback) { + const secureSocket = tlsModule.connect({ + socket: tunnelSocket, + servername: parsedUrl.hostname, + }) + + if (typeof callback === 'function') { + if (typeof secureSocket.once === 'function') { + let settled = false + const finish = (error) => { + if (settled) return + settled = true + callback(error || null, error ? undefined : secureSocket) + } + + secureSocket.once('secureConnect', () => finish(null)) + secureSocket.once('error', (error) => finish(error)) + } else { + callback(null, secureSocket) + } + } + + return secureSocket + } + } + + reqOptions.agent = new TunnelAgent({ keepAlive: false }) + return reqOptions + } + + async function httpGet(url, options = {}) { + const reqOptions = await buildRequestOptions(url, options) + + return new Promise((resolve, reject) => { + const req = httpsModule.get(reqOptions, (res) => { + if (res.statusCode === 301 || res.statusCode === 302) { + res.resume() + httpGet(new URL(res.headers.location, url).href, options) + .then(resolve) + .catch(reject) + return + } + + resolve(res) + }) + + req.on('error', reject) + req.setTimeout(options.timeout || requestTimeout, () => { + req.destroy() + reject(new Error('Request timeout.')) + }) + }) + } + + return { + getProxyUrl, + httpGet, + } +} + +module.exports = { + createReleaseHttpClient, +} diff --git a/cli/release/index.js b/cli/release/index.js index 3d22e65739..85c60ff392 100644 --- a/cli/release/index.js +++ b/cli/release/index.js @@ -6,10 +6,10 @@ const http = require('http') const https = require('https') const os = require('os') const path = require('path') -const tls = require('tls') const zlib = require('zlib') const tar = require('tar') +const { createReleaseHttpClient } = require('./http') const packageName = 'codebuff' @@ -66,6 +66,11 @@ function createConfig(packageName) { } const CONFIG = createConfig(packageName) +const { getProxyUrl, httpGet } = createReleaseHttpClient({ + env: process.env, + userAgent: CONFIG.userAgent, + requestTimeout: CONFIG.requestTimeout, +}) function getPostHogConfig() { const apiKey = @@ -130,76 +135,6 @@ function trackUpdateFailed(errorMessage, version, context = {}) { } } -function getProxyUrl() { - return ( - process.env.HTTPS_PROXY || - process.env.https_proxy || - process.env.HTTP_PROXY || - process.env.http_proxy || - null - ) -} - -function shouldBypassProxy(hostname) { - const noProxy = process.env.NO_PROXY || process.env.no_proxy || '' - if (!noProxy) return false - const domains = noProxy.split(',').map((d) => d.trim().toLowerCase().replace(/:\d+$/, '')) - const host = hostname.toLowerCase() - return domains.some((d) => { - if (d === '*') return true - if (d.startsWith('.')) return host.endsWith(d) || host === d.slice(1) - return host === d || host.endsWith('.' + d) - }) -} - -function connectThroughProxy(proxyUrl, targetHost, targetPort) { - return new Promise((resolve, reject) => { - const proxy = new URL(proxyUrl) - const isHttpsProxy = proxy.protocol === 'https:' - const connectOptions = { - hostname: proxy.hostname, - port: proxy.port || (isHttpsProxy ? 443 : 80), - method: 'CONNECT', - path: `${targetHost}:${targetPort}`, - headers: { - Host: `${targetHost}:${targetPort}`, - }, - } - - if (proxy.username || proxy.password) { - const auth = Buffer.from( - `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent(proxy.password || '')}`, - ).toString('base64') - connectOptions.headers['Proxy-Authorization'] = `Basic ${auth}` - } - - const transport = isHttpsProxy ? https : http - const req = transport.request(connectOptions) - - req.on('connect', (res, socket) => { - if (res.statusCode === 200) { - resolve(socket) - } else { - socket.destroy() - reject( - new Error(`Proxy CONNECT failed with status ${res.statusCode}`), - ) - } - }) - - req.on('error', (err) => { - reject(new Error(`Proxy connection failed: ${err.message}`)) - }) - - req.setTimeout(CONFIG.requestTimeout, () => { - req.destroy() - reject(new Error('Proxy connection timeout.')) - }) - - req.end() - }) -} - const PLATFORM_TARGETS = { 'linux-x64': `${packageName}-linux-x64.tar.gz`, 'linux-arm64': `${packageName}-linux-arm64.tar.gz`, @@ -224,54 +159,6 @@ const term = { }, } -async function httpGet(url, options = {}) { - const parsedUrl = new URL(url) - const proxyUrl = getProxyUrl() - - const reqOptions = { - hostname: parsedUrl.hostname, - path: parsedUrl.pathname + parsedUrl.search, - headers: { - 'User-Agent': CONFIG.userAgent, - ...options.headers, - }, - } - - if (proxyUrl && !shouldBypassProxy(parsedUrl.hostname)) { - const tunnelSocket = await connectThroughProxy( - proxyUrl, - parsedUrl.hostname, - parsedUrl.port || 443, - ) - reqOptions.agent = false - reqOptions.createConnection = () => - tls.connect({ - socket: tunnelSocket, - servername: parsedUrl.hostname, - }) - } - - return new Promise((resolve, reject) => { - const req = https.get(reqOptions, (res) => { - if (res.statusCode === 302 || res.statusCode === 301) { - res.resume() - return httpGet(new URL(res.headers.location, url).href, options) - .then(resolve) - .catch(reject) - } - resolve(res) - }) - - req.on('error', reject) - - const timeout = options.timeout || CONFIG.requestTimeout - req.setTimeout(timeout, () => { - req.destroy() - reject(new Error('Request timeout.')) - }) - }) -} - async function getLatestVersion() { try { const res = await httpGet( diff --git a/cli/release/package.json b/cli/release/package.json index 1eb51b176f..d7fef3c18e 100644 --- a/cli/release/package.json +++ b/cli/release/package.json @@ -13,6 +13,7 @@ }, "files": [ "index.js", + "http.js", "postinstall.js", "README.md" ], diff --git a/cli/src/__tests__/release/proxy-http-get.test.ts b/cli/src/__tests__/release/proxy-http-get.test.ts new file mode 100644 index 0000000000..a0addd586a --- /dev/null +++ b/cli/src/__tests__/release/proxy-http-get.test.ts @@ -0,0 +1,237 @@ +import { describe, expect, test } from 'bun:test' +import { EventEmitter } from 'node:events' +import { createRequire } from 'node:module' +import { fileURLToPath } from 'node:url' +import { Readable } from 'node:stream' + +const require = createRequire(import.meta.url) + +const helperModules = [ + { + name: 'codebuff release helper', + path: fileURLToPath(new URL('../../../release/http.js', import.meta.url)), + }, + { + name: 'codebuff staging release helper', + path: fileURLToPath( + new URL('../../../release-staging/http.js', import.meta.url), + ), + }, + { + name: 'freebuff release helper', + path: fileURLToPath( + new URL('../../../../freebuff/cli/release/http.js', import.meta.url), + ), + }, +] + +function createResponse(statusCode: number, headers: Record, body = '') { + const response = Readable.from(body.length > 0 ? [body] : []) + return Object.assign(response, { + statusCode, + headers, + }) +} + +function createConnectRequest({ + statusCode = 200, + tunnelSocket, + recorder, +}: { + statusCode?: number + tunnelSocket: object + recorder: { timeoutCalls: number } +}) { + const emitter = new EventEmitter() + + return { + on(event: string, listener: (...args: any[]) => void) { + emitter.on(event, listener) + return this + }, + setTimeout() { + recorder.timeoutCalls += 1 + return this + }, + destroy() {}, + end() { + queueMicrotask(() => { + emitter.emit('connect', { statusCode }, tunnelSocket) + }) + }, + } +} + +for (const helperModule of helperModules) { + describe(helperModule.name, () => { + test('uses a tunnel agent instead of createConnection for proxied HTTPS requests', async () => { + const connectCalls: Array> = [] + const httpsGetCalls: Array> = [] + const tlsConnectCalls: Array> = [] + + const tunnelSocket = { kind: 'tunnel-socket' } + const tlsSocket = { kind: 'tls-socket' } + + const { createReleaseHttpClient } = require(helperModule.path) + + const client = createReleaseHttpClient({ + env: { + HTTPS_PROXY: 'http://proxy.internal:7890', + }, + userAgent: 'release-test-agent', + requestTimeout: 2500, + httpModule: { + request(options: Record) { + connectCalls.push(options) + return createConnectRequest({ + tunnelSocket, + recorder: { timeoutCalls: 0 }, + }) + }, + }, + httpsModule: { + Agent: class FakeAgent { + options: Record + + constructor(options: Record) { + this.options = options + } + }, + get(options: Record, callback: (response: Readable) => void) { + httpsGetCalls.push(options) + options.agent.createConnection(options) + queueMicrotask(() => { + callback(createResponse(200, {}, '{"version":"0.0.33"}')) + }) + return { + on() { + return this + }, + setTimeout() { + return this + }, + destroy() {}, + } + }, + }, + tlsModule: { + connect(options: Record) { + tlsConnectCalls.push(options) + return tlsSocket + }, + }, + }) + + const response = await client.httpGet( + 'https://registry.npmjs.org/freebuff/latest', + ) + response.resume() + + expect(connectCalls).toHaveLength(1) + expect(connectCalls[0]).toMatchObject({ + hostname: 'proxy.internal', + port: '7890', + method: 'CONNECT', + path: 'registry.npmjs.org:443', + headers: { + Host: 'registry.npmjs.org:443', + }, + }) + + expect(httpsGetCalls).toHaveLength(1) + expect(httpsGetCalls[0]?.createConnection).toBeUndefined() + expect(httpsGetCalls[0]?.agent).toBeDefined() + expect(httpsGetCalls[0]).toMatchObject({ + hostname: 'registry.npmjs.org', + path: '/freebuff/latest', + headers: { + 'User-Agent': 'release-test-agent', + }, + }) + + expect(tlsConnectCalls).toEqual([ + { + socket: tunnelSocket, + servername: 'registry.npmjs.org', + }, + ]) + }) + + test('reuses the same proxy strategy across redirects', async () => { + const httpsGetCalls: Array> = [] + + const { createReleaseHttpClient } = require(helperModule.path) + + let callCount = 0 + const client = createReleaseHttpClient({ + env: { + HTTPS_PROXY: 'http://proxy.internal:7890', + }, + userAgent: 'release-test-agent', + requestTimeout: 2500, + httpModule: { + request() { + return createConnectRequest({ + tunnelSocket: { kind: 'tunnel-socket' }, + recorder: { timeoutCalls: 0 }, + }) + }, + }, + httpsModule: { + Agent: class FakeAgent {}, + get(options: Record, callback: (response: Readable) => void) { + httpsGetCalls.push(options) + callCount += 1 + + queueMicrotask(() => { + if (callCount === 1) { + callback( + createResponse(302, { + location: '/redirected', + }), + ) + return + } + + callback(createResponse(200, {}, 'ok')) + }) + + return { + on() { + return this + }, + setTimeout() { + return this + }, + destroy() {}, + } + }, + }, + tlsModule: { + connect() { + return { kind: 'tls-socket' } + }, + }, + }) + + const response = await client.httpGet( + 'https://registry.npmjs.org/freebuff/latest', + ) + response.resume() + + expect(httpsGetCalls).toHaveLength(2) + expect(httpsGetCalls[0]).toMatchObject({ + hostname: 'registry.npmjs.org', + path: '/freebuff/latest', + }) + expect(httpsGetCalls[1]).toMatchObject({ + hostname: 'registry.npmjs.org', + path: '/redirected', + }) + expect(httpsGetCalls.every((call) => call.createConnection === undefined)).toBe( + true, + ) + expect(httpsGetCalls.every((call) => call.agent != null)).toBe(true) + }) + }) +} diff --git a/freebuff/cli/release/http.js b/freebuff/cli/release/http.js new file mode 100644 index 0000000000..3419e80ca3 --- /dev/null +++ b/freebuff/cli/release/http.js @@ -0,0 +1,176 @@ +const http = require('http') +const https = require('https') +const tls = require('tls') + +function createReleaseHttpClient({ + env = process.env, + userAgent, + requestTimeout, + httpModule = http, + httpsModule = https, + tlsModule = tls, +}) { + function getProxyUrl() { + return ( + env.HTTPS_PROXY || + env.https_proxy || + env.HTTP_PROXY || + env.http_proxy || + null + ) + } + + function shouldBypassProxy(hostname) { + const noProxy = env.NO_PROXY || env.no_proxy || '' + if (!noProxy) return false + + const domains = noProxy + .split(',') + .map((domain) => domain.trim().toLowerCase().replace(/:\d+$/, '')) + const host = hostname.toLowerCase() + + return domains.some((domain) => { + if (domain === '*') return true + if (domain.startsWith('.')) { + return host.endsWith(domain) || host === domain.slice(1) + } + return host === domain || host.endsWith(`.${domain}`) + }) + } + + function connectThroughProxy(proxyUrl, targetHost, targetPort) { + return new Promise((resolve, reject) => { + const proxy = new URL(proxyUrl) + const isHttpsProxy = proxy.protocol === 'https:' + const connectOptions = { + hostname: proxy.hostname, + port: proxy.port || (isHttpsProxy ? 443 : 80), + method: 'CONNECT', + path: `${targetHost}:${targetPort}`, + headers: { + Host: `${targetHost}:${targetPort}`, + }, + } + + if (proxy.username || proxy.password) { + const auth = Buffer.from( + `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent( + proxy.password || '', + )}`, + ).toString('base64') + connectOptions.headers['Proxy-Authorization'] = `Basic ${auth}` + } + + const transport = isHttpsProxy ? httpsModule : httpModule + const req = transport.request(connectOptions) + + req.on('connect', (res, socket) => { + if (res.statusCode === 200) { + resolve(socket) + return + } + + socket.destroy() + reject(new Error(`Proxy CONNECT failed with status ${res.statusCode}`)) + }) + + req.on('error', (error) => { + reject(new Error(`Proxy connection failed: ${error.message}`)) + }) + + req.setTimeout(requestTimeout, () => { + req.destroy() + reject(new Error('Proxy connection timeout.')) + }) + + req.end() + }) + } + + async function buildRequestOptions(url, options = {}) { + const parsedUrl = new URL(url) + const reqOptions = { + hostname: parsedUrl.hostname, + port: parsedUrl.port || 443, + path: parsedUrl.pathname + parsedUrl.search, + headers: { + 'User-Agent': userAgent, + ...options.headers, + }, + } + + const proxyUrl = getProxyUrl() + if (!proxyUrl || shouldBypassProxy(parsedUrl.hostname)) { + return reqOptions + } + + const tunnelSocket = await connectThroughProxy( + proxyUrl, + parsedUrl.hostname, + parsedUrl.port || 443, + ) + + class TunnelAgent extends httpsModule.Agent { + createConnection(_options, callback) { + const secureSocket = tlsModule.connect({ + socket: tunnelSocket, + servername: parsedUrl.hostname, + }) + + if (typeof callback === 'function') { + if (typeof secureSocket.once === 'function') { + let settled = false + const finish = (error) => { + if (settled) return + settled = true + callback(error || null, error ? undefined : secureSocket) + } + + secureSocket.once('secureConnect', () => finish(null)) + secureSocket.once('error', (error) => finish(error)) + } else { + callback(null, secureSocket) + } + } + + return secureSocket + } + } + + reqOptions.agent = new TunnelAgent({ keepAlive: false }) + return reqOptions + } + + async function httpGet(url, options = {}) { + const reqOptions = await buildRequestOptions(url, options) + + return new Promise((resolve, reject) => { + const req = httpsModule.get(reqOptions, (res) => { + if (res.statusCode === 301 || res.statusCode === 302) { + res.resume() + httpGet(new URL(res.headers.location, url).href, options) + .then(resolve) + .catch(reject) + return + } + + resolve(res) + }) + + req.on('error', reject) + req.setTimeout(options.timeout || requestTimeout, () => { + req.destroy() + reject(new Error('Request timeout.')) + }) + }) + } + + return { + getProxyUrl, + httpGet, + } +} + +module.exports = { + createReleaseHttpClient, +} diff --git a/freebuff/cli/release/index.js b/freebuff/cli/release/index.js index 56d8539df6..db7fe566a8 100644 --- a/freebuff/cli/release/index.js +++ b/freebuff/cli/release/index.js @@ -6,10 +6,10 @@ const http = require('http') const https = require('https') const os = require('os') const path = require('path') -const tls = require('tls') const zlib = require('zlib') const tar = require('tar') +const { createReleaseHttpClient } = require('./http') const packageName = 'freebuff' @@ -66,6 +66,11 @@ function createConfig(packageName) { } const CONFIG = createConfig(packageName) +const { getProxyUrl, httpGet } = createReleaseHttpClient({ + env: process.env, + userAgent: CONFIG.userAgent, + requestTimeout: CONFIG.requestTimeout, +}) function getPostHogConfig() { const apiKey = @@ -130,76 +135,6 @@ function trackUpdateFailed(errorMessage, version, context = {}) { } } -function getProxyUrl() { - return ( - process.env.HTTPS_PROXY || - process.env.https_proxy || - process.env.HTTP_PROXY || - process.env.http_proxy || - null - ) -} - -function shouldBypassProxy(hostname) { - const noProxy = process.env.NO_PROXY || process.env.no_proxy || '' - if (!noProxy) return false - const domains = noProxy.split(',').map((d) => d.trim().toLowerCase().replace(/:\d+$/, '')) - const host = hostname.toLowerCase() - return domains.some((d) => { - if (d === '*') return true - if (d.startsWith('.')) return host.endsWith(d) || host === d.slice(1) - return host === d || host.endsWith('.' + d) - }) -} - -function connectThroughProxy(proxyUrl, targetHost, targetPort) { - return new Promise((resolve, reject) => { - const proxy = new URL(proxyUrl) - const isHttpsProxy = proxy.protocol === 'https:' - const connectOptions = { - hostname: proxy.hostname, - port: proxy.port || (isHttpsProxy ? 443 : 80), - method: 'CONNECT', - path: `${targetHost}:${targetPort}`, - headers: { - Host: `${targetHost}:${targetPort}`, - }, - } - - if (proxy.username || proxy.password) { - const auth = Buffer.from( - `${decodeURIComponent(proxy.username || '')}:${decodeURIComponent(proxy.password || '')}`, - ).toString('base64') - connectOptions.headers['Proxy-Authorization'] = `Basic ${auth}` - } - - const transport = isHttpsProxy ? https : http - const req = transport.request(connectOptions) - - req.on('connect', (res, socket) => { - if (res.statusCode === 200) { - resolve(socket) - } else { - socket.destroy() - reject( - new Error(`Proxy CONNECT failed with status ${res.statusCode}`), - ) - } - }) - - req.on('error', (err) => { - reject(new Error(`Proxy connection failed: ${err.message}`)) - }) - - req.setTimeout(CONFIG.requestTimeout, () => { - req.destroy() - reject(new Error('Proxy connection timeout.')) - }) - - req.end() - }) -} - const PLATFORM_TARGETS = { 'linux-x64': `${packageName}-linux-x64.tar.gz`, 'linux-arm64': `${packageName}-linux-arm64.tar.gz`, @@ -224,54 +159,6 @@ const term = { }, } -async function httpGet(url, options = {}) { - const parsedUrl = new URL(url) - const proxyUrl = getProxyUrl() - - const reqOptions = { - hostname: parsedUrl.hostname, - path: parsedUrl.pathname + parsedUrl.search, - headers: { - 'User-Agent': CONFIG.userAgent, - ...options.headers, - }, - } - - if (proxyUrl && !shouldBypassProxy(parsedUrl.hostname)) { - const tunnelSocket = await connectThroughProxy( - proxyUrl, - parsedUrl.hostname, - parsedUrl.port || 443, - ) - reqOptions.agent = false - reqOptions.createConnection = () => - tls.connect({ - socket: tunnelSocket, - servername: parsedUrl.hostname, - }) - } - - return new Promise((resolve, reject) => { - const req = https.get(reqOptions, (res) => { - if (res.statusCode === 302 || res.statusCode === 301) { - res.resume() - return httpGet(new URL(res.headers.location, url).href, options) - .then(resolve) - .catch(reject) - } - resolve(res) - }) - - req.on('error', reject) - - const timeout = options.timeout || CONFIG.requestTimeout - req.setTimeout(timeout, () => { - req.destroy() - reject(new Error('Request timeout.')) - }) - }) -} - async function getLatestVersion() { try { const res = await httpGet( diff --git a/freebuff/cli/release/package.json b/freebuff/cli/release/package.json index dc00bf86cd..7e0f8ea050 100644 --- a/freebuff/cli/release/package.json +++ b/freebuff/cli/release/package.json @@ -12,6 +12,7 @@ }, "files": [ "index.js", + "http.js", "postinstall.js", "README.md" ],