From f276d0fc305bff750206315eeb47706f06957d69 Mon Sep 17 00:00:00 2001 From: kuldeepsinh19 Date: Tue, 3 Nov 2026 23:22:57 +0530 Subject: [PATCH] fix: send all requests through the configured HTTP proxy `--http-proxy` (which defaults to `HTTP_PROXY`/`HTTPS_PROXY`) was only passed to the CLI's Netlify API client. Every other request bypassed the proxy and failed on networks that require one: - Node.js's built-in `fetch()` ignores both the `agent` option and the proxy environment variables. It's used by `@netlify/config` to fetch the site's extensions and by the CLI's own `fetch()` calls. - `http`/`https` requests that don't set an agent, such as those made by the Netlify API client that `@netlify/config` creates to fetch the site info, use the default global agent. When a proxy is configured, also install an undici `EnvHttpProxyAgent` as the global dispatcher and a proxy agent as `https.globalAgent`. Both use the same proxy URL and CA certificate, respect `NO_PROXY` (with the same matching rules), and never proxy requests to the local machine. Fixes #3115 --- package-lock.json | 2 +- package.json | 1 + src/lib/http-agent.ts | 102 ++++++++++++++++++++++- tests/unit/lib/http-agent.test.ts | 134 +++++++++++++++++++++++++++++- 4 files changed, 234 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f739a04cea..b3eb4a34808 100644 --- a/package-lock.json +++ b/package-lock.json @@ -100,6 +100,7 @@ "toml": "^5.0.0", "tomlify-j0.4": "^3.0.0", "ulid": "^3.0.1", + "undici": "^7.30.1", "update-notifier": "^7.3.1", "write-file-atomic": "^5.0.1", "ws": "^8.19.0", @@ -19607,7 +19608,6 @@ "version": "7.30.1", "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", - "dev": true, "license": "MIT", "engines": { "node": ">=20.18.1" diff --git a/package.json b/package.json index 009c8ab87cc..6675c3fc2ef 100644 --- a/package.json +++ b/package.json @@ -142,6 +142,7 @@ "toml": "^5.0.0", "tomlify-j0.4": "^3.0.0", "ulid": "^3.0.1", + "undici": "^7.30.1", "update-notifier": "^7.3.1", "write-file-atomic": "^5.0.1", "ws": "^8.19.0", diff --git a/src/lib/http-agent.ts b/src/lib/http-agent.ts index d9f44df9b65..224a6a0ff53 100644 --- a/src/lib/http-agent.ts +++ b/src/lib/http-agent.ts @@ -1,6 +1,11 @@ import { readFile } from 'fs/promises' +import type { ClientRequest } from 'http' +import https from 'https' +import net from 'net' +import tls from 'tls' import { HttpsProxyAgent } from 'https-proxy-agent' +import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici' import { NETLIFYDEVERR, NETLIFYDEVWARN, exit, log } from '../utils/command-helpers.js' import { waitPort } from './wait-port.js' @@ -31,6 +36,92 @@ const DEFAULT_HTTPS_PORT = 443 // 50 seconds const AGENT_PORT_TIMEOUT = 50_000 +// Requests to the local machine (e.g. the framework dev server or the functions server in `netlify dev`) +// should never go through the proxy +const LOOPBACK_HOSTS = ['localhost', '127.0.0.1', '[::1]'] + +const getNoProxy = (): string => + [process.env.NO_PROXY ?? process.env.no_proxy, ...LOOPBACK_HOSTS].filter(Boolean).join(',') + +const normalizeHostname = (hostname: string): string => hostname.toLowerCase().replace(/^\[(.*)\]$/, '$1') + +/** + * Whether a request to `hostname` should go through the proxy, using the same `NO_PROXY` rules as undici's + * `EnvHttpProxyAgent`, so `fetch()` and `http`/`https` requests behave the same. + */ +const shouldProxy = (hostname: string, port: number, noProxy: string): boolean => { + const host = normalizeHostname(hostname) + + return noProxy + .split(/[,\s]/) + .filter(Boolean) + .every((entry) => { + if (entry === '*') { + return false + } + + const parsed = /^(.+):(\d+)$/.exec(entry) + const entryHost = normalizeHostname((parsed ? parsed[1] : entry).replace(/^\*?\./, '')) + if (parsed && Number.parseInt(parsed[2], 10) !== port) { + return true + } + + return host !== entryHost && !host.endsWith(`.${entryHost}`) + }) +} + +type ConnectOpts = Parameters['connect']>[1] + +/** + * Agent for `https` requests that don't set their own agent, e.g. the Netlify API client that + * `@netlify/config` creates to retrieve the site info. Connects directly to hosts in `NO_PROXY`. + */ +class HttpsProxyAgentWithNoProxy extends HttpsProxyAgent { + constructor( + proxy: string, + private readonly noProxy: string, + private readonly ca: Buffer | undefined, + ) { + super(proxy) + // Used when connecting to an `https:` proxy + if (ca) { + this.connectOpts.ca = ca + } + } + + async connect(req: ClientRequest, opts: ConnectOpts): Promise { + if (!shouldProxy(opts.host ?? 'localhost', opts.port, this.noProxy)) { + return opts.secureEndpoint + ? tls.connect({ ...opts, servername: net.isIP(opts.host ?? '') ? undefined : opts.host }) + : net.connect(opts) + } + + // Used for the TLS connection to the target host through the proxy + return super.connect(req, opts.secureEndpoint && this.ca ? { ...opts, ca: this.ca } : opts) + } +} + +/** + * Node.js's built-in `fetch()` ignores both `HTTP_PROXY` and the `agent` option, and so does any `http`/`https` + * request that doesn't set an agent. The Netlify API client gets an agent, but other requests, e.g. by + * `@netlify/config` or the CLI's own `fetch()` calls, need the proxy to be set globally. + */ +const getGlobalProxyAgents = (httpProxy: string, certificate: Buffer | undefined) => { + const noProxy = getNoProxy() + const tlsOptions = certificate ? { ca: certificate } : undefined + + return { + dispatcher: new EnvHttpProxyAgent({ + httpProxy, + httpsProxy: httpProxy, + noProxy, + requestTls: tlsOptions, + proxyTls: tlsOptions, + }), + httpsGlobalAgent: new HttpsProxyAgentWithNoProxy(httpProxy, noProxy, certificate), + } +} + export const tryGetAgent = async ({ certificateFile, httpProxy, @@ -45,6 +136,8 @@ export const tryGetAgent = async ({ } | { agent: HttpsProxyAgentWithCA + dispatcher: EnvHttpProxyAgent + httpsGlobalAgent: https.Agent response: unknown } > => { @@ -103,14 +196,15 @@ export const tryGetAgent = async ({ } const agent = new HttpsProxyAgentWithCA(opts) - response = { ...response, agent } + response = { ...response, agent, ...getGlobalProxyAgents(httpProxy, certificate) } return response } // @ts-expect-error TS(7031) FIXME: Binding element 'certificateFile' implicitly has a... Remove this comment to see the full error message export const getAgent = async ({ certificateFile, httpProxy }) => { + const result = await tryGetAgent({ httpProxy, certificateFile }) // @ts-expect-error TS(2339) FIXME: Property 'agent' does not exist on type '{ error?:... Remove this comment to see the full error message - const { agent, error, message, warning } = await tryGetAgent({ httpProxy, certificateFile }) + const { agent, error, message, warning } = result if (error) { log(NETLIFYDEVERR, error, message || '') exit(1) @@ -118,5 +212,9 @@ export const getAgent = async ({ certificateFile, httpProxy }) => { if (warning) { log(NETLIFYDEVWARN, warning, message || '') } + if ('dispatcher' in result) { + setGlobalDispatcher(result.dispatcher) + https.globalAgent = result.httpsGlobalAgent + } return agent } diff --git a/tests/unit/lib/http-agent.test.ts b/tests/unit/lib/http-agent.test.ts index fd925032672..456029f892e 100644 --- a/tests/unit/lib/http-agent.test.ts +++ b/tests/unit/lib/http-agent.test.ts @@ -1,11 +1,68 @@ import net from 'node:net' import http from 'node:http' +import https from 'node:https' import ProxyServer from 'http-proxy' import { HttpsProxyAgent } from 'https-proxy-agent' -import { describe, expect, test } from 'vitest' +import nodeFetch from 'node-fetch' +import { getGlobalDispatcher, setGlobalDispatcher } from 'undici' +import { afterEach, beforeEach, describe, expect, test } from 'vitest' -import { tryGetAgent } from '../../../src/lib/http-agent.js' +import { getAgent, tryGetAgent } from '../../../src/lib/http-agent.js' + +const listen = async (server: net.Server): Promise => { + await new Promise((resolve) => { + server.listen({ port: 0, hostname: 'localhost' }, () => { + resolve() + }) + }) + return (server.address() as net.AddressInfo).port +} + +const close = (server: net.Server) => + new Promise((resolve) => { + server.close(() => { + resolve() + }) + }) + +/** + * Starts a server that responds with `target`, and a proxy that records the requests it receives + * and sends them to that server, no matter which host they are for. + */ +const startProxy = async () => { + const target = http.createServer((_, res) => { + res.end('target') + }) + const targetPort = await listen(target) + + const proxiedRequests: string[] = [] + const proxy = http.createServer((req, res) => { + proxiedRequests.push(req.url ?? '') + res.end('target') + }) + proxy.on('connect', (req: http.IncomingMessage, clientSocket: net.Socket, head: Buffer) => { + proxiedRequests.push(req.url ?? '') + const targetSocket = net.connect(targetPort, 'localhost', () => { + clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n') + targetSocket.write(head) + targetSocket.pipe(clientSocket) + clientSocket.pipe(targetSocket) + }) + }) + const proxyPort = await listen(proxy) + + return { + httpProxy: `http://localhost:${proxyPort.toString()}`, + proxiedRequests, + targetUrl: `http://localhost:${targetPort.toString()}/`, + close: async () => { + proxy.closeAllConnections() + target.closeAllConnections() + await Promise.all([close(proxy), close(target)]) + }, + } +} describe('tryGetAgent', () => { test(`should return an empty object when there is no httpProxy`, async () => { @@ -56,3 +113,76 @@ describe('tryGetAgent', () => { server.close() }) }) + +// https://github.com/netlify/cli/issues/3115 +describe('global proxy support', () => { + const originalDispatcher = getGlobalDispatcher() + const originalHttpsGlobalAgent = https.globalAgent + const originalNoProxy = process.env.NO_PROXY + const originalLowercaseNoProxy = process.env.no_proxy + let proxy: Awaited> + + const restoreEnv = (name: string, value: string | undefined) => { + if (value === undefined) { + // eslint-disable-next-line @typescript-eslint/no-dynamic-delete + delete process.env[name] + } else { + process.env[name] = value + } + } + + beforeEach(async () => { + delete process.env.NO_PROXY + delete process.env.no_proxy + proxy = await startProxy() + }) + + afterEach(async () => { + setGlobalDispatcher(originalDispatcher) + https.globalAgent = originalHttpsGlobalAgent + restoreEnv('NO_PROXY', originalNoProxy) + restoreEnv('no_proxy', originalLowercaseNoProxy) + await proxy.close() + }) + + test('getAgent sends global fetch requests through the proxy', async () => { + await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined }) + + const response = await fetch('http://netlify-cli-test.invalid/path') + + expect(await response.text()).toBe('target') + expect(proxy.proxiedRequests).toHaveLength(1) + expect(proxy.proxiedRequests[0]).toContain('netlify-cli-test.invalid') + }) + + // e.g. the Netlify API client that `@netlify/config` creates, which uses `node-fetch` without an agent + test('getAgent sends https requests without an agent through the proxy', async () => { + await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined }) + + // The test proxy can't complete the TLS handshake, so only check that the request reached it + await nodeFetch('https://netlify-cli-test.invalid/path').catch(() => undefined) + + expect(proxy.proxiedRequests).toEqual(['netlify-cli-test.invalid:443']) + }) + + test('does not send requests to the local machine through the proxy', async () => { + await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined }) + + const response = await fetch(proxy.targetUrl) + + expect(await response.text()).toBe('target') + expect(proxy.proxiedRequests).toEqual([]) + }) + + test('does not send requests to hosts in NO_PROXY through the proxy', async () => { + process.env.NO_PROXY = 'netlify-cli-test.invalid' + await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined }) + + // The host doesn't exist, so the requests fail when they don't go through the proxy + await fetch('http://netlify-cli-test.invalid/path').catch(() => undefined) + await nodeFetch('https://netlify-cli-test.invalid/path').catch(() => undefined) + await nodeFetch('https://api.netlify-cli-test.invalid/path').catch(() => undefined) + + expect(proxy.proxiedRequests).toEqual([]) + }) +})