fix: send all requests through the configured HTTP proxy

package.json

@@ -142,6 +142,7 @@
     "toml": "^5.0.0",
     "tomlify-j0.4": "^3.0.0",
     "ulid": "^3.0.1",
+    "undici": "^7.30.1",
     "update-notifier": "^7.3.1",
     "write-file-atomic": "^5.0.1",
     "ws": "^8.19.0",

src/lib/http-agent.ts

@@ -1,6 +1,11 @@
 import { readFile } from 'fs/promises'
+import type { ClientRequest } from 'http'
+import https from 'https'
+import net from 'net'
+import tls from 'tls'
 
 import { HttpsProxyAgent } from 'https-proxy-agent'
+import { EnvHttpProxyAgent, setGlobalDispatcher } from 'undici'
 
 import { NETLIFYDEVERR, NETLIFYDEVWARN, exit, log } from '../utils/command-helpers.js'
 import { waitPort } from './wait-port.js'
@@ -31,6 +36,92 @@ const DEFAULT_HTTPS_PORT = 443
 // 50 seconds
 const AGENT_PORT_TIMEOUT = 50_000
 
+// Requests to the local machine (e.g. the framework dev server or the functions server in `netlify dev`)
+// should never go through the proxy
+const LOOPBACK_HOSTS = ['localhost', '127.0.0.1', '[::1]']
+
+const getNoProxy = (): string =>
+  [process.env.NO_PROXY ?? process.env.no_proxy, ...LOOPBACK_HOSTS].filter(Boolean).join(',')
+
+const normalizeHostname = (hostname: string): string => hostname.toLowerCase().replace(/^\[(.*)\]$/, '$1')
+
+/**
+ * Whether a request to `hostname` should go through the proxy, using the same `NO_PROXY` rules as undici's
+ * `EnvHttpProxyAgent`, so `fetch()` and `http`/`https` requests behave the same.
+ */
+const shouldProxy = (hostname: string, port: number, noProxy: string): boolean => {
+  const host = normalizeHostname(hostname)
+
+  return noProxy
+    .split(/[,\s]/)
+    .filter(Boolean)
+    .every((entry) => {
+      if (entry === '*') {
+        return false
+      }
+
+      const parsed = /^(.+):(\d+)$/.exec(entry)
+      const entryHost = normalizeHostname((parsed ? parsed[1] : entry).replace(/^\*?\./, ''))
+      if (parsed && Number.parseInt(parsed[2], 10) !== port) {
+        return true
+      }
+
+      return host !== entryHost && !host.endsWith(`.${entryHost}`)
+    })
+}
+
+type ConnectOpts = Parameters<HttpsProxyAgent<string>['connect']>[1]
+
+/**
+ * Agent for `https` requests that don't set their own agent, e.g. the Netlify API client that
+ * `@netlify/config` creates to retrieve the site info. Connects directly to hosts in `NO_PROXY`.
+ */
+class HttpsProxyAgentWithNoProxy extends HttpsProxyAgent<string> {
+  constructor(
+    proxy: string,
+    private readonly noProxy: string,
+    private readonly ca: Buffer | undefined,
+  ) {
+    super(proxy)
+    // Used when connecting to an `https:` proxy
+    if (ca) {
+      this.connectOpts.ca = ca
+    }
+  }
+
+  async connect(req: ClientRequest, opts: ConnectOpts): Promise<net.Socket> {
+    if (!shouldProxy(opts.host ?? 'localhost', opts.port, this.noProxy)) {
+      return opts.secureEndpoint
+        ? tls.connect({ ...opts, servername: net.isIP(opts.host ?? '') ? undefined : opts.host })
+        : net.connect(opts)
+    }
+
+    // Used for the TLS connection to the target host through the proxy
+    return super.connect(req, opts.secureEndpoint && this.ca ? { ...opts, ca: this.ca } : opts)
+  }
+}
+
+/**
+ * Node.js's built-in `fetch()` ignores both `HTTP_PROXY` and the `agent` option, and so does any `http`/`https`
+ * request that doesn't set an agent. The Netlify API client gets an agent, but other requests, e.g. by
+ * `@netlify/config` or the CLI's own `fetch()` calls, need the proxy to be set globally.
+ */
+const getGlobalProxyAgents = (httpProxy: string, certificate: Buffer | undefined) => {
+  const noProxy = getNoProxy()
+  const tlsOptions = certificate ? { ca: certificate } : undefined
+
+  return {
+    dispatcher: new EnvHttpProxyAgent({
+      httpProxy,
+      httpsProxy: httpProxy,
+      noProxy,
+      requestTls: tlsOptions,
+      proxyTls: tlsOptions,
+    }),
+    httpsGlobalAgent: new HttpsProxyAgentWithNoProxy(httpProxy, noProxy, certificate),
+  }
+}
+
 export const tryGetAgent = async ({
   certificateFile,
   httpProxy,
@@ -45,6 +136,8 @@ export const tryGetAgent = async ({
     }
   | {
       agent: HttpsProxyAgentWithCA
+      dispatcher: EnvHttpProxyAgent
+      httpsGlobalAgent: https.Agent
       response: unknown
     }
 > => {
@@ -103,20 +196,25 @@ export const tryGetAgent = async ({
   }
 
   const agent = new HttpsProxyAgentWithCA(opts)
-  response = { ...response, agent }
+  response = { ...response, agent, ...getGlobalProxyAgents(httpProxy, certificate) }
   return response
 }
 
 // @ts-expect-error TS(7031) FIXME: Binding element 'certificateFile' implicitly has a... Remove this comment to see the full error message
 export const getAgent = async ({ certificateFile, httpProxy }) => {
+  const result = await tryGetAgent({ httpProxy, certificateFile })
   // @ts-expect-error TS(2339) FIXME: Property 'agent' does not exist on type '{ error?:... Remove this comment to see the full error message
-  const { agent, error, message, warning } = await tryGetAgent({ httpProxy, certificateFile })
+  const { agent, error, message, warning } = result
   if (error) {
     log(NETLIFYDEVERR, error, message || '')
     exit(1)
   }
   if (warning) {
     log(NETLIFYDEVWARN, warning, message || '')
   }
+  if ('dispatcher' in result) {
+    setGlobalDispatcher(result.dispatcher)
+    https.globalAgent = result.httpsGlobalAgent
+  }
   return agent
 }

tests/unit/lib/http-agent.test.ts

@@ -1,11 +1,68 @@
 import net from 'node:net'
 import http from 'node:http'
+import https from 'node:https'
 
 import ProxyServer from 'http-proxy'
 import { HttpsProxyAgent } from 'https-proxy-agent'
-import { describe, expect, test } from 'vitest'
+import nodeFetch from 'node-fetch'
+import { getGlobalDispatcher, setGlobalDispatcher } from 'undici'
+import { afterEach, beforeEach, describe, expect, test } from 'vitest'
 
-import { tryGetAgent } from '../../../src/lib/http-agent.js'
+import { getAgent, tryGetAgent } from '../../../src/lib/http-agent.js'
+
+const listen = async (server: net.Server): Promise<number> => {
+  await new Promise<void>((resolve) => {
+    server.listen({ port: 0, hostname: 'localhost' }, () => {
+      resolve()
+    })
+  })
+  return (server.address() as net.AddressInfo).port
+}
+
+const close = (server: net.Server) =>
+  new Promise<void>((resolve) => {
+    server.close(() => {
+      resolve()
+    })
+  })
+
+/**
+ * Starts a server that responds with `target`, and a proxy that records the requests it receives
+ * and sends them to that server, no matter which host they are for.
+ */
+const startProxy = async () => {
+  const target = http.createServer((_, res) => {
+    res.end('target')
+  })
+  const targetPort = await listen(target)
+
+  const proxiedRequests: string[] = []
+  const proxy = http.createServer((req, res) => {
+    proxiedRequests.push(req.url ?? '')
+    res.end('target')
+  })
+  proxy.on('connect', (req: http.IncomingMessage, clientSocket: net.Socket, head: Buffer) => {
+    proxiedRequests.push(req.url ?? '')
+    const targetSocket = net.connect(targetPort, 'localhost', () => {
+      clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n')
+      targetSocket.write(head)
+      targetSocket.pipe(clientSocket)
+      clientSocket.pipe(targetSocket)
+    })
+  })
+  const proxyPort = await listen(proxy)
+
+  return {
+    httpProxy: `http://localhost:${proxyPort.toString()}`,
+    proxiedRequests,
+    targetUrl: `http://localhost:${targetPort.toString()}/`,
+    close: async () => {
+      proxy.closeAllConnections()
+      target.closeAllConnections()
+      await Promise.all([close(proxy), close(target)])
+    },
+  }
+}
 
 describe('tryGetAgent', () => {
   test(`should return an empty object when there is no httpProxy`, async () => {
@@ -56,3 +113,76 @@ describe('tryGetAgent', () => {
     server.close()
   })
 })
+
+// https://github.com/netlify/cli/issues/3115
+describe('global proxy support', () => {
+  const originalDispatcher = getGlobalDispatcher()
+  const originalHttpsGlobalAgent = https.globalAgent
+  const originalNoProxy = process.env.NO_PROXY
+  const originalLowercaseNoProxy = process.env.no_proxy
+  let proxy: Awaited<ReturnType<typeof startProxy>>
+
+  const restoreEnv = (name: string, value: string | undefined) => {
+    if (value === undefined) {
+      // eslint-disable-next-line @typescript-eslint/no-dynamic-delete
+      delete process.env[name]
+    } else {
+      process.env[name] = value
+    }
+  }
+
+  beforeEach(async () => {
+    delete process.env.NO_PROXY
+    delete process.env.no_proxy
+    proxy = await startProxy()
+  })
+
+  afterEach(async () => {
+    setGlobalDispatcher(originalDispatcher)
+    https.globalAgent = originalHttpsGlobalAgent
+    restoreEnv('NO_PROXY', originalNoProxy)
+    restoreEnv('no_proxy', originalLowercaseNoProxy)
+    await proxy.close()
+  })
+
+  test('getAgent sends global fetch requests through the proxy', async () => {
+    await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined })
+
+    const response = await fetch('http://netlify-cli-test.invalid/path')
+
+    expect(await response.text()).toBe('target')
+    expect(proxy.proxiedRequests).toHaveLength(1)
+    expect(proxy.proxiedRequests[0]).toContain('netlify-cli-test.invalid')
+  })
+
+  // e.g. the Netlify API client that `@netlify/config` creates, which uses `node-fetch` without an agent
+  test('getAgent sends https requests without an agent through the proxy', async () => {
+    await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined })
+
+    // The test proxy can't complete the TLS handshake, so only check that the request reached it
+    await nodeFetch('https://netlify-cli-test.invalid/path').catch(() => undefined)
+
+    expect(proxy.proxiedRequests).toEqual(['netlify-cli-test.invalid:443'])
+  })
+
+  test('does not send requests to the local machine through the proxy', async () => {
+    await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined })
+
+    const response = await fetch(proxy.targetUrl)
+
+    expect(await response.text()).toBe('target')
+    expect(proxy.proxiedRequests).toEqual([])
+  })
+
+  test('does not send requests to hosts in NO_PROXY through the proxy', async () => {
+    process.env.NO_PROXY = 'netlify-cli-test.invalid'
+    await getAgent({ httpProxy: proxy.httpProxy, certificateFile: undefined })
+
+    // The host doesn't exist, so the requests fail when they don't go through the proxy
+    await fetch('http://netlify-cli-test.invalid/path').catch(() => undefined)
+    await nodeFetch('https://netlify-cli-test.invalid/path').catch(() => undefined)
+    await nodeFetch('https://api.netlify-cli-test.invalid/path').catch(() => undefined)
+
+    expect(proxy.proxiedRequests).toEqual([])
+  })
+})