diff --git a/.github/workflows/edgee-release.yml b/.github/workflows/edgee-release.yml index eb0d7aac3bd..0c2e0a93c5e 100644 --- a/.github/workflows/edgee-release.yml +++ b/.github/workflows/edgee-release.yml @@ -54,6 +54,7 @@ jobs: -run 'Test(UtlsClient|InvalidProxyURL|ClientsForTheSameProxy|Connect|RetryBackoff|ReviewNotEstablishedConnIsReused|EquivalentProxySettings|NewClaudeAuth|PatchAuthFileFields|ConfigHandlersRejectInvalidProxyURL|UploadAuthFileRejectsBadProxyURL)' ./internal/api/handlers/management ./internal/auth/claude + ./internal/runtime/executor ./internal/runtime/executor/helps ./sdk/proxyutil diff --git a/internal/runtime/executor/codex_executor.go b/internal/runtime/executor/codex_executor.go index e2b6d42545e..65b75b8af87 100644 --- a/internal/runtime/executor/codex_executor.go +++ b/internal/runtime/executor/codex_executor.go @@ -1179,6 +1179,7 @@ func (e *CodexExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Au if bytes.HasPrefix(line, dataTag) { data := bytes.TrimSpace(line[5:]) + markCodexResponseComplete(httpResp.Body, data) if streamErr, terminalBody, ok := codexTerminalStreamErr(data); ok { if errClearReplay := clearCodexReasoningReplayOnInvalidSignature(ctx, replayScope, streamErr.StatusCode(), terminalBody); errClearReplay != nil { helps.RecordAPIResponseError(ctx, e.cfg, errClearReplay) @@ -1233,6 +1234,13 @@ func (e *CodexExecutor) ExecuteStream(ctx context.Context, auth *cliproxyauth.Au return &cliproxyexecutor.StreamResult{Headers: httpResp.Header.Clone(), Chunks: out}, nil } +func markCodexResponseComplete(body io.ReadCloser, data []byte) { + switch gjson.GetBytes(data, "type").String() { + case "response.completed", "response.incomplete", "response.failed", "error": + helps.MarkResponseComplete(body) + } +} + func (e *CodexExecutor) CountTokens(ctx context.Context, auth *cliproxyauth.Auth, req cliproxyexecutor.Request, opts cliproxyexecutor.Options) (cliproxyexecutor.Response, error) { baseModel := thinking.ParseSuffix(req.Model).ModelName diff --git a/internal/runtime/executor/codex_utls_pool_test.go b/internal/runtime/executor/codex_utls_pool_test.go new file mode 100644 index 00000000000..fd4ee9de4e9 --- /dev/null +++ b/internal/runtime/executor/codex_utls_pool_test.go @@ -0,0 +1,240 @@ +package executor + +import ( + "bufio" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + stdtls "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/router-for-me/CLIProxyAPI/v7/internal/config" + "github.com/router-for-me/CLIProxyAPI/v7/internal/runtime/executor/helps" + cliproxyauth "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/auth" + cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v7/sdk/cliproxy/executor" + sdkconfig "github.com/router-for-me/CLIProxyAPI/v7/sdk/config" + sdktranslator "github.com/router-for-me/CLIProxyAPI/v7/sdk/translator" +) + +const ( + codexPoolCreatedEvent = "event: response.created\ndata: {\"type\":\"response.created\",\"response\":{\"id\":\"r1\"}}\n\n" + codexPoolCompletedEvent = "event: response.completed\ndata: {\"type\":\"response.completed\",\"response\":{\"id\":\"r1\",\"status\":\"completed\",\"output\":[],\"usage\":{\"input_tokens\":1,\"output_tokens\":1,\"total_tokens\":2}}}\n\n" +) + +// startCodexChatgptServer serves handler over HTTP/2 with a chatgpt.com certificate the uTLS +// pool trusts, so CodexExecutor traffic goes through the shared pool. +func startCodexChatgptServer(t *testing.T, handler http.Handler) *httptest.Server { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "chatgpt.com"}, + DNSNames: []string{"chatgpt.com"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + roots.AddCert(leaf) + helps.SetUtlsRootCAsForTest(roots) + t.Cleanup(func() { helps.SetUtlsRootCAsForTest(nil) }) + server := httptest.NewUnstartedServer(handler) + server.EnableHTTP2 = true + server.TLS = &stdtls.Config{Certificates: []stdtls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key, Leaf: leaf}}} + server.StartTLS() + t.Cleanup(server.Close) + return server +} + +// codexTunnelProxy is a CONNECT proxy that sends every tunnel to the local server and counts them. +// With drop set it stops forwarding bytes while keeping tunnels open. +type codexTunnelProxy struct { + url string + opened atomic.Int32 + open atomic.Int32 + drop atomic.Bool +} + +func startCodexTunnelProxy(t *testing.T) *codexTunnelProxy { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + p := &codexTunnelProxy{url: "http://" + ln.Addr().String()} + go func() { + for { + client, errAccept := ln.Accept() + if errAccept != nil { + return + } + go p.serve(client) + } + }() + return p +} + +func (p *codexTunnelProxy) serve(client net.Conn) { + defer client.Close() + req, err := http.ReadRequest(bufio.NewReader(client)) + if err != nil || req.Method != http.MethodConnect { + return + } + _, port, _ := net.SplitHostPort(req.Host) + upstream, err := net.Dial("tcp", net.JoinHostPort("127.0.0.1", port)) + if err != nil { + return + } + defer upstream.Close() + p.opened.Add(1) + p.open.Add(1) + defer p.open.Add(-1) + _, _ = io.WriteString(client, "HTTP/1.1 200 Connection established\r\n\r\n") + done := make(chan struct{}, 2) + pipe := func(dst, src net.Conn) { + buf := make([]byte, 32*1024) + for { + n, errRead := src.Read(buf) + if n > 0 && !p.drop.Load() { + if _, errWrite := dst.Write(buf[:n]); errWrite != nil { + break + } + } + if errRead != nil { + break + } + } + done <- struct{}{} + } + go pipe(upstream, client) + go pipe(client, upstream) + <-done +} + +func codexPoolExecutor(p *codexTunnelProxy, server *httptest.Server, authID string) (*CodexExecutor, *cliproxyauth.Auth) { + _, port, _ := net.SplitHostPort(server.Listener.Addr().String()) + auth := &cliproxyauth.Auth{ID: authID, Attributes: map[string]string{ + "base_url": "https://chatgpt.com:" + port + "/backend-api/codex", + "api_key": "test", + }} + return NewCodexExecutor(&config.Config{SDKConfig: sdkconfig.SDKConfig{ProxyURL: p.url}}), auth +} + +func codexPoolStream(t *testing.T, ctx context.Context, exec *CodexExecutor, auth *cliproxyauth.Auth) *cliproxyexecutor.StreamResult { + t.Helper() + result, err := exec.ExecuteStream(ctx, auth, + cliproxyexecutor.Request{Model: "gpt-5.5", Payload: []byte(`{"model":"gpt-5.5","input":"hello"}`)}, + cliproxyexecutor.Options{SourceFormat: sdktranslator.FromString("openai-response"), Stream: true}) + if err != nil { + t.Fatalf("ExecuteStream: %v", err) + } + return result +} + +func TestUtlsClientCodexStreamCancelledAfterCompletedKeepsTheTunnel(t *testing.T) { + server := startCodexChatgptServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, codexPoolCreatedEvent+codexPoolCompletedEvent) + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-time.After(200 * time.Millisecond): + } + })) + p := startCodexTunnelProxy(t) + exec, auth := codexPoolExecutor(p, server, "codex-pool-completed") + for i := 0; i < 5; i++ { + ctx, cancel := context.WithCancel(context.Background()) + result := codexPoolStream(t, ctx, exec, auth) + saw := false + for chunk := range result.Chunks { + if chunk.Err != nil { + t.Fatalf("request %d: %v", i, chunk.Err) + } + if strings.Contains(string(chunk.Payload), "response.completed") { + saw = true + cancel() + break + } + } + for range result.Chunks { + } + cancel() + if !saw { + t.Fatalf("request %d never saw response.completed", i) + } + } + if got := p.opened.Load(); got != 1 { + t.Fatalf("tunnels opened = %d, want 1", got) + } +} + +func TestUtlsClientCodexStreamSilentWithoutFinalEventRetiresTheTunnel(t *testing.T) { + quit := make(chan struct{}) + t.Cleanup(func() { close(quit) }) + server := startCodexChatgptServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, codexPoolCreatedEvent) + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-quit: + } + })) + p := startCodexTunnelProxy(t) + exec, auth := codexPoolExecutor(p, server, "codex-pool-silent") + + ctx, cancel := context.WithCancel(context.Background()) + result := codexPoolStream(t, ctx, exec, auth) + <-result.Chunks + p.drop.Store(true) + cancel() + for range result.Chunks { + } + deadline := time.Now().Add(3 * time.Second) + for p.open.Load() != 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if got := p.open.Load(); got != 0 { + t.Fatalf("silent tunnel still open: %d", got) + } + + p.drop.Store(false) + ctx2, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel2() + result = codexPoolStream(t, ctx2, exec, auth) + if chunk := <-result.Chunks; chunk.Err != nil { + t.Fatalf("request after the silent stream: %v", chunk.Err) + } + cancel2() + for range result.Chunks { + } + if got := p.opened.Load(); got != 2 { + t.Fatalf("tunnels opened = %d, want 2", got) + } +} diff --git a/internal/runtime/executor/helps/utls_client.go b/internal/runtime/executor/helps/utls_client.go index e74a6e3a0d6..63a345eb87d 100644 --- a/internal/runtime/executor/helps/utls_client.go +++ b/internal/runtime/executor/helps/utls_client.go @@ -39,6 +39,14 @@ var ( utlsTestRootCAs *x509.CertPool ) +// SetUtlsRootCAsForTest makes pools created afterwards trust roots; tests outside this +// package use it to reach a local TLS server. nil restores the system roots. +func SetUtlsRootCAsForTest(roots *x509.CertPool) { + sharedUtlsMu.Lock() + utlsTestRootCAs = roots + sharedUtlsMu.Unlock() +} + // UtlsTransport returns the process-wide Chrome-fingerprinted HTTP/2 transport for // proxyURL ("" dials directly) used without an account, such as login flows. // An invalid proxy URL yields a transport that fails every request. @@ -426,9 +434,26 @@ type utlsResponseBody struct { conn *utlsPooledConn release func() lastRead atomic.Uint64 + complete atomic.Bool once sync.Once } +// MarkResponseComplete records that the caller has read the response's final event, so a +// cancel after it is not taken as a silent connection. +func MarkResponseComplete(body io.ReadCloser) { + for body != nil { + switch b := body.(type) { + case *utlsResponseBody: + b.complete.Store(true) + return + case *usageTTFTReadCloser: + body = b.ReadCloser + default: + return + } + } +} + func (b *utlsResponseBody) Read(p []byte) (int, error) { n, err := b.ReadCloser.Read(p) if err != nil { @@ -448,7 +473,7 @@ func (b *utlsResponseBody) Close() error { func (b *utlsResponseBody) finish(complete bool) { b.once.Do(func() { b.release() - if !complete && b.ctx.Err() != nil { + if !complete && !b.complete.Load() && b.ctx.Err() != nil { b.pool.dropIfSilent(b.conn, b.lastRead.Load()) } }) diff --git a/internal/runtime/executor/helps/utls_client_test.go b/internal/runtime/executor/helps/utls_client_test.go index ac89f3c07dd..dec6b7186f9 100644 --- a/internal/runtime/executor/helps/utls_client_test.go +++ b/internal/runtime/executor/helps/utls_client_test.go @@ -1140,6 +1140,41 @@ func TestUtlsClientRetiresSilentTunnelWhenAStreamedResponseIsCancelled(t *testin waitFor(t, "the silent tunnel to close", func() bool { return p.open.Load() == 1 }) } +func TestUtlsClientKeepsTheTunnelWhenAStreamIsCancelledAfterItsFinalEvent(t *testing.T) { + t.Parallel() + server := startChatgptServer(t, true, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "data: {\"type\":\"response.completed\"}\n\n") + w.(http.Flusher).Flush() + select { + case <-r.Context().Done(): + case <-time.After(50 * time.Millisecond): + } + })) + p := startConnectProxy(t, false) + client := newProxiedUtlsClient(p.url) + + for i := 0; i < 5; i++ { + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, chatgptURL(server, "/sse"), strings.NewReader("{}")) + if err != nil { + t.Fatal(err) + } + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + if _, err = resp.Body.Read(make([]byte, 64)); err != nil { + t.Fatalf("request %d final event: %v", i, err) + } + MarkResponseComplete(&usageTTFTReadCloser{ReadCloser: resp.Body}) // as executors see it via TrackHTTPClient + cancel() + _ = resp.Body.Close() + } + if got := p.opened.Load(); got != 1 { + t.Fatalf("tunnels opened = %d, want 1 for streams cancelled after their final event", got) + } +} + func TestUtlsClientClosesARetiredConnectionStuckBehindAFullSendBuffer(t *testing.T) { t.Parallel() startChatgptServer(t, true, http.NotFoundHandler()) // installs the chatgpt.com cert and test roots