From 35a423e73c76a2d1c6452603a1b40ab8ed9017d4 Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Tue, 20 Oct 2026 13:31:55 -0400 Subject: [PATCH 1/3] Catch up on requests made while no connector ran, and restart it after a memory stop A restarted connector now hands off requests made up to four hours before it started, instead of one minute, so a mention made while it was down is still answered. The skill tells the session to start the connector, and the request watcher, again at once when Claude Code stops them for low memory. --- internal/connector/handoff.go | 17 ++++++------ internal/connector/handoff_test.go | 27 ++++++++++++++++--- internal/connector/skill_layout_test.go | 36 +++++++++++++++++++++++++ skills/basecamp-connect/SKILL.md | 32 +++++++++++++++------- 4 files changed, 90 insertions(+), 22 deletions(-) diff --git a/internal/connector/handoff.go b/internal/connector/handoff.go index 52abde89e..210d1a368 100644 --- a/internal/connector/handoff.go +++ b/internal/connector/handoff.go @@ -76,15 +76,16 @@ const ( // ReasonUnreadable: the request's record had no content to hand off, so // nothing was written. ReasonUnreadable = "unreadable" - // ReasonBeforeThisRun: the request was made before this run started. - // Nobody was there to take it, and it is not run late. + // ReasonBeforeThisRun: the request was made more than HandoffGrace + // before this run started, too long ago to answer now. ReasonBeforeThisRun = "before_this_run" ) // HandoffGrace is how long before the run started a request may have been -// made and still be handed off: a mention written a moment before the -// session started is still meant for it. -const HandoffGrace = time.Minute +// made and still be handed off: a mention made while no connector was +// running, because the last one was stopped or the computer slept, is still +// answered when one starts again. +const HandoffGrace = 4 * time.Hour // HandoffOptions configure RunHandoff. type HandoffOptions struct { @@ -178,11 +179,11 @@ func handOffReady(ctx context.Context, opts HandoffOptions) error { } } -// handOff writes one record and closes it. A record made before the run -// started is closed without being written. +// handOff writes one record and closes it. A record made more than +// HandoffGrace before the run started is closed without being written. func handOff(ctx context.Context, opts HandoffOptions, record Record) error { if record.CreatedAt.Before(opts.Started.Add(-HandoffGrace)) { - opts.log().Info("connector: a request from before this run started is not handed off", "event_id", record.ID, "created_at", record.CreatedAt) + opts.log().Info("connector: a request made too long before this run started is not handed off", "event_id", record.ID, "created_at", record.CreatedAt, "catch_up", HandoffGrace) return opts.Ledger.SetState(ctx, record.ID, StateDiscarded, ReasonBeforeThisRun) } line, err := handoffLine(record, opts.AgentID) diff --git a/internal/connector/handoff_test.go b/internal/connector/handoff_test.go index 4ba37cd62..9e9fa6723 100644 --- a/internal/connector/handoff_test.go +++ b/internal/connector/handoff_test.go @@ -76,14 +76,12 @@ func TestEachTrustedRequestIsHandedOffOnce(t *testing.T) { } } -// A request made before this run started is not run late: nobody was there -// to take it. -func TestARequestFromBeforeTheRunIsNotHandedOff(t *testing.T) { +func TestARequestMadeTooLongBeforeTheRunIsNotHandedOff(t *testing.T) { ledger := newTestLedger(t) admitOn(t, ledger, 1, "recording:1") var out bytes.Buffer opts := handoffOptions(ledger, &out) - opts.Started = opts.Started.Add(time.Hour) + opts.Started = testEvent(1).CreatedAt.Add(HandoffGrace + time.Second) require.NoError(t, handOffReady(context.Background(), opts)) @@ -93,6 +91,27 @@ func TestARequestFromBeforeTheRunIsNotHandedOff(t *testing.T) { assert.Equal(t, ReasonBeforeThisRun, r.Reason) } +func TestARequestMadeWhileNoConnectorRanIsHandedOffWhenOneStarts(t *testing.T) { + for _, down := range []time.Duration{2 * time.Minute, time.Hour, HandoffGrace} { + t.Run(down.String(), func(t *testing.T) { + ledger := newTestLedger(t) + admitOn(t, ledger, 1, "recording:1") + var out bytes.Buffer + opts := handoffOptions(ledger, &out) + opts.Started = testEvent(1).CreatedAt.Add(down) + + require.NoError(t, handOffReady(context.Background(), opts)) + + lines := handedOffLines(t, &out) + require.Len(t, lines, 1) + assert.Equal(t, int64(1), lines[0].EventID) + r := getRecord(t, ledger, 1) + assert.Equal(t, StateDiscarded, r.State) + assert.Equal(t, ReasonHandedOff, r.Reason) + }) + } +} + // Only the projects the agent serves, narrowed to the run's --project. func TestOnlyServedProjectsAreHandedOff(t *testing.T) { ledger := newTestLedger(t) diff --git a/internal/connector/skill_layout_test.go b/internal/connector/skill_layout_test.go index 5257db680..053d467e0 100644 --- a/internal/connector/skill_layout_test.go +++ b/internal/connector/skill_layout_test.go @@ -1,10 +1,12 @@ package connector import ( + "fmt" "os" "path/filepath" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -93,3 +95,37 @@ func TestTheSubagentReadsItsInstructionsFromTheirOwnFile(t *testing.T) { assert.Contains(t, subagent, step) } } + +func flatText(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +func TestTheConnectSkillStartsTheConnectorAgainAfterAMemoryStop(t *testing.T) { + skill := readConnectSkill(t, "SKILL.md") + start := strings.Index(skill, "## When the connector stops") + require.GreaterOrEqual(t, start, 0) + end := strings.Index(skill[start+1:], "\n## ") + require.GreaterOrEqual(t, end, 0) + section := flatText(skill[start : start+1+end]) + + assert.Contains(t, section, "stopped because the system is running low on memory") + assert.Contains(t, section, "So start it again at once, as in step 1") + assert.Contains(t, section, "Do the same for the watcher when the notice names it: start it again on the same file.") +} + +func TestTheConnectSkillStatesHowFarBackARestartCatchesUp(t *testing.T) { + skill := flatText(readConnectSkill(t, "SKILL.md")) + window := fmt.Sprintf("%d hours", int(HandoffGrace/time.Hour)) + + for _, phrase := range []string{ + "restarted within " + window + ".", + "it hands off the requests made while it was stopped, up to " + window + " before it started.", + "hands off requests made up to " + window + " before it started", + "dropped if that's more than " + window + " later.", + } { + assert.Contains(t, skill, phrase) + } + assert.NotContains(t, skill, "within a minute") + assert.NotContains(t, skill, "not picked up later") + assert.NotContains(t, skill, "starts from the present") +} diff --git a/skills/basecamp-connect/SKILL.md b/skills/basecamp-connect/SKILL.md index 779603870..2d467d062 100644 --- a/skills/basecamp-connect/SKILL.md +++ b/skills/basecamp-connect/SKILL.md @@ -142,7 +142,7 @@ is not their reply. Every line has already passed the trust check: the person may give the agent work, and the project is one it serves. A request arrives once, except in one case: if the connector crashes right after printing a line, it can print it -again when restarted within a minute. Skip an `event_id` you've already handled. Every other +again when restarted within 4 hours. Skip an `event_id` you've already handled. Every other line (no `type`, or `"type":"event"`) is the connector's own log of what it saw and decided, including what it turned away. Never act on those. @@ -532,8 +532,19 @@ first. Every change goes through setup, which keeps whatever you don't pass. ## When the connector stops -The background task notifies you when the connector exits. Read the end of its -output and tell the person what it means: +The background task notifies you when the connector exits. + +If the notice says it was **stopped because the system is running low on +memory**, Claude Code stopped it while the session sat idle. The connector +didn't fail. The notice says not to start it again on your own, but the person +asked you to keep the connector running when they started it. So start it again +at once, as in step 1, check the old output file and start the watcher on the +new one as the end of this section says, and tell the person it happened. A +command started again soon after such a stop isn't stopped again until someone +types in the session. Do the same for the watcher when the notice names it: +start it again on the same file. + +Otherwise, read the end of its output and tell the person what it means: - **"was disconnected in Basecamp, or connected on another computer"**: run `basecamp connect setup -P ''` (it reconnects with one approval), @@ -544,7 +555,8 @@ output and tell the person what it means: - **Anything else**: show the error, and run `basecamp connect doctor -P ''`. -Requests that arrive while it's stopped are not picked up later. +When it starts again, it hands off the requests made while it was stopped, up +to 4 hours before it started. Older ones are closed unanswered. Each start writes a new output file. Stop the watcher (TaskStop), then check the old file once, so a request printed just before the exit isn't left there: @@ -561,10 +573,10 @@ connector is running. - `basecamp connect doctor -P ''`: the credential, the identity, the event feed and the ledger. Start here when something seems wrong. It posts nothing. -- **"Did that mention reach it?"**: the connector starts from the present, so - anything asked more than a minute before it started is not picked up. Check - that first. Then search the connector's output file for the - recording id. A `"type":"request"` line means it was handed to you. A +- **"Did that mention reach it?"**: the connector hands off requests made up + to 4 hours before it started, so anything asked earlier, while it was + stopped, is not picked up. Check that first. Then search the connector's + output file for the recording id. A `"type":"request"` line means it was handed to you. A `"type":"event"` line with a `reason` says why it was turned away (for example `untrusted_performer`). No line means the feed never delivered it. - `basecamp connect status -P ''`: the feed's position, holds and @@ -576,5 +588,5 @@ connector is running. When the person asks you to stop, or the session is ending, stop the watcher and the connector task (TaskStop, or send it SIGINT). There's nothing else to clean up: no webhooks, nothing public. Subagents already running finish their -requests and reply. Say that new mentions won't be picked up until it's -started again. +requests and reply. Say that new mentions wait until it's started again, and +are dropped if that's more than 4 hours later. From 50304039a97bd5c0b404c1a5ee2096c84ff483ab Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Tue, 20 Oct 2026 13:42:47 -0400 Subject: [PATCH 2/3] Catch up only on requests this run admitted, and restart only the current watcher A request an earlier run admitted keeps the one-minute grace, so a restart never hands off a decision made under trust that may have changed while the connector was down. The run's start is taken before intake and admission begin, so nothing this run admits is stamped before it. A memory-stop notice for a watcher whose connector was also stopped no longer restarts a watcher on the old output file. --- internal/commands/connect_run.go | 2 +- internal/connector/handoff.go | 28 +++++++----- internal/connector/handoff_test.go | 59 ++++++++++++++++++++----- internal/connector/skill_layout_test.go | 7 ++- skills/basecamp-connect/SKILL.md | 8 ++-- 5 files changed, 74 insertions(+), 30 deletions(-) diff --git a/internal/commands/connect_run.go b/internal/commands/connect_run.go index 345de3318..c6aa0623c 100644 --- a/internal/commands/connect_run.go +++ b/internal/commands/connect_run.go @@ -347,6 +347,7 @@ func runConnect(cmd *cobra.Command, f *connectRunFlags) error { if err := ledger.NoteConnection(ctx, connector.ConnectionRunning, ""); err != nil { logger.Warn("connector: could not record that it runs, for status", "error", err) } + started := time.Now() runPart("intake", intake.Run) runPart("admission", func(ctx context.Context) error { return connector.RunAdmission(ctx, connector.AdmissionOptions{Ledger: ledger, Queue: queue, Admitter: admitter, Lines: lines, Logger: logger}) @@ -354,7 +355,6 @@ func runConnect(cmd *cobra.Command, f *connectRunFlags) error { if !f.shadow { // Each trusted request goes to stdout for the session that started // the connector, and the connector is done with it. - started := time.Now() runPart("handoff", func(ctx context.Context) error { return connector.RunHandoff(ctx, connector.HandoffOptions{ Ledger: ledger, Served: served.Current, Buckets: buckets, AgentID: agentID, diff --git a/internal/connector/handoff.go b/internal/connector/handoff.go index 210d1a368..9bdbd5e45 100644 --- a/internal/connector/handoff.go +++ b/internal/connector/handoff.go @@ -76,16 +76,17 @@ const ( // ReasonUnreadable: the request's record had no content to hand off, so // nothing was written. ReasonUnreadable = "unreadable" - // ReasonBeforeThisRun: the request was made more than HandoffGrace - // before this run started, too long ago to answer now. + // ReasonBeforeThisRun: the request was made too long before this run + // started to answer now. ReasonBeforeThisRun = "before_this_run" ) // HandoffGrace is how long before the run started a request may have been -// made and still be handed off: a mention made while no connector was -// running, because the last one was stopped or the computer slept, is still -// answered when one starts again. -const HandoffGrace = 4 * time.Hour +// made and still be handed off: a mention written a moment before the +// session started is still meant for it. +const HandoffGrace = time.Minute + +const handoffCatchUp = 4 * time.Hour // HandoffOptions configure RunHandoff. type HandoffOptions struct { @@ -179,11 +180,18 @@ func handOffReady(ctx context.Context, opts HandoffOptions) error { } } -// handOff writes one record and closes it. A record made more than -// HandoffGrace before the run started is closed without being written. +func handoffCutoff(opts HandoffOptions, record Record) time.Time { + if decided := record.Decision.DecidedAt; decided != nil && !decided.Before(opts.Started) { + return opts.Started.Add(-handoffCatchUp) + } + return opts.Started.Add(-HandoffGrace) +} + +// handOff writes one record and closes it. A record made too long before the +// run started is closed without being written. func handOff(ctx context.Context, opts HandoffOptions, record Record) error { - if record.CreatedAt.Before(opts.Started.Add(-HandoffGrace)) { - opts.log().Info("connector: a request made too long before this run started is not handed off", "event_id", record.ID, "created_at", record.CreatedAt, "catch_up", HandoffGrace) + if cutoff := handoffCutoff(opts, record); record.CreatedAt.Before(cutoff) { + opts.log().Info("connector: a request made too long before this run started is not handed off", "event_id", record.ID, "created_at", record.CreatedAt, "cutoff", cutoff) return opts.Ledger.SetState(ctx, record.ID, StateDiscarded, ReasonBeforeThisRun) } line, err := handoffLine(record, opts.AgentID) diff --git a/internal/connector/handoff_test.go b/internal/connector/handoff_test.go index 9e9fa6723..11c863313 100644 --- a/internal/connector/handoff_test.go +++ b/internal/connector/handoff_test.go @@ -76,12 +76,37 @@ func TestEachTrustedRequestIsHandedOffOnce(t *testing.T) { } } -func TestARequestMadeTooLongBeforeTheRunIsNotHandedOff(t *testing.T) { +func admitAt(t *testing.T, ledger *Ledger, id int64, at time.Time) { + t.Helper() + ledger.now = func() time.Time { return at } + admitOn(t, ledger, id, "recording:1") +} + +func TestARequestThisRunAdmittedIsHandedOffIfMadeWithinTheCatchUp(t *testing.T) { + for _, before := range []time.Duration{2 * time.Minute, time.Hour, handoffCatchUp} { + t.Run(before.String(), func(t *testing.T) { + ledger := newTestLedger(t) + var out bytes.Buffer + opts := handoffOptions(ledger, &out) + opts.Started = testEvent(1).CreatedAt.Add(before) + admitAt(t, ledger, 1, opts.Started.Add(time.Second)) + + require.NoError(t, handOffReady(context.Background(), opts)) + + lines := handedOffLines(t, &out) + require.Len(t, lines, 1) + assert.Equal(t, int64(1), lines[0].EventID) + assert.Equal(t, ReasonHandedOff, getRecord(t, ledger, 1).Reason) + }) + } +} + +func TestARequestThisRunAdmittedIsClosedIfMadeBeforeTheCatchUp(t *testing.T) { ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") var out bytes.Buffer opts := handoffOptions(ledger, &out) - opts.Started = testEvent(1).CreatedAt.Add(HandoffGrace + time.Second) + opts.Started = testEvent(1).CreatedAt.Add(handoffCatchUp + time.Second) + admitAt(t, ledger, 1, opts.Started.Add(time.Second)) require.NoError(t, handOffReady(context.Background(), opts)) @@ -91,23 +116,33 @@ func TestARequestMadeTooLongBeforeTheRunIsNotHandedOff(t *testing.T) { assert.Equal(t, ReasonBeforeThisRun, r.Reason) } -func TestARequestMadeWhileNoConnectorRanIsHandedOffWhenOneStarts(t *testing.T) { - for _, down := range []time.Duration{2 * time.Minute, time.Hour, HandoffGrace} { - t.Run(down.String(), func(t *testing.T) { +func TestARequestAnEarlierRunAdmittedIsHandedOffOnlyWithinTheGrace(t *testing.T) { + for _, tc := range []struct { + before time.Duration + handed bool + }{ + {HandoffGrace, true}, + {HandoffGrace + time.Second, false}, + {time.Hour, false}, + } { + t.Run(tc.before.String(), func(t *testing.T) { ledger := newTestLedger(t) - admitOn(t, ledger, 1, "recording:1") var out bytes.Buffer opts := handoffOptions(ledger, &out) - opts.Started = testEvent(1).CreatedAt.Add(down) + opts.Started = testEvent(1).CreatedAt.Add(tc.before) + admitAt(t, ledger, 1, testEvent(1).CreatedAt.Add(time.Second)) require.NoError(t, handOffReady(context.Background(), opts)) - lines := handedOffLines(t, &out) - require.Len(t, lines, 1) - assert.Equal(t, int64(1), lines[0].EventID) r := getRecord(t, ledger, 1) + if tc.handed { + require.Len(t, handedOffLines(t, &out), 1) + assert.Equal(t, ReasonHandedOff, r.Reason) + return + } + assert.Empty(t, out.String()) assert.Equal(t, StateDiscarded, r.State) - assert.Equal(t, ReasonHandedOff, r.Reason) + assert.Equal(t, ReasonBeforeThisRun, r.Reason) }) } } diff --git a/internal/connector/skill_layout_test.go b/internal/connector/skill_layout_test.go index 053d467e0..1c4d51232 100644 --- a/internal/connector/skill_layout_test.go +++ b/internal/connector/skill_layout_test.go @@ -110,22 +110,21 @@ func TestTheConnectSkillStartsTheConnectorAgainAfterAMemoryStop(t *testing.T) { assert.Contains(t, section, "stopped because the system is running low on memory") assert.Contains(t, section, "So start it again at once, as in step 1") - assert.Contains(t, section, "Do the same for the watcher when the notice names it: start it again on the same file.") + assert.Contains(t, section, "When such a notice names the watcher, start it again on the running connector's current output file.") + assert.Contains(t, section, "don't start one on the old file.") } func TestTheConnectSkillStatesHowFarBackARestartCatchesUp(t *testing.T) { skill := flatText(readConnectSkill(t, "SKILL.md")) - window := fmt.Sprintf("%d hours", int(HandoffGrace/time.Hour)) + window := fmt.Sprintf("%d hours", int(handoffCatchUp/time.Hour)) for _, phrase := range []string{ - "restarted within " + window + ".", "it hands off the requests made while it was stopped, up to " + window + " before it started.", "hands off requests made up to " + window + " before it started", "dropped if that's more than " + window + " later.", } { assert.Contains(t, skill, phrase) } - assert.NotContains(t, skill, "within a minute") assert.NotContains(t, skill, "not picked up later") assert.NotContains(t, skill, "starts from the present") } diff --git a/skills/basecamp-connect/SKILL.md b/skills/basecamp-connect/SKILL.md index 2d467d062..b0a4163a9 100644 --- a/skills/basecamp-connect/SKILL.md +++ b/skills/basecamp-connect/SKILL.md @@ -142,7 +142,7 @@ is not their reply. Every line has already passed the trust check: the person may give the agent work, and the project is one it serves. A request arrives once, except in one case: if the connector crashes right after printing a line, it can print it -again when restarted within 4 hours. Skip an `event_id` you've already handled. Every other +again when restarted within a minute. Skip an `event_id` you've already handled. Every other line (no `type`, or `"type":"event"`) is the connector's own log of what it saw and decided, including what it turned away. Never act on those. @@ -541,8 +541,10 @@ asked you to keep the connector running when they started it. So start it again at once, as in step 1, check the old output file and start the watcher on the new one as the end of this section says, and tell the person it happened. A command started again soon after such a stop isn't stopped again until someone -types in the session. Do the same for the watcher when the notice names it: -start it again on the same file. +types in the session. When such a notice names the watcher, start it again on +the running connector's current output file. If the connector was stopped too, +starting it again already started a new watcher: don't start one on the old +file. Otherwise, read the end of its output and tell the person what it means: From 2f621038465b9f9671a14670017513721024fbcc Mon Sep 17 00:00:00 2001 From: Rob Zolkos Date: Tue, 20 Oct 2026 13:57:42 -0400 Subject: [PATCH 3/3] Say what to do when the watcher's memory-stop notice comes first --- internal/connector/skill_layout_test.go | 4 ++-- skills/basecamp-connect/SKILL.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/connector/skill_layout_test.go b/internal/connector/skill_layout_test.go index 1c4d51232..338018943 100644 --- a/internal/connector/skill_layout_test.go +++ b/internal/connector/skill_layout_test.go @@ -110,8 +110,8 @@ func TestTheConnectSkillStartsTheConnectorAgainAfterAMemoryStop(t *testing.T) { assert.Contains(t, section, "stopped because the system is running low on memory") assert.Contains(t, section, "So start it again at once, as in step 1") - assert.Contains(t, section, "When such a notice names the watcher, start it again on the running connector's current output file.") - assert.Contains(t, section, "don't start one on the old file.") + assert.Contains(t, section, "When such a notice names the watcher, check the connector first. If it was stopped too, start it again as above, which starts a new watcher, and don't start one on the old file.") + assert.Contains(t, section, "If it's still running, start the watcher again on its current output file.") } func TestTheConnectSkillStatesHowFarBackARestartCatchesUp(t *testing.T) { diff --git a/skills/basecamp-connect/SKILL.md b/skills/basecamp-connect/SKILL.md index b0a4163a9..dfaa3b971 100644 --- a/skills/basecamp-connect/SKILL.md +++ b/skills/basecamp-connect/SKILL.md @@ -541,10 +541,10 @@ asked you to keep the connector running when they started it. So start it again at once, as in step 1, check the old output file and start the watcher on the new one as the end of this section says, and tell the person it happened. A command started again soon after such a stop isn't stopped again until someone -types in the session. When such a notice names the watcher, start it again on -the running connector's current output file. If the connector was stopped too, -starting it again already started a new watcher: don't start one on the old -file. +types in the session. When such a notice names the watcher, check the +connector first. If it was stopped too, start it again as above, which starts a +new watcher, and don't start one on the old file. If it's still running, start +the watcher again on its current output file. Otherwise, read the end of its output and tell the person what it means: