From bcf6423ebc8effa160c6136d63854be29bff32a6 Mon Sep 17 00:00:00 2001 From: norci <2986988+norci@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:14:55 +0800 Subject: [PATCH 1/4] Charge measured context changes by character difference --- AUTOLITH.org | 51 +++-- src/agent/context.lisp | 51 +++-- src/agent/runtime.lisp | 13 +- src/conversation/store.lisp | 66 ++----- src/provider/context-admission.lisp | 270 +++++++++++++------------- tests/agent-tests.lisp | 91 ++++++++- tests/context-admission-tests.lisp | 285 ++++++++++++++++++---------- tests/test-suites.lisp | 1 + 8 files changed, 481 insertions(+), 347 deletions(-) diff --git a/AUTOLITH.org b/AUTOLITH.org index 9ff558496..d4e357546 100644 --- a/AUTOLITH.org +++ b/AUTOLITH.org @@ -37,38 +37,37 @@ a user-local init.lisp change, or a private image commit. - Problem: a changed prompt fingerprint discarded valid measured history after restart, inflating estimates for serialized encrypted reasoning. -- Behavior: validate retained history and provider/model/reasoning/endpoint - identity, then charge changed prompt and tool components separately. Store - fingerprint-only format-two evidence; require exact headers for legacy samples. +- Behavior: a sample stays usable while its policy (class, model, model-provider, + reasoning effort, endpoint) and its measured history prefix are unchanged. + Prompt, tool header, retained history additions and request-local input are + charged as signed character differences against the measured request, so a + rewritten covered prefix is the only rejection. - Scope and location: tracked provider admission, agent context and conversation - storage in =e8a8b625= and =73babdcc=. Nine functions and their declarations - were installed live and persisted in private image commit - =b1620f5e-2077-4652-9d9e-c5f8aefb446e=, private Git =f56a6ae9=; - papercut =Q3TfMSy=. -- Verification: 24 focused cases with 248 checks passed. Real active traffic - produced valid format-two evidence. Active checks covered persistence reload, - history and policy validation, component charging, admission and tamper - rejection. Clean-process private replay passed. -- Removal condition: discard the private accounting definitions after booting - the equivalent tracked implementation. + storage in =src/provider/context-admission.lisp=, =src/agent/context.lisp= and + =src/conversation/store.lisp=. Samples store format-three evidence; + conversation storage validates that shape only, and prefix digests decide + usability. +- Verification: focused cases in =tests/context-admission-tests.lisp= cover + component charging, request-local differences, policy invalidation, replaced + history and wire-field compatibility; =./script/check= runs the + repository check. +- Removal condition: none. The tracked implementation owns the accounting and + supersedes private image commit =b1620f5e-2077-4652-9d9e-c5f8aefb446e=. ** Measured context through failed compaction - Problem: side-channel compaction metering erased the ordinary context sample even when compaction failed without replacing retained history. -- Behavior: retain the ordinary sample during compaction requests; invalidate it - when publishing replacement history or changing ownership or configuration. -- Scope and location: tracked =src/conversation/store.lisp= in =d0085f7c=; - the single invalidator definition was installed live and persisted in private - image commit =ad5fa7b4-6e5a-4830-ba2f-b114f8137a24=, private Git =dce50286=; - papercut =H4RdvUE=. -- Verification: source checks, 24 context-admission cases with 254 checks and - the rejected-summary case with four checks passed. Historical replay retained - the original valid sample through the failed attempt and subsequent paste, - with identical retained history. Active boundary assertions and clean-process - private replay passed; running and selected private commits are synchronized. -- Removal condition: omit the private invalidator after booting the equivalent - tracked implementation. +- Behavior: the ordinary sample survives compaction requests. A published + summary or native compaction replaces the covered history prefix, and the + prefix digest then rejects the sample on the next request; a configuration or + ownership change rejects it through the policy and conversation identity. +- Scope and location: tracked =src/conversation/store.lisp= and + =src/provider/context-admission.lisp=; conversation storage validates sample + shape only and admission decides usability. +- Verification: the context-admission cases covering summary publication, + native compaction metadata and configuration invalidation. +- Removal condition: none. ** Default child-agent depth diff --git a/src/agent/context.lisp b/src/agent/context.lisp index 8149066fb..104524c07 100644 --- a/src/agent/context.lisp +++ b/src/agent/context.lisp @@ -544,31 +544,42 @@ stays cached across midnight; this separate block carries the change instead." (configuration conversation &key tool-registry hurry-up-p family provider) "Return shared input accounting and whether no unmeasured history was added. -A matched measurement already includes request-local guidance. Charge only -new retained history here; admission replaces temporary context when encoding." - (let* ((tools (if tool-registry (tool-registry-provider-schemas tool-registry) #())) - (items (conversation-input-items-for-family - conversation - (or family (model-family (config :model configuration) - (config :model-provider configuration))))) - (sample (conversation-context-usage-sample conversation)) - (components (and provider (provider-context--header-components provider tools)))) - (when (and provider - (null (conversation-ephemeral-input-entries conversation)) +A matched measurement already includes request-local guidance. Charge new +retained history, prompt and tool changes, and request-local character changes +here; admission charges the encoded request instead." + (let* ((tools (if tool-registry (tool-registry-provider-schemas tool-registry) #())) + (family (or family (model-family (config :model configuration) + (config :model-provider configuration)))) + (items (conversation-input-items-for-family + conversation family :include-ephemeral-p nil)) + (sample (conversation-context-usage-sample conversation)) + (policy (and provider (provider-context--request-policy provider)))) + (when (and policy (provider-context--sample-matches-p - sample (provider-context--soft-header provider tools) - (mapcar #'provider-cache-prefix--entry items) :components components)) - (let* ((new (nthcdr (getf sample :history-count) items)) - (charge (if (= 2 (getf sample :format)) - (provider-context--header-charge sample components) 0))) - (return-from context-estimate-known-tokens - (values (+ charge (provider-context--input-tokens sample (and new (coerce new 'vector)))) - (and (null new) (zerop charge)))))) + sample policy (mapcar #'provider-cache-prefix--entry items))) + (let* ((count (getf sample :history-count)) + (added (when (< count (length items)) (subseq items count))) + (ephemera (- (provider-context--ephemera-characters conversation) + (getf sample :ephemera-characters 0))) + (header (provider-context--header-charge + (getf sample :chargeables) + (provider-context--request-chargeables provider tools)))) + (multiple-value-bind (characters) + (if added + (provider-context--request-characters (coerce added 'vector)) + (values 0)) + (return-from context-estimate-known-tokens + (values + (provider-context--input-tokens + sample nil + :additional-characters (+ characters ephemera header)) + (and (null added) (zerop ephemera) (zerop header))))))) (values (provider-context--input-tokens nil (vector (system-prompt configuration) (request-context-session-state configuration :hurry-up-p hurry-up-p) - (coerce items 'vector) tools)) + (coerce (conversation-input-items-for-family conversation family) 'vector) + tools)) nil))) (-> context--render (list) (option string)) diff --git a/src/agent/runtime.lisp b/src/agent/runtime.lisp index 816a03abc..8b4f4da87 100644 --- a/src/agent/runtime.lisp +++ b/src/agent/runtime.lisp @@ -726,13 +726,12 @@ retries stay auditable next to the request they belong to." ':durable)))) (conversation-append-provider-metadata conversation - (append (list :request-number request-number - :context-usage-pending t - :response-id (provider-result-response-id result) - :usage (agent--portable-value - (provider-usage-normalize - (provider-result-usage result))) - :request-identity (conversation-request-identity conversation)) + (append (list :request-number request-number + :response-id (provider-result-response-id result) + :usage (agent--portable-value + (provider-usage-normalize + (provider-result-usage result))) + :request-identity (conversation-request-identity conversation)) ;; The service's effective retention mode, when it reports one. (let ((context (agent--result-reasoning-context result))) (and context (list :reasoning-context context)))))) diff --git a/src/conversation/store.lisp b/src/conversation/store.lisp index 6d8c4b7ae..fa84ef908 100644 --- a/src/conversation/store.lisp +++ b/src/conversation/store.lisp @@ -3128,31 +3128,6 @@ later picker searches read it without scanning the log." (:method ((kind t) conversation properties) nil)) -(-> conversation--invalidate-context-usage (conversation keyword &optional list) null) -(defun conversation--invalidate-context-usage (conversation kind &optional properties) - "Discard measured context evidence when KIND replaces its coverage. - -PROPERTIES is the durable record property list when available. Provider usage -metadata preserves coverage; published compaction, ownership changes and pending -ordinary responses replace it." - (let ((metadata (getf properties :metadata))) - (when (or (member kind '(:summary :native-compaction :context-adoption - :configuration :precompaction)) - (and (eq kind :provider) - (or (getf metadata :context-adoption) - (getf metadata :context-usage-pending)))) - (setf (conversation-context-usage-sample conversation) nil))) - nil) - -(defmethod conversation-append-record :after ((conversation conversation) (record list)) - "Invalidate measured coverage consistently for live appends and durable replay." - (conversation--invalidate-context-usage conversation (first record) (rest record))) - -(defmethod conversation--project-record :before (kind conversation properties) - "Invalidate earlier measured coverage before projecting a durable change." - (conversation--invalidate-context-usage conversation kind properties) - nil) - (-> conversation--context-fingerprint-p (t) boolean) (defun conversation--context-fingerprint-p (value) "Recognize a portable SHA-256 digest and serialized character count." @@ -3165,37 +3140,26 @@ ordinary responses replace it." (defun conversation-context-usage-sample-p (sample) "Validate bounded fingerprint-only measured context evidence loaded from storage." (and (proper-list-p sample) - (evenp (length sample)) + (= 18 (length sample)) (loop for tail on sample by (partial #'nthcdr 2) always (keywordp (first tail))) - (case (getf sample :format) - (1 - (member (length sample) '(16 18))) - (2 - (and (= 26 (length sample)) - (every (lambda (key) (get-properties sample (list key))) - '(:format :soft-header :policy :prompt :tool-header - :history-count :history-digest :wire-fields :encoder-fields - :wire-count :wire-digest :tokens :temporary-characters)) - (every #'conversation--context-fingerprint-p - (list (getf sample :policy) (getf sample :prompt) - (getf sample :tool-header)))))) - (typep (getf sample :temporary-characters 0) '(integer 0)) + (eql 3 (getf sample :format)) (typep (getf sample :tokens) '(integer 1)) (typep (getf sample :history-count) '(integer 0)) - (typep (getf sample :wire-count) '(integer 0)) + (typep (getf sample :wire-characters) '(integer 0)) + (typep (getf sample :ephemera-characters) '(integer 0)) (every #'conversation--context-fingerprint-p - (list (getf sample :soft-header) (getf sample :history-digest) - (getf sample :wire-digest))) - (every (lambda (fields) - (and (proper-list-p fields) - (every (lambda (field) - (and (consp field) (stringp (first field)) - (conversation--context-fingerprint-p (rest field)))) - fields))) - (if (= 2 (getf sample :format)) - (list (getf sample :wire-fields) (getf sample :encoder-fields)) - (list (getf sample :wire-fields)))) + (list (getf sample :policy) (getf sample :history-digest))) + (proper-list-p (getf sample :chargeables)) + (every (lambda (component) + (and (consp component) (keywordp (first component)) + (conversation--context-fingerprint-p (rest component)))) + (getf sample :chargeables)) + (proper-list-p (getf sample :wire-fields)) + (every (lambda (field) + (and (consp field) (stringp (first field)) + (conversation--context-fingerprint-p (rest field)))) + (getf sample :wire-fields)) t)) (defmethod conversation--project-record diff --git a/src/provider/context-admission.lisp b/src/provider/context-admission.lisp index 6171e4f3c..f6090e624 100644 --- a/src/provider/context-admission.lisp +++ b/src/provider/context-admission.lisp @@ -43,37 +43,61 @@ may use provider token counts when available; the default is not a tokenizer.")) (values tokens uncertain-p) (provider-context--input-tokens nil request)))) -(-> provider-context--soft-header (model-provider vector) (option list)) -(defun provider-context--soft-header (provider tools) - "Fingerprint prompt policy and tools when the provider owns a configuration." +(-> provider-context--request-policy (model-provider) (option list)) +(defun provider-context--request-policy (provider) + "Fingerprint the provider identity a measured sample belongs to. + +A provider without a configuration method has no identity to anchor a +measurement on, so its requests use the unmeasured fallback." (let ((configuration (and (compute-applicable-methods #'provider-configuration (list provider)) (provider-configuration provider)))) (when configuration (provider-cache-prefix--entry (vector (symbol-name (class-name (class-of provider))) (config :model configuration) (config :model-provider configuration) - (config :reasoning-effort configuration) (config :provider-endpoint configuration) - (system-prompt configuration) tools))))) + (config :reasoning-effort configuration) (config :provider-endpoint configuration)))))) + +(-> provider-context--request-chargeables (model-provider vector) (option list)) +(defun provider-context--request-chargeables (provider tools) + "Fingerprint and count the configuration components charged by character difference. -(-> provider-context--header-components (model-provider vector) list) -(defun provider-context--header-components (provider tools) - "Separate stable accounting policy from replaceable prompt and tool fingerprints." +Each component is a (key digest characters) triple, the same shape a transported +wire field carries, so one accountant prices both sources." (let ((configuration (and (compute-applicable-methods #'provider-configuration (list provider)) (provider-configuration provider)))) (when configuration - (list :policy (provider-cache-prefix--entry - (vector (symbol-name (class-name (class-of provider))) - (config :model configuration) (config :model-provider configuration) - (config :reasoning-effort configuration) (config :provider-endpoint configuration))) - :prompt (provider-cache-prefix--entry (system-prompt configuration)) - :tool-header (provider-cache-prefix--entry tools))))) - -(-> provider-context--header-charge (list list) (integer 0)) -(defun provider-context--header-charge (sample components) - "Charge full current changed components without subtracting unmeasured old costs." - (loop for key in '(:prompt :tool-header) - unless (equal (getf sample key) (getf components key)) - sum (ceiling (second (getf components key)) 4))) + (loop for key in '(:prompt :tools) + for value = (if (eq key ':prompt) (system-prompt configuration) tools) + collect (cons key (provider-cache-prefix--entry value)))))) + +(-> provider-context--header-charge (list list) integer) +(defun provider-context--header-charge (measured current) + "Sum the signed character difference of every component either list carries. + +Each component is a (key digest characters) triple. A component absent from +CURRENT is credited in full, so removing measured prompt or tool characters +lowers the charge. A component absent from MEASURED is not priced: the estimate +path derives exactly the components it charges, and admission rejects a wire +field the transport never observed." + (loop for key in (remove-duplicates (append (mapcar #'first measured) + (mapcar #'first current)) + :test #'equal) + for before = (assoc key measured :test #'equal) + for after = (assoc key current :test #'equal) + when before + sum (- (if after (third after) 0) (third before)))) + +(-> provider-context--ephemera-characters (conversation) (integer 0)) +(defun provider-context--ephemera-characters (conversation) + "Count the request-local input characters currently attached to CONVERSATION." + (let ((entries (conversation-ephemeral-input-entries conversation))) + (if entries + (multiple-value-bind (characters uncertain-p) + (provider-context--request-characters + (coerce (mapcar (lambda (entry) (getf entry :item)) entries) 'vector)) + (declare (ignore uncertain-p)) + characters) + 0))) (-> provider-context--history (model-provider conversation) list) (defun provider-context--history (provider conversation) @@ -90,16 +114,19 @@ may use provider token counts when available; the default is not a tokenizer.")) (-> provider-context-request-create (model-provider conversation vector) list) (defun provider-context-request-create (provider conversation tools) - "Start ordinary request-local evidence; ephemera require the unmeasured fallback." - (let ((header (provider-context--soft-header provider tools))) - ;; Preallocate mutable fields to preserve the shared evidence plist head. - (append (and header (provider-context--header-components provider tools)) - (list :provider provider :conversation conversation - :history (and header (provider-context--history provider conversation)) - :soft-header header - :eligible-p (and header (null (conversation-ephemeral-input-entries conversation)) t) - :encoder nil :projection nil :tools nil :wire-provider nil :wire nil - :wire-characters 0)))) + "Start request-local evidence for measured usage accounting." + (let ((policy (provider-context--request-policy provider))) + (list :provider provider + :conversation conversation + :history (and policy (provider-context--history provider conversation)) + :policy policy + :chargeables (and policy (provider-context--request-chargeables provider tools)) + :ephemera-characters (provider-context--ephemera-characters conversation) + :encoder nil + :projection nil + :tools nil + :wire-provider nil + :wire nil))) (-> provider-context--wire-body (hash-table) hash-table) (defun provider-context--wire-body (request) @@ -140,16 +167,23 @@ may use provider token counts when available; the default is not a tokenizer.")) (provider-cache-prefix--entry (coerce (subseq entries 0 count) 'vector))) t)) -(-> provider-context--sample-matches-p (list (option list) list &key (:components list)) boolean) -(defun provider-context--sample-matches-p (sample header history &key components) - "Verify measured history under exact headers or a format-two stable policy." +(-> provider-context--sample-matches-p + (list (option list) list &key (:wire-fields (option list))) boolean) +(defun provider-context--sample-matches-p (sample policy history &key wire-fields) + "Verify a measured SAMPLE against the current request POLICY and HISTORY. + +The sample stays usable only while the provider identity is unchanged and +HISTORY still starts with the measured prefix. A changed prompt or tool +component does not void it; the shared accountant charges the difference. When +the caller supplies transported wire fields, they may only name fields the +sample observed." (and (conversation-context-usage-sample-p sample) - header - (or (equal (getf sample :soft-header) header) - (and (= 2 (getf sample :format)) components - (equal (getf sample :policy) (getf components :policy)))) + policy + (equal (getf sample :policy) policy) (provider-context--prefix-matches-p (getf sample :history-digest) (getf sample :history-count) history) + (or (null wire-fields) + (provider-context--wire-fields-compatible-p wire-fields (getf sample :wire-fields))) t)) (-> provider-context--request-items (hash-table) (option vector)) @@ -162,83 +196,54 @@ may use provider token counts when available; the default is not a tokenizer.")) return items))) (-> provider-context--input-tokens - (list t &key (:replaced-characters (integer 0))) + (list t &key (:additional-characters integer)) (values (integer 0) boolean)) -(defun provider-context--input-tokens (sample additions &key (replaced-characters 0)) +(defun provider-context--input-tokens (sample additions &key (additional-characters 0)) "Return one input count and non-text uncertainty for measured or cold context. -Retain measured occupancy, charge additions, and replace previously measured -temporary context instead of charging its replacement a second time." +Retain measured occupancy and charge the signed character difference between +the measured request and this one, so a shorter request credits its characters +back and the count never falls below zero. A nonzero charge is an estimate +rather than the provider's own count, so it reports uncertainty." (multiple-value-bind (characters uncertain-p) (if additions (provider-context--request-characters additions) (values 0 nil)) - (values (max 0 (+ (getf sample :tokens 0) - (ceiling (- characters - (if (and sample (vectorp additions) - (not (stringp additions))) 2 0) - replaced-characters) - 4))) - uncertain-p))) - -(-> provider-context--current-wire-p (list hash-table list) boolean) -(defun provider-context--current-wire-p (context request sample) - "Reproduce current encoding while retaining measured transport-only fields." - (let ((adapter (getf context :encoder)) - (projection (getf context :projection)) - (tools (getf context :tools))) - (when (and adapter projection tools) - (let* ((*provider-context-request* nil) - (*provider-cache-prefix-function* nil) - (encoded (provider-context--wire-plan - (provider-request-object adapter projection tools))) - (current (provider-context--wire-plan request)) - (transport-fields - (remove-if (lambda (field) - (equal field (assoc (first field) (getf sample :encoder-fields) :test #'equal))) - (getf sample :wire-fields))) - (expected-fields - (append transport-fields - (remove-if (lambda (field) - (assoc (first field) transport-fields :test #'equal)) - (getf encoded :fields))))) - (and (equal (getf encoded :items) (getf current :items)) - (= (length expected-fields) (length (getf current :fields))) - (every (lambda (field) - (equal field (assoc (first field) (getf current :fields) :test #'equal))) - expected-fields) - t))))) + (let ((difference (+ characters additional-characters))) + (values (max 0 (+ (getf sample :tokens 0) (ceiling difference 4))) + (or uncertain-p (not (zerop difference))))))) (-> provider-context--matched-input (model-provider hash-table) (values (integer 0) boolean boolean)) (defun provider-context--matched-input (provider request) - "Revalidate history, encoding and policy before retaining measured occupancy." - (let* ((context *provider-context-request*) - (conversation (getf context :conversation)) - (sample (and conversation (conversation-context-usage-sample conversation))) - (header-changed-p (not (equal (getf sample :soft-header) (getf context :soft-header))))) - (when (and (eq provider (getf context :provider)) (getf context :eligible-p) + "Revalidate coverage and charge the encoded request against the measured one. + +The retained history is re-read here rather than trusted from the captured +snapshot, so a conversation edited after the request object was created cannot +keep measured occupancy. A request whose encoder produced no conversation +sequence cannot be priced against measured items, so it falls back to the cold +estimate instead of matching at zero." + (let* ((context *provider-context-request*) + (conversation (getf context :conversation)) + (sample (and conversation (conversation-context-usage-sample conversation))) + (policy (getf context :policy)) + (current (and sample policy (provider-context--wire-plan request))) + (items (and current (provider-context--request-items request)))) + (when (and items + (eq provider (getf context :provider)) (provider-context--sample-matches-p - sample (getf context :soft-header) (provider-context--history provider conversation) - :components context)) - (let* ((current (provider-context--wire-plan request)) - (items (provider-context--request-items request)) - (fields (getf current :fields)) - (old-fields (getf sample :wire-fields))) - (when (and items - (if header-changed-p - (provider-context--current-wire-p context request sample) - (equal old-fields fields)) - (provider-context--prefix-matches-p - (getf sample :wire-digest) (getf sample :wire-count) (getf current :items))) - (let ((count (getf sample :wire-count))) - (multiple-value-bind (tokens uncertain-p) - (provider-context--input-tokens - sample (unless (= count (length items)) (subseq items count)) - :replaced-characters (getf sample :temporary-characters 0)) - (return-from provider-context--matched-input - (values (+ tokens (if header-changed-p - (provider-context--header-charge sample context) 0)) - t (or header-changed-p uncertain-p)))))))) - (values 0 nil nil))) + sample policy (provider-context--history provider conversation) + :wire-fields (getf current :fields))) + (multiple-value-bind (characters uncertain-p) + (provider-context--request-characters items) + (return-from provider-context--matched-input + (values + (provider-context--input-tokens + sample nil + :additional-characters + (+ (- characters (getf sample :wire-characters 0)) + (provider-context--header-charge + (getf sample :wire-fields) (getf current :fields)))) + t uncertain-p))))) + (values 0 nil nil)) (-> provider-context--response-entries-p (list list) boolean) (defun provider-context--response-entries-p (entries output) @@ -266,8 +271,9 @@ the encoder's own fields must still be present there." "Anchor ordinary usage after persistence, before locally generated tool evidence. Re-encode the measured projection through its original encoder, with its -persisted response and no temporary suffix. Retain fingerprints only. Merged -messages that later change fail closed." +persisted response and no temporary suffix. Retain fingerprints, character +counts and the provider-reported input tokens only. Merged messages that later +change fail closed." (let* ((conversation (getf context :conversation)) (provider (getf context :provider)) (adapter (getf context :encoder)) @@ -277,8 +283,7 @@ messages that later change fail closed." (output (and (json-object-p usage) (json-get usage "output_tokens"))) (total (and (json-object-p usage) (json-get usage "total_tokens")))) (when conversation - (setf (conversation-context-usage-sample conversation) nil) - (when (and (getf context :eligible-p) adapter projection + (when (and (getf context :policy) adapter projection (getf context :wire) (typep input '(integer 1)) (typep output '(integer 0)) (or (plusp output) (null (provider-result-output-items result))) @@ -296,41 +301,28 @@ messages that later change fail closed." (let* ((options (copy-list (cl-llm-provider-api:wire-request-options projection))) (*provider-context-request* nil) (*provider-cache-prefix-function* nil) - (base + (covered (make-instance 'cl-llm-provider-api:wire-request :model (cl-llm-provider-api:wire-request-model projection) :prefix (cl-llm-provider-api:wire-request-prefix projection) - :items (cl-llm-provider-api:wire-request-items projection) + :items (append (cl-llm-provider-api:wire-request-items projection) + (nthcdr (length before) history)) :options options)) - (base-characters - (provider-context--request-characters - (provider-context--request-items - (provider-request-object adapter base (getf context :tools))))) - (covered - (make-instance 'cl-llm-provider-api:wire-request - :model (cl-llm-provider-api:wire-request-model projection) - :prefix (cl-llm-provider-api:wire-request-prefix projection) - :items (append (cl-llm-provider-api:wire-request-items projection) - (nthcdr (length before) history)) - :options options)) - (wire (provider-context--wire-plan - (provider-request-object adapter covered (getf context :tools))))) - (when (provider-context--wire-fields-compatible-p (getf wire :fields) (getf (getf context :wire) :fields)) + (request (provider-request-object adapter covered (getf context :tools))) + (wire (provider-context--wire-plan request))) + (when (provider-context--wire-fields-compatible-p + (getf wire :fields) (getf (getf context :wire) :fields)) (let ((sample - (list :format 2 :soft-header (getf context :soft-header) - :policy (getf context :policy) - :prompt (getf context :prompt) - :tool-header (getf context :tool-header) - :history-count (length entries) - :history-digest (provider-cache-prefix--entry (coerce entries 'vector)) - :wire-fields (getf (getf context :wire) :fields) - :encoder-fields (getf wire :fields) - :wire-count (length (getf wire :items)) - :wire-digest (provider-cache-prefix--entry - (coerce (getf wire :items) 'vector)) - :tokens (max (+ input output) (or total 0)) - :temporary-characters - (max 0 (- (getf context :wire-characters) base-characters))))) + (list :format 3 + :policy (getf context :policy) + :chargeables (getf context :chargeables) + :history-count (length entries) + :history-digest (provider-cache-prefix--entry (coerce entries 'vector)) + :wire-fields (getf (getf context :wire) :fields) + :wire-characters (provider-context--request-characters + (or (provider-context--request-items request) #())) + :ephemera-characters (getf context :ephemera-characters) + :tokens (max (+ input output) (or total 0))))) (conversation-append-record conversation (list :context-usage :sample sample)) (setf (conversation-context-usage-sample conversation) sample))))))))) nil) diff --git a/tests/agent-tests.lisp b/tests/agent-tests.lisp index e72728bb1..b77ba49a5 100644 --- a/tests/agent-tests.lisp +++ b/tests/agent-tests.lisp @@ -2377,6 +2377,84 @@ REASONING-CONTEXT is the retention mode its response metadata reports." "both requests are accounted")))) nil) +(-> test-agent-compaction-missing-summary () null) +(defun test-agent-compaction-missing-summary () + "Test empty compaction output is a recoverable provider protocol failure." + (let* ((configuration (test-configuration)) + (root (test-configuration-root configuration))) + (unwind-protect + (let* ((conversation + (conversation-create configuration + :identifier "agent-empty-compaction")) + (provider + (make-instance + 'scripted-provider + :configuration configuration + :results + (list (agent-test-result "compact-empty" nil + :turn-completion ':end)))) + (agent (agent-create :configuration configuration + :provider provider + :conversation conversation + :tool-registry (agent-test-registry) + :worker nil)) + (sample + (list :format 3 + :policy (list (make-string 64 :initial-element #\a) 0) + :chargeables (list (list ':prompt (make-string 64 :initial-element #\b) 12)) + :history-count 0 + :history-digest (list (make-string 64 :initial-element #\c) 0) + :wire-fields nil + :wire-characters 0 + :ephemera-characters 0 + :tokens 1200)) + (records-before + (progn + (conversation-append-record conversation (list :context-usage :sample sample)) + (setf (conversation-context-usage-sample conversation) sample) + (conversation--read-records (conversation-pathname conversation)))) + (failure + (handler-case + (progn + (agent-compact-conversation + agent (make-instance 'agent-observer)) + nil) + (provider-protocol-error (condition) + condition)))) + (test-assert + (and failure + (string= (autolith-error-message failure) + "Compaction produced no usable summary handoff: the response has no visible text.") + (string= (provider-error-response-id failure) + "compact-empty") + (null (provider-error-status failure)) + (null (provider-error-code failure)) + (null (provider-error-request-id failure)) + (typep failure 'agent-compaction-summary-error) + (let ((response (json-decode (provider-error-response failure)))) + (and (string= "compact-empty" (json-get response "response_id")) + (zerop (length (json-get response "output_items"))) + (equalp (json-get response "usage") + (json-object "input_tokens" 1 "output_tokens" 1)))) + (equal (provider-error-response failure) + (getf (agent--provider-error-metadata failure) :response))) + "empty compaction output is a structured provider protocol error") + (test-assert (equal sample (conversation-context-usage-sample conversation)) + "failed compaction preserves the live ordinary measured anchor") + (test-assert (equal sample (conversation-context-usage-sample + (conversation-load (conversation-pathname conversation)))) + "disk replay preserves the ordinary measured anchor after failed compaction") + (let* ((records-after + (conversation--read-records (conversation-pathname conversation))) + (added (nthcdr (length records-before) records-after))) + (test-assert + (and (= 1 (count ':provider added :key #'first)) + (notany (lambda (record) (member (first record) '(:summary :native-compaction))) added) + (null (conversation-input-items conversation))) + "rejected paid compaction records usage without publishing a checkpoint"))) + (platform-delete-directory-tree *platform* root :validate t :if-does-not-exist ':ignore))) + nil) + (-> test-agent-compaction-failed-generation () null) (defun test-agent-compaction-failed-generation () "Test a generation that fails after it began costs one request and is asked again." @@ -2416,10 +2494,15 @@ REASONING-CONTEXT is the retention mode its response metadata reports." collect (agent-test-result "empty" nil :turn-completion ':end)) (list (agent-test-result "refusal" (list (agent-test-refusal))))))) (sample - (list :format 1 :soft-header (provider-context--soft-header (agent-provider agent) #()) - :history-count 0 :history-digest (provider-cache-prefix--entry #()) - :wire-fields nil :wire-count 0 :wire-digest (provider-cache-prefix--entry #()) - :tokens 1200))) + (list :format 3 + :policy (list (make-string 64 :initial-element #\a) 0) + :chargeables (list (list ':prompt (make-string 64 :initial-element #\b) 12)) + :history-count 0 + :history-digest (list (make-string 64 :initial-element #\c) 0) + :wire-fields nil + :wire-characters 0 + :ephemera-characters 0 + :tokens 1200))) (conversation-append-user-message conversation "first request") (let ((carried (1- (conversation-next-sequence conversation)))) (agent-compact-conversation agent observer) diff --git a/tests/context-admission-tests.lisp b/tests/context-admission-tests.lisp index 208b92c48..63817a01e 100644 --- a/tests/context-admission-tests.lisp +++ b/tests/context-admission-tests.lisp @@ -957,9 +957,13 @@ (request (provider-request-object provider conversation #()))) (test-assert (nth-value 1 (provider-context--matched-input provider request)) "changed policy preserves measured transport fields and overrides") - (setf (gethash "session_id" request) "tampered transport") - (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) - "encoder-invisible transport edits reject the anchor")))))) + (multiple-value-bind (before before-p) (provider-context--matched-input provider request) + (setf (gethash "session_id" request) "tampered transport") + (multiple-value-bind (tokens matched-p) (provider-context--matched-input provider request) + (test-assert (and before-p matched-p) + "encoder-invisible transport edits keep the anchor") + (test-assert (= before tokens) + "fields the encoder never reads do not change the charge")))))))) :provider-class 'context-admission-transport-fields-test-provider) nil) @@ -1006,18 +1010,22 @@ configuration conversation :provider provider)) "the meter counts already measured nonempty guidance once") (setf conversation (conversation-load (conversation-pathname conversation))) - (dolist (entry (list (list guidance #'=) - (list (make-string 1200 :initial-element #\g) #'>) - (list (make-string 400 :initial-element #\g) #'<) - (list nil #'<))) - (let* ((*provider-context-request* - (provider-context-request-create provider conversation #())) - (request (provider-request-object provider conversation #() - :goal-context (first entry)))) - (test-assert (funcall (second entry) - (provider-request-input-token-estimate provider request) - 1200) - "admission replaces measured guidance after reload")))))))) + (flet ((measure (guidance) + (let* ((*provider-context-request* + (provider-context-request-create provider conversation #())) + (request (provider-request-object provider conversation #() + :goal-context guidance))) + (provider-request-input-token-estimate provider request)))) + (let ((exact (measure guidance)) + (long (measure (make-string 1200 :initial-element #\g))) + (short (measure (make-string 400 :initial-element #\g))) + (none (measure nil))) + (test-assert (and (< short exact long) (= 1200 none)) + (format nil "guidance length orders the charge: ~D < ~D < ~D" + short exact long)) + (test-assert (and (= 100 (- exact short)) (= 100 (- long exact))) + (format nil "each 400 guidance characters add 100 tokens: ~D ~D ~D" + short exact long)))))))))n nil) (-> test-context-admission-measured-invalidation () null) @@ -1033,14 +1041,15 @@ (json-object "input_tokens" 1000 "output_tokens" 0) (json-object "input_tokens" 1000 "output_tokens" 200 "total_tokens" 2) (json-object "input_tokens" 1000 "output_tokens" 200 "cached_input_tokens" 2000))) - (context-admission-test--measure agent :envelope-length 64) - (context-admission-test--measure agent :usage usage :envelope-length 64) - (test-assert (null (conversation-context-usage-sample conversation)) - "an incomplete or inconsistent response cannot establish a sample")) + (context-admission-test--measure agent :envelope-length 64) + (let ((before (conversation-context-usage-sample conversation))) + (context-admission-test--measure agent :usage usage :envelope-length 64) + (test-assert (eq before (conversation-context-usage-sample conversation)) + "an incomplete or inconsistent response cannot establish a sample"))) (context-admission-test--measure agent :envelope-length 64) (let* ((unconfigured (make-instance 'model-provider)) (context (provider-context-request-create unconfigured conversation #()))) - (test-assert (null (getf context :eligible-p)) + (test-assert (null (getf context :policy)) "providers without a configuration method use unmeasured admission") (multiple-value-bind (tokens known-p) (context-estimate-known-tokens (agent-configuration agent) conversation @@ -1059,8 +1068,8 @@ (vector (json-object "type" "message" "role" "user" "content" "A changed covered prefix.")) "A changed request field.")) (multiple-value-bind (tokens matched-p) (provider-context--matched-input provider request) - (declare (ignore tokens)) - (test-assert (not matched-p) "changed encoded headers or covered items reject the baseline")))) + (test-assert (and matched-p (plusp tokens)) + "a rewritten request field is charged rather than rejected")))) (let* ((other (make-instance 'context-admission-test-provider :configuration (agent-configuration agent))) (*provider-context-request* (provider-context-request-create other conversation #())) @@ -1074,12 +1083,13 @@ (multiple-value-bind (tokens matched-p) (provider-context--matched-input provider request) (declare (ignore tokens)) (test-assert (not matched-p) "conversations do not share usage anchors"))) - (context-admission-test--measure - agent :envelope-length 64 - :during-request (lambda (conversation) - (conversation-append-user-message conversation "Arrived while awaiting a reply."))) - (test-assert (null (conversation-context-usage-sample conversation)) - "input arriving in flight is not charged to the provider output allowance") + (let ((before (conversation-context-usage-sample conversation))) + (context-admission-test--measure + agent :envelope-length 64 + :during-request (lambda (conversation) + (conversation-append-user-message conversation "Arrived while awaiting a reply."))) + (test-assert (eq before (conversation-context-usage-sample conversation)) + "input arriving in flight is not charged to the provider output allowance")) (context-admission-test--measure agent :envelope-length 64) (conversation-append-provider-metadata conversation '(:usage (("input_tokens" 1200) ("output_tokens" 200)))) (test-assert (conversation-context-usage-sample conversation) @@ -1095,20 +1105,24 @@ (test-assert (equal sample (conversation-context-usage-sample (conversation-load (conversation-pathname conversation)))) "disk replay preserves the ordinary anchor after compaction metering"))) + (context-admission-test--measure agent :envelope-length 64) (conversation-append-summary conversation (make-continuation :intent "A replacement context.")) - (test-assert (null (conversation-context-usage-sample conversation)) "summary publication invalidates the sample") - (test-assert (null (conversation-context-usage-sample - (conversation-load (conversation-pathname conversation)))) - "disk replay invalidates the sample at summary publication") + (test-assert (conversation-context-usage-sample conversation) + "summary publication leaves the anchor to prefix validation") + (let* ((*provider-context-request* (provider-context-request-create provider conversation #())) + (request (provider-request-object provider conversation #()))) + (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) + "a published summary replaces the measured covered prefix")) (context-admission-test--measure agent :envelope-length 64) (conversation-append-native-compaction conversation (list (json-object "type" "compaction" "encrypted_content" "checkpoint")) :family (provider-family provider) :continuation (make-continuation :intent "A native replacement context.")) - (test-assert (null (conversation-context-usage-sample conversation)) - "native checkpoint publication invalidates the live sample") - (test-assert (null (conversation-context-usage-sample - (conversation-load (conversation-pathname conversation)))) - "disk replay invalidates the sample at native checkpoint publication") + (test-assert (conversation-context-usage-sample conversation) + "native checkpoint publication leaves the anchor to prefix validation") + (let* ((*provider-context-request* (provider-context-request-create provider conversation #())) + (request (provider-request-object provider conversation #()))) + (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) + "a native checkpoint replaces the measured covered prefix")) (context-admission-test--measure agent :envelope-length 64) (test-assert (conversation-context-usage-sample (conversation-load (conversation-pathname conversation))) @@ -1138,82 +1152,121 @@ (-> test-context-admission-measured-header-resume () null) (defun test-context-admission-measured-header-resume () - "Keep opaque measured history through disk resume and changed environment policy." + "Charge changed prompt and tool headers over measured occupancy through disk resume." (context-admission-test--with-measured-context (lambda (original) (let ((tools (vector (json-object "type" "function" "name" "lookup" "description" (make-string 2000 :initial-element #\d) "parameters" (json-object "type" "object"))))) (context-admission-test--measure original :tools tools) - (let* ((configuration (agent-configuration original)) - (conversation (conversation-load (conversation-pathname (agent-conversation original)))) - (provider (make-instance 'context-admission-test-provider :configuration configuration)) - (registry (make-instance 'tool-registry)) - (agent (agent-create :configuration configuration :conversation conversation - :provider provider :tool-registry registry)) - (application (make-instance 'application :configuration configuration - :conversation conversation :provider provider - :agent agent :tool-registry registry)) - (prompt "small policy with new working environment")) + (let* ((configuration (agent-configuration original)) + (conversation (conversation-load + (conversation-pathname (agent-conversation original)))) + (provider (make-instance 'context-admission-test-provider + :configuration configuration)) + (registry (make-instance 'tool-registry)) + (agent (agent-create :configuration configuration + :conversation conversation + :provider provider + :tool-registry registry)) + (application (make-instance 'application + :configuration configuration + :conversation conversation + :provider provider + :agent agent + :tool-registry registry)) + (measured-prompt (second (provider-cache-prefix--entry (system-prompt configuration)))) + (prompt "small policy with new working environment") + (charge (- (second (provider-cache-prefix--entry prompt)) measured-prompt)) + (expected (+ 1200 (ceiling charge 4)))) (test-call-with-function-replacements (list (list 'system-prompt - (lambda (configuration) (declare (ignore configuration)) prompt)) + (lambda (configuration) + (declare (ignore configuration)) + prompt)) (list 'tool-registry-provider-schemas (lambda (registry &key canonical-names) - (declare (ignore registry canonical-names)) tools))) + (declare (ignore registry canonical-names)) + tools))) (lambda () - (let* ((charge (ceiling (second (provider-cache-prefix--entry prompt)) 4)) - (expected (+ 1200 charge))) - (multiple-value-bind (tokens exact-p) - (context-estimate-known-tokens configuration conversation - :provider provider :tool-registry registry) - (test-assert (= tokens expected) "only the changed prompt is charged over measured occupancy") - (test-assert (not exact-p) "changed policy is an estimated meter")) - (multiple-value-bind (tokens estimated-p) (application-context-usage application) - (test-assert (and (= expected tokens) estimated-p) - "the resumed UI exposes the conservative estimated occupancy")) - (let* ((*provider-context-request* (provider-context-request-create provider conversation tools)) - (request (provider-request-object provider conversation tools))) - (test-assert (null (getf *provider-context-request* :wire)) - "initial admission has no transport observation") - (multiple-value-bind (tokens uncertain-p) - (provider-request-input-token-estimate provider request) - (test-assert (and (= expected tokens) uncertain-p) - "wire admission retains opaque history before transport observation")) - (provider-open-response-stream provider request :conversation conversation) - (setf (gethash "instructions" request) "Post-encode tamper") - (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) - "changed soft headers do not authorize post-encode edits") - (setf (gethash "instructions" request) "small policy") - (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) - "restoring an old measured header also rejects post-encode tamper"))) + (multiple-value-bind (tokens exact-p) + (context-estimate-known-tokens configuration conversation + :provider provider + :tool-registry registry) + (test-assert (= tokens expected) + "only the changed prompt is charged over measured occupancy") + (test-assert (not exact-p) + "a changed prompt is an estimated meter")) + (multiple-value-bind (tokens estimated-p) + (application-context-usage application) + (test-assert (and (= expected tokens) estimated-p) + "the resumed UI exposes the conservative estimated occupancy")) + (let* ((*provider-context-request* (provider-context-request-create + provider conversation tools)) + (request (provider-request-object + provider conversation tools))) + (test-assert (null (getf *provider-context-request* :wire)) + "initial admission has no transport observation") + (multiple-value-bind (tokens uncertain-p) + (provider-request-input-token-estimate provider request) + (test-assert (and (= expected tokens) uncertain-p) + "wire admission charges the changed prompt before transport observation")) + (provider-open-response-stream provider request :conversation conversation) + (setf (gethash "instructions" request) "Post-encode tamper") + (multiple-value-bind (tokens matched-p) + (provider-context--matched-input provider request) + (test-assert matched-p + "a post-encode edit is charged rather than trusted") + (test-assert (= tokens (+ 1200 (ceiling (- (second (provider-cache-prefix--entry "Post-encode tamper")) + measured-prompt) + 4))) + "the edited instruction characters are charged")) + (setf (gethash "instructions" request) "small policy") + (multiple-value-bind (tokens matched-p) + (provider-context--matched-input provider request) + (test-assert (and matched-p (= tokens 1200)) + "restoring the measured header returns to the measured charge"))) + (setf prompt "small policy") (setf tools #()) - (let* ((*provider-context-request* (provider-context-request-create provider conversation tools)) - (request (provider-request-object provider conversation tools)) - (expected (+ 1200 (ceiling (second (provider-cache-prefix--entry prompt)) 4) - (ceiling (second (provider-cache-prefix--entry tools)) 4)))) - (test-assert (= expected (provider-request-input-token-estimate provider request)) - "tool replacement charges its current size without subtracting unmeasured old cost")) + (let* ((*provider-context-request* (provider-context-request-create + provider conversation tools)) + (request (provider-request-object + provider conversation tools)) + (tokens (provider-request-input-token-estimate provider request)) + (measured-tools (third (assoc "tools" + (getf (conversation-context-usage-sample + conversation) + :wire-fields) + :test #'equal)))) + (test-assert (= tokens (- 1200 (ceiling measured-tools 4))) + (format nil "tool replacement credits the measured tool characters: ~D tokens for ~D measured tool characters" + tokens measured-tools))) (setf prompt (make-string 20000 :initial-element #\p)) - (test-assert (agent-should-compact-p agent) "genuine prompt growth crosses soft admission") - (let* ((*provider-context-request* (provider-context-request-create provider conversation tools)) - (request (provider-request-object provider conversation tools))) + (test-assert (agent-should-compact-p agent) + "genuine prompt growth crosses soft admission") + (let* ((*provider-context-request* (provider-context-request-create + provider conversation tools)) + (request (provider-request-object + provider conversation tools))) (test-assert - (handler-case (progn (provider-open-response-stream provider request :conversation conversation) nil) + (handler-case + (progn + (provider-open-response-stream provider request :conversation conversation) + nil) (provider-context-budget-error () t)) "genuine prompt growth crosses hard admission")))))))) nil) (-> test-context-admission-measured-header-identity () null) (defun test-context-admission-measured-header-identity () - "Reject model changes, modified measured history and malformed format-two persistence." + "Reject model changes, modified measured history and malformed format-three persistence." (context-admission-test--with-measured-context (lambda (agent) (context-admission-test--measure agent) (let* ((conversation (agent-conversation agent)) (provider (agent-provider agent)) (sample (copy-tree (conversation-context-usage-sample conversation)))) - (dolist (key '(:policy :prompt :tool-header :encoder-fields)) + (dolist (key '(:policy :prompt :tools :wire-fields :history-digest)) (let ((malformed (copy-tree sample))) (setf (getf malformed key) '("invalid" 1)) (conversation-append-record conversation (list :context-usage :sample malformed)) @@ -1221,11 +1274,12 @@ (conversation-load (conversation-pathname conversation)))) "malformed component fingerprints are rejected on disk resume"))) (setf (conversation-context-usage-sample conversation) sample) - (let* ((components (provider-context--header-components provider #())) - (history (provider-context--history provider conversation))) - (setf (getf components :policy) (provider-cache-prefix--entry "different model")) + (let ((history (provider-context--history provider conversation))) + (test-assert (provider-context--sample-matches-p + sample (provider-context--request-policy provider) history) + "the measured policy revalidates its own history") (test-assert (not (provider-context--sample-matches-p - sample (provider-cache-prefix--entry "different header") history :components components)) + sample (provider-cache-prefix--entry "different model") history)) "a different model policy cannot retain measured occupancy")) (let ((*provider-context-request* (provider-context-request-create provider conversation #()))) (let ((request (provider-request-object provider conversation #()))) @@ -1235,12 +1289,12 @@ (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) "history is revalidated at admission rather than trusting its captured snapshot"))) (let ((legacy (copy-tree sample))) - (dolist (key '(:policy :prompt :tool-header :encoder-fields)) (remf legacy key)) (setf (getf legacy :format) 1) - (test-assert (conversation-context-usage-sample-p legacy) "legacy exact anchors are readable") + (test-assert (not (conversation-context-usage-sample-p legacy)) + "an older sample format is discarded rather than reinterpreted") (test-assert (not (provider-context--sample-matches-p - legacy (provider-cache-prefix--entry "changed legacy header") nil)) - "legacy samples do not authorize a changed header"))))) + legacy (provider-context--request-policy provider) nil)) + "a discarded sample cannot authorize measured occupancy"))))) nil) (-> test-context-admission-measured-resume () null) @@ -1267,8 +1321,13 @@ (test-assert (= 1200 (provider-request-input-token-estimate provider request)) "the final encoded prefix revalidates the historical measurement") (setf (gethash "instructions" request) "Different wire policy.") - (test-assert (not (nth-value 1 (provider-context--matched-input provider request))) - "restart does not bypass final wire validation")) + (multiple-value-bind (tokens matched-p) (provider-context--matched-input provider request) + (test-assert matched-p + "restart keeps the anchor when only instruction characters change") + (test-assert (= tokens (+ 1200 (ceiling (- (second (provider-cache-prefix--entry "Different wire policy.")) + (second (provider-cache-prefix--entry "small policy"))) + 4))) + "the restarted request is charged for its instruction difference"))) (multiple-value-bind (before estimated-p) (application-context-usage application) (test-assert (not estimated-p) @@ -1293,11 +1352,37 @@ :usage (json-object "input_tokens" 1000 "output_tokens" 200 "total_tokens" 1200)) :request-number 2) - (test-assert (null (conversation-context-usage-sample - (conversation-load (conversation-pathname conversation)))) - "a response interrupted before publishing coverage invalidates the older anchor") + (test-assert (conversation-context-usage-sample + (conversation-load (conversation-pathname conversation))) + "a response interrupted before publishing coverage keeps the older anchor") (conversation-append-record conversation '(:context-usage :sample (:format 1))) (test-assert (null (conversation-context-usage-sample (conversation-load (conversation-pathname conversation)))) "malformed coverage is discarded during durable replay")))) nil) + +(-> test-context-admission-measured-request-shape () null) +(defun test-context-admission-measured-request-shape () + "Refuse measured occupancy when the encoder produced no conversation sequence." + (context-admission-test--with-measured-context + (lambda (agent) + (let* ((conversation (agent-conversation agent)) + (provider (agent-provider agent))) + (conversation-append-user-message conversation "Before the measured request.") + (context-admission-test--measure agent) + (let* ((*provider-context-request* (provider-context-request-create provider conversation #())) + (request (provider-request-object provider conversation #())) + (body (provider-context--wire-body request))) + (test-assert (= 1200 (provider-request-input-token-estimate provider request)) + "the measured request is charged while its sequence is present") + (dolist (field *provider-cache-wire-sequence-fields*) + (remhash field body)) + (multiple-value-bind (tokens matched-p) + (provider-context--matched-input provider request) + (test-assert (not matched-p) + "a request without a conversation sequence is not measured occupancy") + (test-assert (zerop tokens) + "an unmatched request reports no measured tokens")) + (test-assert (plusp (provider-request-input-token-estimate provider request)) + "an unrecognized sequence falls back to the full cold estimate"))))) + nil) diff --git a/tests/test-suites.lisp b/tests/test-suites.lisp index 0baceea97..dbea8ad41 100644 --- a/tests/test-suites.lisp +++ b/tests/test-suites.lisp @@ -42,6 +42,7 @@ test-context-admission-measured-resume test-context-admission-measured-header-resume test-context-admission-measured-header-identity + test-context-admission-measured-request-shape test-context-admission-final-request test-context-admission-images test-context-admission-checkpoint-transaction) From 2cba3db504d0cab4aeadc2caab0b94e2dfbffbe7 Mon Sep 17 00:00:00 2001 From: norci <2986988+norci@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:03:26 +0800 Subject: [PATCH 2/4] Resume measured estimates from the last recorded total Evidence written before component charging is no longer usable for pricing prompt and tools, so a reloaded conversation fell back to dividing the stored characters of every retained item by four. That re-priced history the provider had already counted and pushed the estimate over the compaction limit with no new work in the conversation. Keep the measured total and history length beside the sample and start the estimate from them, charging only the history appended after the recorded count. --- AUTOLITH.org | 13 +++++---- src/agent/context.lisp | 32 ++++++++++++++++------ src/conversation/store.lisp | 25 ++++++++++++++++- src/provider/context-admission.lisp | 4 ++- tests/context-admission-tests.lisp | 42 +++++++++++++++++++++++++++++ tests/test-suites.lisp | 1 + 6 files changed, 102 insertions(+), 15 deletions(-) diff --git a/AUTOLITH.org b/AUTOLITH.org index d4e357546..0eb48c5ef 100644 --- a/AUTOLITH.org +++ b/AUTOLITH.org @@ -41,16 +41,19 @@ a user-local init.lisp change, or a private image commit. reasoning effort, endpoint) and its measured history prefix are unchanged. Prompt, tool header, retained history additions and request-local input are charged as signed character differences against the measured request, so a - rewritten covered prefix is the only rejection. + rewritten covered prefix is the only rejection. When no sample is usable, the + estimate starts from the newest recorded measurement and charges only the + history added after it, so evidence of an older shape still bounds a reloaded + conversation. - Scope and location: tracked provider admission, agent context and conversation storage in =src/provider/context-admission.lisp=, =src/agent/context.lisp= and =src/conversation/store.lisp=. Samples store format-three evidence; - conversation storage validates that shape only, and prefix digests decide - usability. + conversation storage validates that shape only and keeps the recorded token + count and history length beside it, and prefix digests decide usability. - Verification: focused cases in =tests/context-admission-tests.lisp= cover component charging, request-local differences, policy invalidation, replaced - history and wire-field compatibility; =./script/check= runs the - repository check. + history, wire-field compatibility and estimates that resume from an older + recorded measurement; =./script/check= runs the repository check. - Removal condition: none. The tracked implementation owns the accounting and supersedes private image commit =b1620f5e-2077-4652-9d9e-c5f8aefb446e=. diff --git a/src/agent/context.lisp b/src/agent/context.lisp index 104524c07..022ae356c 100644 --- a/src/agent/context.lisp +++ b/src/agent/context.lisp @@ -546,7 +546,11 @@ stays cached across midnight; this separate block carries the change instead." A matched measurement already includes request-local guidance. Charge new retained history, prompt and tool changes, and request-local character changes -here; admission charges the encoded request instead." +here; admission charges the encoded request instead. + +Without a usable sample the estimate starts from the newest recorded measurement +and charges only the history added after it, so a conversation reloaded with +evidence of an older shape is not priced as if nothing had been measured." (let* ((tools (if tool-registry (tool-registry-provider-schemas tool-registry) #())) (family (or family (model-family (config :model configuration) (config :model-provider configuration)))) @@ -574,13 +578,25 @@ here; admission charges the encoded request instead." sample nil :additional-characters (+ characters ephemera header)) (and (null added) (zerop ephemera) (zerop header))))))) - (values - (provider-context--input-tokens - nil (vector (system-prompt configuration) - (request-context-session-state configuration :hurry-up-p hurry-up-p) - (coerce (conversation-input-items-for-family conversation family) 'vector) - tools)) - nil))) + (let ((baseline (conversation-context-usage-baseline conversation))) + (when (and baseline (<= (getf baseline :history-count) (length items))) + (return-from context-estimate-known-tokens + (values + (provider-context--input-tokens + baseline nil + :additional-characters + (let ((added (nthcdr (getf baseline :history-count) items))) + (if added + (provider-context--request-characters (coerce added 'vector)) + 0))) + nil))) + (values + (provider-context--input-tokens + nil (vector (system-prompt configuration) + (request-context-session-state configuration :hurry-up-p hurry-up-p) + (coerce (conversation-input-items-for-family conversation family) 'vector) + tools)) + nil)))) (-> context--render (list) (option string)) (defun context--render (contributions) diff --git a/src/conversation/store.lisp b/src/conversation/store.lisp index fa84ef908..a65a70627 100644 --- a/src/conversation/store.lisp +++ b/src/conversation/store.lisp @@ -374,6 +374,14 @@ not yet been told that work from the previous process is gone.") :accessor conversation-context-usage-sample :type list :documentation "Portable ordinary usage evidence, revalidated against retained history and encoded request fingerprints before reuse.") + (context-usage-baseline + :initform nil + :accessor conversation-context-usage-baseline + :type (option list) + :documentation + "The token count and history length of the newest measurement, whatever + +shape its evidence had.") (last-total-tokens :initform 0 :accessor conversation-last-total-tokens @@ -3167,7 +3175,22 @@ later picker searches read it without scanning the log." "Restore portable evidence without treating its historical token count as verified." (let ((sample (getf properties :sample))) (setf (conversation-context-usage-sample conversation) - (when (conversation-context-usage-sample-p sample) sample)))) + (when (conversation-context-usage-sample-p sample) sample) + (conversation-context-usage-baseline conversation) + (conversation--context-usage-baseline sample)))) + +(-> conversation--context-usage-baseline (t) (option list)) +(defun conversation--context-usage-baseline (sample) + "Return the token count and history length SAMPLE recorded, whatever its shape. + +Evidence of an older shape no longer prices components, but the total it measured +still bounds the estimate a cold conversation would otherwise compute from the +stored characters of every retained item." + (when (and (proper-list-p sample) (evenp (length sample))) + (let ((tokens (getf sample :tokens)) + (count (getf sample :history-count))) + (when (and (typep tokens '(integer 1)) (typep count '(integer 0))) + (list :tokens tokens :history-count count))))) (defmethod conversation--project-record ((kind (eql :message)) conversation properties) diff --git a/src/provider/context-admission.lisp b/src/provider/context-admission.lisp index f6090e624..b1a947400 100644 --- a/src/provider/context-admission.lisp +++ b/src/provider/context-admission.lisp @@ -324,7 +324,9 @@ change fail closed." :ephemera-characters (getf context :ephemera-characters) :tokens (max (+ input output) (or total 0))))) (conversation-append-record conversation (list :context-usage :sample sample)) - (setf (conversation-context-usage-sample conversation) sample))))))))) + (setf (conversation-context-usage-sample conversation) sample + (conversation-context-usage-baseline conversation) + (conversation--context-usage-baseline sample)))))))))) nil) (-> provider-context--request-characters (t) (values (integer 0) boolean)) diff --git a/tests/context-admission-tests.lisp b/tests/context-admission-tests.lisp index 63817a01e..eea37b587 100644 --- a/tests/context-admission-tests.lisp +++ b/tests/context-admission-tests.lisp @@ -1386,3 +1386,45 @@ (test-assert (plusp (provider-request-input-token-estimate provider request)) "an unrecognized sequence falls back to the full cold estimate"))))) nil) + +(-> test-context-admission-measured-baseline () null) +(defun test-context-admission-measured-baseline () + "Price a reloaded conversation from its last measurement, whatever shape it has." + (context-admission-test--with-measured-context + (lambda (agent) + (let* ((configuration (agent-configuration agent)) + (conversation (agent-conversation agent)) + (provider (agent-provider agent)) + (registry (agent-tool-registry agent)) + (family (model-family (config :model configuration) + (config :model-provider configuration)))) + (conversation-append-user-message conversation "Older measurement.") + (conversation-append-tool-result conversation "baseline-output" :tool-name "test" + :output (make-string 20000 :initial-element (code-char 102)) + :success-p t) + (let* ((first (conversation-load (conversation-pathname conversation))) + (cold (context-estimate-known-tokens configuration first + :provider provider :tool-registry registry)) + (count (length (conversation-input-items-for-family + first family :include-ephemeral-p nil)))) + (conversation-append-record + first + (list :context-usage :sample (list :format 2 :tokens 1200 :history-count count))) + (let* ((reloaded (conversation-load (conversation-pathname conversation))) + (warm (context-estimate-known-tokens configuration reloaded + :provider provider :tool-registry registry))) + (test-assert (null (conversation-context-usage-sample reloaded)) + "an older evidence shape is not reusable for component charging") + (test-assert (equal (list :tokens 1200 :history-count count) + (conversation-context-usage-baseline reloaded)) + "the reloaded conversation keeps the measured total and history length") + (test-assert (= 1200 warm) + "the estimate starts from the measured total rather than stored characters") + (test-assert (< warm cold) + "the stored characters alone would price the request higher") + (conversation-append-user-message reloaded (make-string 400 :initial-element (code-char 103))) + (test-assert (> (context-estimate-known-tokens configuration reloaded + :provider provider :tool-registry registry) + warm) + "history appended after the measurement is charged")))))) + nil) diff --git a/tests/test-suites.lisp b/tests/test-suites.lisp index dbea8ad41..45d62b616 100644 --- a/tests/test-suites.lisp +++ b/tests/test-suites.lisp @@ -43,6 +43,7 @@ test-context-admission-measured-header-resume test-context-admission-measured-header-identity test-context-admission-measured-request-shape + test-context-admission-measured-baseline test-context-admission-final-request test-context-admission-images test-context-admission-checkpoint-transaction) From 7343759ee2fb20d60954863ac3d37043f5e4f124 Mon Sep 17 00:00:00 2001 From: norci <2986988+norci@users.noreply.github.com> Date: Sun, 15 Nov 2026 20:35:06 +0800 Subject: [PATCH 3/4] Tie recorded context totals to the tokenizer that measured them --- AUTOLITH.org | 6 +++--- src/agent/context.lisp | 4 +++- src/conversation/store.lisp | 13 +++++++----- tests/context-admission-tests.lisp | 32 +++++++++++++++++++++++------- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/AUTOLITH.org b/AUTOLITH.org index 0eb48c5ef..2e144be6d 100644 --- a/AUTOLITH.org +++ b/AUTOLITH.org @@ -42,9 +42,9 @@ a user-local init.lisp change, or a private image commit. Prompt, tool header, retained history additions and request-local input are charged as signed character differences against the measured request, so a rewritten covered prefix is the only rejection. When no sample is usable, the - estimate starts from the newest recorded measurement and charges only the - history added after it, so evidence of an older shape still bounds a reloaded - conversation. + estimate starts from the newest recorded measurement taken under the same + policy and charges only the history added after it, so evidence of an older + shape still bounds a reloaded conversation. - Scope and location: tracked provider admission, agent context and conversation storage in =src/provider/context-admission.lisp=, =src/agent/context.lisp= and =src/conversation/store.lisp=. Samples store format-three evidence; diff --git a/src/agent/context.lisp b/src/agent/context.lisp index 022ae356c..e258772a0 100644 --- a/src/agent/context.lisp +++ b/src/agent/context.lisp @@ -579,7 +579,9 @@ evidence of an older shape is not priced as if nothing had been measured." :additional-characters (+ characters ephemera header)) (and (null added) (zerop ephemera) (zerop header))))))) (let ((baseline (conversation-context-usage-baseline conversation))) - (when (and baseline (<= (getf baseline :history-count) (length items))) + (when (and baseline + (equal (getf baseline :policy) policy) + (<= (getf baseline :history-count) (length items))) (return-from context-estimate-known-tokens (values (provider-context--input-tokens diff --git a/src/conversation/store.lisp b/src/conversation/store.lisp index a65a70627..b8e264284 100644 --- a/src/conversation/store.lisp +++ b/src/conversation/store.lisp @@ -3181,16 +3181,19 @@ later picker searches read it without scanning the log." (-> conversation--context-usage-baseline (t) (option list)) (defun conversation--context-usage-baseline (sample) - "Return the token count and history length SAMPLE recorded, whatever its shape. + "Return the recorded total, history length and tokenizer identity of SAMPLE. Evidence of an older shape no longer prices components, but the total it measured still bounds the estimate a cold conversation would otherwise compute from the -stored characters of every retained item." +stored characters of every retained item. The policy fingerprint keeps that +total from being reused under a different tokenizer." (when (and (proper-list-p sample) (evenp (length sample))) (let ((tokens (getf sample :tokens)) - (count (getf sample :history-count))) - (when (and (typep tokens '(integer 1)) (typep count '(integer 0))) - (list :tokens tokens :history-count count))))) + (count (getf sample :history-count)) + (policy (getf sample :policy))) + (when (and (typep tokens '(integer 1)) (typep count '(integer 0)) + (conversation--context-fingerprint-p policy)) + (list :tokens tokens :history-count count :policy policy))))) (defmethod conversation--project-record ((kind (eql :message)) conversation properties) diff --git a/tests/context-admission-tests.lisp b/tests/context-admission-tests.lisp index eea37b587..37c0162f0 100644 --- a/tests/context-admission-tests.lisp +++ b/tests/context-admission-tests.lisp @@ -1025,7 +1025,7 @@ short exact long)) (test-assert (and (= 100 (- exact short)) (= 100 (- long exact))) (format nil "each 400 guidance characters add 100 tokens: ~D ~D ~D" - short exact long)))))))))n + short exact long))))))))) nil) (-> test-context-admission-measured-invalidation () null) @@ -1407,17 +1407,20 @@ :provider provider :tool-registry registry)) (count (length (conversation-input-items-for-family first family :include-ephemeral-p nil)))) - (conversation-append-record - first - (list :context-usage :sample (list :format 2 :tokens 1200 :history-count count))) + (conversation-append-record + first + (list :context-usage + :sample (list :format 2 :policy (provider-context--request-policy provider) + :tokens 1200 :history-count count))) (let* ((reloaded (conversation-load (conversation-pathname conversation))) (warm (context-estimate-known-tokens configuration reloaded :provider provider :tool-registry registry))) (test-assert (null (conversation-context-usage-sample reloaded)) "an older evidence shape is not reusable for component charging") - (test-assert (equal (list :tokens 1200 :history-count count) + (test-assert (equal (list :tokens 1200 :history-count count + :policy (provider-context--request-policy provider)) (conversation-context-usage-baseline reloaded)) - "the reloaded conversation keeps the measured total and history length") + "the reloaded conversation keeps the measured total and tokenizer") (test-assert (= 1200 warm) "the estimate starts from the measured total rather than stored characters") (test-assert (< warm cold) @@ -1426,5 +1429,20 @@ (test-assert (> (context-estimate-known-tokens configuration reloaded :provider provider :tool-registry registry) warm) - "history appended after the measurement is charged")))))) + "history appended after the measurement is charged") + (conversation-append-record + reloaded + (list :context-usage + :sample (list :format 2 :tokens 1200 :history-count count + :policy (list (make-string 64 :initial-element (code-char 97)) 0)))) + (let ((foreign (conversation-load (conversation-pathname conversation)))) + (let ((foreign-tokens (context-estimate-known-tokens + configuration foreign + :provider provider :tool-registry registry))) + (setf (conversation-context-usage-baseline foreign) nil) + (test-assert (= foreign-tokens + (context-estimate-known-tokens + configuration foreign + :provider provider :tool-registry registry)) + "a total measured under another tokenizer is not reused")))))))) nil) From f357b5b11c013b5f31cfd5d8282f442e20d44319 Mon Sep 17 00:00:00 2001 From: norci <2986988+norci@users.noreply.github.com> Date: Sun, 15 Nov 2026 22:56:45 +0800 Subject: [PATCH 4/4] Keep measured context across reasoning effort switches --- AUTOLITH.org | 6 +++--- src/provider/context-admission.lisp | 6 ++++-- tests/context-admission-tests.lisp | 22 ++++++++++++++++++++++ tests/test-suites.lisp | 1 + 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/AUTOLITH.org b/AUTOLITH.org index 2e144be6d..1a2d5bb3f 100644 --- a/AUTOLITH.org +++ b/AUTOLITH.org @@ -38,7 +38,7 @@ a user-local init.lisp change, or a private image commit. - Problem: a changed prompt fingerprint discarded valid measured history after restart, inflating estimates for serialized encrypted reasoning. - Behavior: a sample stays usable while its policy (class, model, model-provider, - reasoning effort, endpoint) and its measured history prefix are unchanged. + endpoint) and its measured history prefix are unchanged. Prompt, tool header, retained history additions and request-local input are charged as signed character differences against the measured request, so a rewritten covered prefix is the only rejection. When no sample is usable, the @@ -51,7 +51,7 @@ a user-local init.lisp change, or a private image commit. conversation storage validates that shape only and keeps the recorded token count and history length beside it, and prefix digests decide usability. - Verification: focused cases in =tests/context-admission-tests.lisp= cover - component charging, request-local differences, policy invalidation, replaced + component charging, request-local differences, policy invalidation, effort switches that keep the anchor, replaced history, wire-field compatibility and estimates that resume from an older recorded measurement; =./script/check= runs the repository check. - Removal condition: none. The tracked implementation owns the accounting and @@ -398,7 +398,7 @@ a user-local init.lisp change, or a private image commit. ** Lower-cost delegated inference - Problem: General task children and unqualified RLM calls inherited the primary - model and reasoning effort, consuming the user's subscription allowance on + model and consuming the user's subscription allowance on routine delegated work. - Behavior: Top-level RLM calls default to =gpt-5.6-luna= at =low= effort. Explicit model/effort arguments override those defaults; supplied Lisp diff --git a/src/provider/context-admission.lisp b/src/provider/context-admission.lisp index b1a947400..be355683b 100644 --- a/src/provider/context-admission.lisp +++ b/src/provider/context-admission.lisp @@ -47,7 +47,9 @@ may use provider token counts when available; the default is not a tokenizer.")) (defun provider-context--request-policy (provider) "Fingerprint the provider identity a measured sample belongs to. -A provider without a configuration method has no identity to anchor a +Reasoning effort selects how much the model may think and changes neither the +encoder nor the tokenizer, so a measurement survives an effort switch. A +provider without a configuration method has no identity to anchor a measurement on, so its requests use the unmeasured fallback." (let ((configuration (and (compute-applicable-methods #'provider-configuration (list provider)) (provider-configuration provider)))) @@ -55,7 +57,7 @@ measurement on, so its requests use the unmeasured fallback." (provider-cache-prefix--entry (vector (symbol-name (class-name (class-of provider))) (config :model configuration) (config :model-provider configuration) - (config :reasoning-effort configuration) (config :provider-endpoint configuration)))))) + (config :provider-endpoint configuration)))))) (-> provider-context--request-chargeables (model-provider vector) (option list)) (defun provider-context--request-chargeables (provider tools) diff --git a/tests/context-admission-tests.lisp b/tests/context-admission-tests.lisp index 37c0162f0..04670a498 100644 --- a/tests/context-admission-tests.lisp +++ b/tests/context-admission-tests.lisp @@ -1446,3 +1446,25 @@ :provider provider :tool-registry registry)) "a total measured under another tokenizer is not reused")))))))) nil) + +(-> test-context-admission-measured-effort () null) +(defun test-context-admission-measured-effort () + "Keep a measured anchor when only the reasoning effort changes." + (context-admission-test--with-measured-context + (lambda (agent) + (let ((conversation (agent-conversation agent)) + (provider (agent-provider agent))) + (conversation-append-user-message conversation "Before the measured request.") + (context-admission-test--measure agent) + (let* ((configuration (configuration-copy (provider-configuration provider) + :reasoning-effort "low")) + (switched (make-instance (class-of provider) :configuration configuration)) + (context (provider-context-request-create switched conversation #())) + (*provider-context-request* context) + (request (provider-request-object switched conversation #()))) + (multiple-value-bind (tokens matched-p) + (provider-context--matched-input switched request) + (test-assert (and matched-p (= 1200 tokens)) + (format nil "an effort change keeps the anchor: ~D tokens, matched ~A" + tokens matched-p)))))) + :provider-class 'context-admission-test-provider)) diff --git a/tests/test-suites.lisp b/tests/test-suites.lisp index 45d62b616..ec67be414 100644 --- a/tests/test-suites.lisp +++ b/tests/test-suites.lisp @@ -44,6 +44,7 @@ test-context-admission-measured-header-identity test-context-admission-measured-request-shape test-context-admission-measured-baseline + test-context-admission-measured-effort test-context-admission-final-request test-context-admission-images test-context-admission-checkpoint-transaction)