Per-kind ids: schema 3 and docket migrate

CHANGELOG.md

@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- **Breaking:** schema 3 numbers each kind on its own, so claims are `c1…`, decisions `d1…` and questions `q1…`. Every record in an existing ledger is renumbered once, retired records included, and keeps its schema 2 ID in `migrated_from`. Old IDs stay searchable and no command resolves them. Every collaborator must upgrade docket and the plugin before merging a migrated ledger, because 0.25.x cannot read schema 3 and its session hook reports an error. Each project runs `docket migrate` once and commits the result.
+- Each branch migrates itself before it merges. The merge driver refuses a ledger or features side that is still at an older schema, names the branch to migrate, and leaves conflict markers; abort the merge, migrate that branch, commit, and merge again.
+- A merge or `docket rebase` now rewrites `c`, `d` and `q` IDs mentioned in a record's prose along with its relations, and matching ignores those IDs so a record merged under a new number is not appended twice.
+- `docket migrate` runs every step from the ledger's schema to the current one, so a schema 1 ledger reaches schema 3 in one run. It prints a summary by default and, with `--dry-run`, the per-record report, every prose field it would rewrite and every ID it would leave as written. It keeps the original at `ledger.jsonl.schemaN`, where N is the starting schema. `--map` and `--emit-map` apply to schema 1 only.
+- The migration rewrites `.docket/features.jsonl` and `.docket/archive/features-*.jsonl` to the new IDs and moves their schema to 2.
+- `docket check` compares ID order per kind, and on a ledger older than the running docket prints the migrate instruction once.
+- The pre-edit hook asks before an edit to any `ledger.jsonl.schemaN` backup, not only `.schema1`.
+
+### Added
+
+- `docket migrate --rewrite FILE...` rewrites ID citations in the named files in the same run. Each file must be tracked by git and have no uncommitted changes. With `--dry-run` it lists the files and counts and writes nothing. It is refused on a ledger already at schema 3.
+- `docket list --where was:ID` finds a record by the ID it had before a migration, and `docket show ID` names that command for an ID that no longer exists.
+- `list --json` and `show --json` drop `migrated_from` along with `legacy` unless `--legacy` is passed.
+- The session hook prints the migrate instruction on stdout, with exit 0, when the ledger is older than the running docket, so the agent sees it. Any other ledger error still prints on stderr with exit 1.
+
 ## [0.25.1] - 2026-10-05
 
 ### Fixed

README.md

@@ -176,3 +176,4 @@ records, reports what was omitted, and includes a command to retrieve more.
 **Upgrading from before 0.8?** The ledger format and commands changed.
 Follow the [migration guide](docs/ledger.md#migrating-a-schema-1-ledger)
 before using an existing ledger.
+Moving from 0.25 or earlier to per-kind ids? See [Schema versions](docs/schemas.md).

docket/cli/__init__.py

@@ -197,7 +197,7 @@ def main(argv: list[str] | None = None) -> int:
         help="with --json, keep only these fields, e.g. id,kind,state,text",
     )
     ls.add_argument(
-        "--legacy", action="store_true", help="with --json, keep the schema 1 migration audit field"
+        "--legacy", action="store_true", help="with --json, keep the migration audit fields"
     )
     ls.add_argument("--plain", action="store_true", help="force colour off")
     ls.add_argument("--pretty", action="store_true", help="force colour on, e.g. piping to less -R")
@@ -225,7 +225,7 @@ def main(argv: list[str] | None = None) -> int:
     sh.add_argument("id")
     sh.add_argument("--json", action="store_true", help="print the entry as JSON")
     sh.add_argument(
-        "--legacy", action="store_true", help="with --json, keep the schema 1 migration audit field"
+        "--legacy", action="store_true", help="with --json, keep the migration audit fields"
     )
     sh.add_argument("--at", default="", help="print the record as history stood at this record ID")
     sh.set_defaults(func=cmd_show)
@@ -286,11 +286,22 @@ def main(argv: list[str] | None = None) -> int:
     md.add_argument("theirs")
     md.set_defaults(func=cmd_merge_driver)
 
-    mg = sub.add_parser("migrate", help="convert a legacy ledger to the current schema")
+    mg = sub.add_parser("migrate", help="convert a ledger to the current schema")
     source = mg.add_mutually_exclusive_group()
     source.add_argument("--map", help="classification map to apply instead of the derived one")
     source.add_argument("--emit-map", help="write the derived map to this path and stop")
-    mg.add_argument("--dry-run", action="store_true", help="report the conversion and stop")
+    mg.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="print the per-record report, every prose change and unmapped id, and write nothing",
+    )
+    mg.add_argument(
+        "--rewrite",
+        nargs="+",
+        default=[],
+        metavar="FILE",
+        help="also rewrite id citations in these tracked, clean files; only alongside a migration",
+    )
     mg.set_defaults(func=cmd_migrate)
 
     it = sub.add_parser("init", help="move this project's ledger into the repository")

docket/cli/admin.py

@@ -13,13 +13,16 @@
 from pathlib import Path
 
 from docket import env, feature_archive, feature_project, features, merge_setup
+from docket.cli.context_cmd import migrate_instruction
 from docket.ledger import (
     ID_RE,
     LedgerError,
+    SchemaTooOld,
     _ledger_lock,
     _Prefix,
     append,
     rebind,
+    stale_keys,
     validate_record,
 )
 
@@ -38,7 +41,8 @@ def cmd_rebase(args: argparse.Namespace) -> int:
         theirs = env.read(other, strict=True)
         tail, mapping = renumber(mine, theirs)
     except (LedgerError, RebaseError, OSError) as exc:
-        print(f"docket: {exc}", file=sys.stderr)
+        # Ledger errors, SchemaTooOld among them, already carry the prefix.
+        print(f"docket: {str(exc).removeprefix('docket: ')}", file=sys.stderr)
         return 1
     if not tail:
         print("docket: nothing to rebase; the histories already agree")
@@ -68,9 +72,14 @@ def cmd_rebase(args: argparse.Namespace) -> int:
     return 0
 
 
+def _count(number: int, noun: str) -> str:
+    return f"{number} {noun}{'' if number == 1 else 's'}"
+
+
 def cmd_migrate(args: argparse.Namespace) -> int:
     """Convert this project's ledger to the current schema."""
     from docket.migrate import (
+        SCHEMA_LATEST,
         MigrationError,
         derive_mapping,
         detect_version,
@@ -82,13 +91,19 @@ def cmd_migrate(args: argparse.Namespace) -> int:
     if not path.exists():
         print(f"docket: no ledger at {path}")
         return 0
+    version = None
     try:
+        version = detect_version(read_source(path))
         if args.emit_map:
-            source = read_source(path)
-            if detect_version(source) == 2:
-                print("docket: already schema 2")
+            if version == SCHEMA_LATEST:
+                print(f"docket: already schema {SCHEMA_LATEST}")
                 return 0
-            mapping = derive_mapping(source)
+            if version != 1:
+                raise MigrationError(
+                    f"--emit-map applies to a schema 1 ledger; this one is at schema {version}, "
+                    "whose records are already typed"
+                )
+            mapping = derive_mapping(read_source(path))
             Path(args.emit_map).write_text(
                 json.dumps(mapping, indent=2, sort_keys=True) + "\n", encoding="utf-8"
             )
@@ -97,32 +112,70 @@ def cmd_migrate(args: argparse.Namespace) -> int:
                 f"{'y' if len(mapping) == 1 else 'ies'} to {args.emit_map}"
             )
             return 0
-        count, report, notes = migrate_in_place(path, mapping_path=args.map, dry_run=args.dry_run)
+        result = migrate_in_place(
+            path, mapping_path=args.map, dry_run=args.dry_run, rewrite=args.rewrite
+        )
     except MigrationError as exc:
         print(f"docket: {exc}", file=sys.stderr)
-        print(
-            "docket: to classify records by hand, run 'docket migrate --emit-map FILE', "
-            "edit FILE, then 'docket migrate --map FILE'",
-            file=sys.stderr,
-        )
+        if version == 1:
+            print(
+                "docket: to classify records by hand, run 'docket migrate --emit-map FILE', "
+                "edit FILE, then 'docket migrate --map FILE'",
+                file=sys.stderr,
+            )
         return 2
     except OSError as exc:
         print(f"docket: {exc}", file=sys.stderr)
         return 1
-    if not count:
-        print("docket: already schema 2")
+    if not result.count:
+        # A current ledger whose features files were not.
+        extra = (
+            f"; remapped {_count(result.features_events, 'features event')}"
+            if result.features_events
+            else ""
+        )
+        print(f"docket: already schema {SCHEMA_LATEST}{extra}")
         return 0
-    for note in notes:
+    for note in result.notes:
         print(f"docket: warning: {note}", file=sys.stderr)
-    for line in report:
-        print(line)
-    if args.dry_run:
-        print(f"docket: would convert {count} record{'' if count == 1 else 's'}")
-        return 0
+    dry = args.dry_run
+    if dry:
+        for line in result.report:
+            print(line)
+        for change in result.prose_changes:
+            if change.after is None:
+                print(f"  {change.line_id} {change.field}: unmapped {change.before}")
+            else:
+                print(f"  {change.line_id} {change.field}: {change.before} -> {change.after}")
+    kinds = ", ".join(
+        _count(result.per_kind.get(kind, 0), kind) for kind in ("claim", "decision", "question")
+    )
     print(
-        f"docket: converted {count} record{'' if count == 1 else 's'}; "
-        f"original kept at {path}.schema1"
+        f"docket: {'would migrate' if dry else 'migrated'} {_count(result.count, 'record')} "
+        f"from schema {version} to schema {SCHEMA_LATEST} ({kinds})"
     )
+    rewritten = sum(1 for change in result.prose_changes if change.after is not None)
+    unmapped = len(result.prose_changes) - rewritten
+    done = "would rewrite" if dry else "rewrote"
+    parts = [_count(rewritten, "prose field")]
+    if result.features_events:
+        parts.append(_count(result.features_events, "features event"))
+    print(f"docket: {done} {' and '.join(parts)}")
+    if unmapped:
+        print(f"docket: left {_count(unmapped, 'unmapped id')} as written")
+    if result.rewritten:
+        total = sum(result.rewrite_counts.values())
+        print(
+            f"docket: {done} {_count(total, 'id mention')} in "
+            f"{_count(len(result.rewritten), 'file')}"
+        )
+        for name in result.rewritten:
+            if dry:
+                print(f"  {name}: {_count(result.rewrite_counts[name], 'id')}")
+            else:
+                print(f"  {name}")
+    if not dry:
+        print(f"docket: original kept at {path}.schema{version}")
     return 0
 
 
@@ -149,7 +202,7 @@ def cmd_check(args: argparse.Namespace) -> int:
     # already shed.
     prefix = _Prefix([])
     seen: dict[str, int] = {}
-    highest = 0
+    highest: dict[str, int] = {}
     records = 0
     correction_lines = 0
     review_lines = 0
@@ -170,6 +223,9 @@ def cmd_check(args: argparse.Namespace) -> int:
             # nothing a brief can attach, and the feature check reports it.
             try:
                 checked = validate_record(record, prefix=prefix)
+            except SchemaTooOld as exc:
+                print(migrate_instruction(exc), end="")
+                return 1
             except LedgerError as exc:
                 faults.append(f"line {number}: {str(exc).removeprefix('docket: ')}")
             else:
@@ -185,17 +241,22 @@ def cmd_check(args: argparse.Namespace) -> int:
             faults.append(f"line {number}: duplicate id {ident}, first seen on line {seen[ident]}")
             continue
         seen[ident] = number
+        letter = match.group(1)
+        past = highest.get(letter, 0)
         sequence = int(match.group(2))
-        if sequence <= highest:
-            faults.append(f"line {number}: id {ident} does not increase past {highest}")
+        if sequence <= past:
+            faults.append(f"line {number}: id {ident} does not increase past {letter}{past}")
             continue
-        highest = sequence
+        highest[letter] = sequence
         # The ID checks alone let a hand-resolved merge pass: a record pointing
         # at a same-numbered record from the other branch has a target that
         # exists and has the right kind. validate_record is what catches a
         # dangling or wrong-kind reference.
         try:
             checked = validate_record(record, prefix=prefix)
+        except SchemaTooOld as exc:
+            print(migrate_instruction(exc), end="")
+            return 1
         except LedgerError as exc:
             faults.append(f"line {number}: {str(exc).removeprefix('docket: ')}")
         else:
@@ -249,6 +310,18 @@ def cmd_check(args: argparse.Namespace) -> int:
                             f"renumbered ({', '.join(moved)}); run docket feature remap"
                         )
                         failed = True
+                    stale = (
+                        []
+                        if feature["state"] in features.TERMINAL_STATES
+                        else stale_keys(feature[field], feature["keys"], good)
+                    )
+                    if stale:
+                        print(
+                            f"{store}: {feature['id']} {field} names {', '.join(stale)}, "
+                            "whose stored key matches no record; check which record was "
+                            "meant and re-cite it with docket feature amend"
+                        )
+                        failed = True
 
     for fault in feature_archive.faults(store):
         print(fault)

docket/cli/context_cmd.py

@@ -17,7 +17,7 @@
 from docket.cli.autoscope import auto_scope_files
 from docket.context_model import positions
 from docket.env import read
-from docket.ledger import LedgerError, project
+from docket.ledger import SCHEMA, LedgerError, SchemaTooOld, project
 
 # Harnesses that want context as their own hook envelope instead of plain
 # text, keyed by the --for value. docs/installation.md is the source for
@@ -123,6 +123,17 @@ def _feature_block(root, entries=None) -> str:
         return ""
 
 
+def migrate_instruction(exc: SchemaTooOld) -> str:
+    command = ROOT / "bin" / "docket"
+    return (
+        f"docket: this project's ledger is schema {exc.version} and this docket reads "
+        f"schema {SCHEMA}. Run `{command} migrate` to convert it. If any files cite "
+        "ledger ids, add `--rewrite FILE...` to rewrite them in the same run. "
+        "Then commit ledger.jsonl and the features files, but not the ledger.jsonl.schema* "
+        "backup.\n"
+    )
+
+
 def cmd_context(args: argparse.Namespace) -> int:
     from docket.config import ConfigError
     from docket.config import load as load_settings
@@ -146,6 +157,8 @@ def cmd_context(args: argparse.Namespace) -> int:
 
         try:
             raw = read(env.ledger_path())
+        except SchemaTooOld as exc:
+            return _print_context(migrate_instruction(exc), args, line)
         except (LedgerError, OSError) as exc:
             print(str(exc), file=sys.stderr)
             return 1
@@ -198,10 +211,12 @@ def cmd_context(args: argparse.Namespace) -> int:
             command=str(ROOT / "bin" / "docket"),
             explain=args.explain,
         )
+    except SchemaTooOld as exc:
+        return _print_context(migrate_instruction(exc), args, line)
     except (LedgerError, OSError) as exc:
         print(str(exc), file=sys.stderr)
         return 1
     return _print_context(text, args, line)
 
 
-__all__ = ["CONTEXT_ENVELOPES", "cmd_context", "update_line"]
+__all__ = ["CONTEXT_ENVELOPES", "cmd_context", "migrate_instruction", "update_line"]

docket/cli/graph.py

@@ -332,10 +332,11 @@ def walk(eid: str, ancestor_last: list[bool]) -> None:
 
 def _find_or_alloc(columns: list[str | None], label: str) -> int:
     """A column labelled `label`, reusing the leftmost freed one if none
-    exists. The ledger's ids are monotonic and append-only, so processing
-    newest-first means every column we need has either already been opened by
-    a dependent, or never existed; there is no need for git's full graph
-    algorithm to find it."""
+    exists. The ledger is append-only and a record only cites earlier lines, so
+    processing newest-first in file order means every column we need has either
+    already been opened by a dependent, or never existed; there is no need for
+    git's full graph algorithm to find it. File position carries this, not the
+    id: ids count per kind and say nothing about order across kinds."""
     if label in columns:
         return columns.index(label)
     for i, c in enumerate(columns):

docket/cli/query.py

@@ -53,11 +53,11 @@ def _list_dim_tail(line: str, marker: str, use_color: bool) -> str:
 JSON_FIELDS = tuple(sorted(ledger.ALLOWED_FIELDS - ledger.AUDIT_FIELDS | _DERIVED_FIELDS))
 
 
-def _without_legacy(entry: dict, keep: bool) -> dict:
-    """Schema 1 migration audit blobs are about a tenth of a ledger's JSON and nothing reads them."""
-    if keep or "legacy" not in entry:
+def _without_audit(entry: dict, keep: bool) -> dict:
+    """Migration audit fields (legacy alone is about a tenth of a ledger's JSON) are read by nothing but --legacy."""
+    if keep or ledger.AUDIT_FIELDS.isdisjoint(entry):
         return entry
-    return {k: v for k, v in entry.items() if k != "legacy"}
+    return {k: v for k, v in entry.items() if k not in ledger.AUDIT_FIELDS}
 
 
 def fields_arg(value: str) -> list[str]:
@@ -93,7 +93,7 @@ def cmd_list(args: argparse.Namespace) -> int:
         print("docket: nothing recorded")
         return 0
     if getattr(args, "json", False):
-        rows = [_without_legacy(e, args.legacy) for e in entries]
+        rows = [_without_audit(e, args.legacy) for e in entries]
         if args.fields:
             rows = [{k: e[k] for k in args.fields if k in e} for e in rows]
         print(json.dumps(rows, ensure_ascii=False, indent=2))
@@ -160,10 +160,18 @@ def cmd_show(args: argparse.Namespace) -> int:
     e = by_id.get(args.id)
     if not e:
         print(f"docket: no entry {args.id}", file=sys.stderr)
+        # A current id never reaches this branch, so an old id that is also a
+        # new id resolves above and stays silent.
+        if any(args.id in where.old_ids(item) for item in entries):
+            print(
+                f"docket: {args.id} was renumbered by a migration; "
+                f"find it with: docket list --where was:{args.id}",
+                file=sys.stderr,
+            )
         return 1
 
     if args.json:
-        print(json.dumps(_without_legacy(e, getattr(args, "legacy", False)), indent=2))
+        print(json.dumps(_without_audit(e, getattr(args, "legacy", False)), indent=2))
         return 0
 
     def cite(i: str) -> str: