Repository navigation
Expand file tree
/
Copy pathverify_range.rs
More file actions
1256 lines (1218 loc) · 56.5 KB
/
Copy pathverify_range.rs
File metadata and controls
1256 lines (1218 loc) · 56.5 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
//! Verify a range of blocks against rskj's own state, reporting every
//! divergence rather than stopping at the first.
//!
//! A syncing node is the wrong instrument for this. It executes forward from
//! its own post-state, so the first mismatch both halts it and poisons every
//! block after; finding the second costs a fix, a rebuild and a restart.
//!
//! This takes each block's pre-state from **rskj's** database — the parent
//! header's state root, resolved in rskj's unitrie — so blocks are verified
//! independently. A wrong block says nothing about the next one, and a single
//! pass over a fork boundary reports everything that disagrees.
//!
//! Nothing is written to either database. Blocks are read from a rustock store
//! opened read-only, state from rskj's unitrie opened read-only, and the trie
//! the replay computes goes to a scratch store that can be deleted afterwards.
//!
//! ```text
//! verify_range <data-dir> <rskj-unitrie-dir> <from> <to> [options]
//!
//! --network-id N 30 mainnet, 31 testnet (default), else regtest
//! --scratch DIR where the replay's own trie writes go (default: a temp dir)
//! --max-failures N stop after this many (default 10; 0 means never)
//! --cache-mb N block cache for the rskj unitrie (default 256)
//! --explain on a state-root divergence, print the leaves that
//! differ from rskj's own post-state, then stop
//! --state-format F how to read <rskj-unitrie-dir>: "rskj" (default, a
//! flat RocksDB keyed by node hash) or "rustock" (one of
//! our own trie stores, e.g. a segbuild chunk's `sealed`
//! directory, which keys nodes in a `trie_nodes` CF)
//! --state-roots DIR rskj's `stateRoots` table, which maps a pre-RSKIP126
//! header's ORCHID state root to the UNITRIE root holding
//! the same state. It is what makes the early chain
//! reachable at all: it supplies both the parent to read
//! and the root to verify against, with no Orchid
//! conversion anywhere.
//! --parent-roots F CSV of `block,...,unitrie_root,...` giving the UNITRIE
//! root each block computed. Needed below wasabi100,
//! where the header's own stateRoot is an orchid root and
//! cannot address a unitrie node.
//! --execution-threads N
//! threads for RSKIP-144 parallel sublists (default 1)
//! --sequential execute the range in order, carrying each block's
//! post-state forward instead of resolving every block's
//! parent root from the source. The only way to reach
//! blocks below wasabi100 (#1,591,000 on mainnet), whose
//! headers carry an ORCHID-format state root that is not
//! a unitrie node hash and so can never be looked up.
//! --seed-genesis build the genesis state into the scratch store before
//! starting, so a contiguous replay from #1 needs no
//! historical state source at all
//! --blocks FILE verify only the block numbers in FILE (one per line)
//! instead of a contiguous range; <from> and <to> are
//! then omitted
//! --record-silent DIR
//! also record the Bridge decisions that emit no event
//! into a silent-decision store at DIR (created if
//! absent, appended to otherwise)
//! --record-legacy-events DIR
//! also reconstruct, into a legacy-event store at DIR,
//! the peg events the Bridge of that height did not log
//! -- `pegin_btc`, `release_requested`, `rejected_pegin`,
//! `unrefundable_pegin` -- in the same key and value
//! layout as the node's own `bridge_events` index, so
//! one reader spans both. Nothing is written for a
//! height where the Bridge really logged, so the union
//! of the two never double-counts. Read it back with
//! the `legacy_events` tool.
//! ```
//!
//! Both recording options ride on the same drained decisions, so asking for
//! both costs one replay rather than two.
//!
//! `--record-silent` makes the pass pay for itself twice. Verifying a range
//! already executes every block against rskj's own state; recording while it
//! does means a question like "was a peg-in ever refunded here" is answered
//! from an index afterwards instead of by replaying the range again.
//!
//! Only blocks that verified contribute records. A block whose state root
//! disagrees with the chain was executed by this implementation but not by
//! the network, so its decisions are this implementation's, and storing them
//! beside real ones would make the store untrustworthy exactly where it
//! matters. Such blocks become recorded gaps instead.
//!
//! The stop exists because one defect usually breaks every block it touches.
//! Ten instances of a rule that is wrong at every block say exactly what one
//! says, so running the remaining fifty thousand buys nothing and costs an
//! hour. The same goes for a range the snapshot cannot serve: if the first
//! blocks have no pre-state, none of them will.
#[path = "support/mod.rs"]
mod support;
use alloy_primitives::B256;
use anyhow::{Context, Result};
use rocksdb::{DB, Options};
use rustock_core::Block;
use rustock_execution::BlockProcessor;
use rustock_storage::BlockStore;
use rustock_trie::{OverlayTrieStore, TrieNode, TrieStore};
use std::sync::Arc;
/// Closes an open reconstructed-event coverage span at the last good block.
///
/// Mirrors `close_run` for the silent store, and for the same reason. A span
/// is a claim that every block in it was examined; a block whose decisions
/// were dropped -- an execution error, an unresolvable parent, an orchid
/// comparison that yielded no verdict -- breaks that claim. Without this the
/// span would run straight over the hole, and a later `covers()` would answer
/// "yes, examined" for a block whose peg-ins are missing. That is worse than
/// an admitted gap: it reads as authoritative.
#[allow(clippy::too_many_arguments)]
/// Renders the labelled skip counts as a JSON object for the coverage record.
fn render_skips(m: &std::collections::BTreeMap<String, u64>) -> String {
if m.is_empty() {
return "{}".to_string();
}
let body: Vec<String> = m.iter().map(|(k, v)| format!("\"{k}\":{v}")).collect();
format!("{{{}}}", body.join(","))
}
#[allow(clippy::too_many_arguments)]
fn close_legacy_run(
legacy: &Option<rustock_storage::legacy_event_store::LegacyEventStore>,
start: &mut Option<u64>,
last_good: u64,
build: &str,
blocks: u64,
verified: u64,
events: u64,
skipped: u64,
skip_detail: &std::collections::BTreeMap<String, u64>,
) -> anyhow::Result<()> {
let (Some(store), Some(from)) = (legacy, *start) else {
*start = None;
return Ok(());
};
if last_good >= from {
store.add_coverage(&rustock_storage::legacy_event_store::Coverage {
from,
to: last_good,
build: build.to_string(),
blocks,
verified,
events,
skipped,
skip_detail: render_skips(skip_detail),
})?;
}
*start = None;
Ok(())
}
/// How often the reconstructed-event coverage span is rewritten. Keyed on
/// the span start, so each write replaces the last rather than accumulating.
const LEGACY_COVERAGE_EVERY: u64 = 50_000;
const USAGE: &str = "usage: verify_range <data-dir> <rskj-unitrie-dir> <from> <to> \
[--network-id N] [--scratch DIR] [--max-failures N] [--cache-mb N] \
[--blocks FILE] [--record-silent DIR] [--record-legacy-events DIR]";
/// Splits `--key value` and `--key=value` options out of the command line,
/// leaving the positional arguments.
///
/// Hand-rolled because these diagnostics take no dependency on an argument
/// parser, and an unknown option must be an error rather than be silently
/// read as a positional -- a mistyped `--max-failures` that became a block
/// number would be worse than no option at all.
fn split_options(argv: Vec<String>) -> Result<(Vec<String>, Vec<(String, String)>)> {
const WITH_VALUE: [&str; 11] = [
"--record-legacy-events",
"--network-id",
"--scratch",
"--max-failures",
"--cache-mb",
"--record-silent",
"--blocks",
"--state-format",
"--parent-roots",
"--state-roots",
"--execution-threads",
];
const SEED: [&str; 2] = ["--seed-genesis", "--sequential"];
const FLAGS: [&str; 2] = ["--explain", "--read-only"];
let mut positional = Vec::new();
let mut options = Vec::new();
let mut it = argv.into_iter();
while let Some(a) = it.next() {
if !a.starts_with("--") {
positional.push(a);
continue;
}
let (key, inline) = match a.split_once('=') {
Some((k, v)) => (k.to_string(), Some(v.to_string())),
None => (a.clone(), None),
};
if FLAGS.contains(&key.as_str()) || SEED.contains(&key.as_str()) {
options.push((key, "true".to_string()));
continue;
}
anyhow::ensure!(WITH_VALUE.contains(&key.as_str()), "unknown option {key}\n{USAGE}");
let value = match inline {
Some(v) => v,
None => it.next().with_context(|| format!("{key} needs a value"))?,
};
options.push((key, value));
}
Ok((positional, options))
}
fn option<T: std::str::FromStr>(
options: &[(String, String)],
key: &str,
default: T,
) -> Result<T>
where
T::Err: std::fmt::Display,
{
match options.iter().rev().find(|(k, _)| k == key) {
Some((_, v)) => v
.parse()
.map_err(|e| anyhow::anyhow!("{key}: {e}")),
None => Ok(default),
}
}
/// rskj's unitrie, read-only. Content-addressed exactly as rustock's is.
struct RskjUnitrie {
db: DB,
}
impl TrieStore for RskjUnitrie {
fn get(&self, key: &[u8]) -> Option<Vec<u8>> {
self.db.get(key).ok().flatten()
}
fn put(&self, _key: &[u8], _value: &[u8]) {
panic!("the rskj source must never be written to");
}
}
/// What went wrong with one block, if anything.
#[derive(Default)]
struct Verdict {
state: bool,
receipts: bool,
gas: bool,
}
impl Verdict {
fn ok(&self) -> bool {
!self.state && !self.receipts && !self.gas
}
fn label(&self) -> String {
let mut parts = Vec::new();
if self.state {
parts.push("state root");
}
if self.receipts {
parts.push("receipts root");
}
if self.gas {
parts.push("gas used");
}
parts.join(", ")
}
}
/// Records `start..=end` as replayed and verified by `build`.
fn write_coverage(
silent: &Option<rustock_storage::silent_store::SilentStore>,
start: u64,
end: u64,
build: &str,
) -> Result<()> {
let Some(store) = silent else { return Ok(()) };
if end < start {
return Ok(());
}
// A block that verifies is no longer a gap, whatever an earlier attempt
// recorded about it.
for b in start..=end {
store.clear_gap(b)?;
}
store.add_coverage(&rustock_storage::silent_store::Coverage {
from: start,
to: end,
build: build.to_string(),
recorded_at: std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0),
verified: end - start + 1,
diverged: 0,
})
}
/// Ends the contiguous verified run at the block before `failed`, and notes
/// why it ended. Coverage must stop short of a block whose decisions were not
/// confirmed against the chain.
fn close_run(
silent: &Option<rustock_storage::silent_store::SilentStore>,
run_start: &mut Option<u64>,
failed: u64,
build: &str,
why: &str,
) -> Result<()> {
let Some(store) = silent else { return Ok(()) };
if let Some(start) = run_start.take() {
write_coverage(silent, start, failed - 1, build)?;
}
store.add_gap(failed, why)
}
fn main() -> Result<()> {
// Honour RUST_LOG so a divergence can be chased with the execution
// crate's own debug output, which is usually the fastest next step.
let filter = tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn"));
tracing_subscriber::fmt().with_env_filter(filter).with_target(false).init();
let (positional, options) = split_options(std::env::args().skip(1).collect())?;
let blocks_file: Option<String> =
options.iter().rev().find(|(k, _)| k == "--blocks").map(|(_, v)| v.clone());
// A block list replaces the range, so <from> and <to> go away with it.
let targets: Vec<u64> = match &blocks_file {
Some(path) => {
anyhow::ensure!(positional.len() == 2, "{USAGE}");
let text = std::fs::read_to_string(path)
.with_context(|| format!("reading the block list at {path}"))?;
let mut v: Vec<u64> = text
.lines()
.map(str::trim)
.filter(|l| !l.is_empty() && !l.starts_with('#'))
.map(|l| l.parse::<u64>().with_context(|| format!("block number {l:?}")))
.collect::<Result<_>>()?;
// Sorted so the replay walks the state source forward, and deduped
// so a block listed twice is not verified twice.
v.sort_unstable();
v.dedup();
anyhow::ensure!(!v.is_empty(), "the block list at {path} is empty");
v
}
None => {
anyhow::ensure!(positional.len() == 4, "{USAGE}");
let from: u64 = positional[2].parse().context("<from>")?;
let to: u64 = positional[3].parse().context("<to>")?;
anyhow::ensure!(from <= to, "empty range: {from} > {to}");
(from..=to).collect()
}
};
let data_dir = positional[0].clone();
let unitrie_dir = positional[1].clone();
let from = *targets.first().expect("non-empty");
let to = *targets.last().expect("non-empty");
let network_id: u64 = option(&options, "--network-id", support::TESTNET)?;
let scratch: Option<String> =
options.iter().rev().find(|(k, _)| k == "--scratch").map(|(_, v)| v.clone());
// 0 disables the stop, for a run that is deliberately exhaustive.
let max_failures: u64 = option(&options, "--max-failures", 10)?;
// Small by default: this runs beside whatever else is on the machine, and
// a cache sized for the database rather than the box gets it killed.
let cache_mb: usize = option(&options, "--cache-mb", 256)?;
let explain = options.iter().any(|(k, _)| k == "--explain");
let seed_genesis = options.iter().any(|(k, _)| k == "--seed-genesis");
let sequential = options.iter().any(|(k, _)| k == "--sequential");
let state_roots: Option<DB> = match options
.iter()
.rev()
.find(|(k, _)| k == "--state-roots")
.map(|(_, v)| v.clone())
{
None => None,
Some(dir) => {
let db = DB::open_for_read_only(&Options::default(), &dir, false)
.with_context(|| format!("opening rskj stateRoots at {dir} read-only"))?;
eprintln!(" roots {dir} (orchid -> unitrie translation)");
Some(db)
}
};
// rskj `stateRootsDB`: the Unitrie root holding the state an Orchid root
// named. `None` when the table has no entry, which for a pre-wasabi block
// means the block cannot be addressed or checked at all.
let translate = |orchid: &B256| -> Option<B256> {
let db = state_roots.as_ref()?;
let v = db.get(orchid.as_slice()).ok().flatten()?;
(v.len() == 32).then(|| B256::from_slice(&v))
};
let parent_roots: std::collections::HashMap<u64, Vec<u8>> = match options
.iter()
.rev()
.find(|(k, _)| k == "--parent-roots")
.map(|(_, v)| v.clone())
{
None => Default::default(),
Some(path) => {
let text = std::fs::read_to_string(&path)
.with_context(|| format!("reading parent roots from {path}"))?;
let mut m = std::collections::HashMap::new();
for line in text.lines().skip(1) {
let mut it = line.split(',');
let (Some(b), Some(_), Some(r)) = (it.next(), it.next(), it.next()) else {
continue;
};
let (Ok(b), Some(r)) = (b.trim().parse::<u64>(), r.trim().strip_prefix("0x"))
else {
continue;
};
if let Ok(bytes) = alloy_primitives::hex::decode(r) {
m.insert(b, bytes);
}
}
eprintln!(" parent roots: {} entries from {path}", m.len());
m
}
};
let state_format: String = options
.iter()
.rev()
.find(|(k, _)| k == "--state-format")
.map(|(_, v)| v.clone())
.unwrap_or_else(|| "rskj".to_string());
// Coverage over a block list cannot be recorded as a span: the blocks
// between two listed ones were never replayed, and claiming them would
// make the store assert exactly the thing it exists to disprove.
let contiguous = blocks_file.is_none();
let record_silent: Option<String> =
options.iter().rev().find(|(k, _)| k == "--record-silent").map(|(_, v)| v.clone());
// Purely reads: no scratch database, writes held in memory and dropped
// after each block. Only sound when every block resolves its own parent
// from the source -- i.e. NOT with --sequential, where the last block's
// writes are the next one's parent state.
let read_only = options.iter().any(|(k, _)| k == "--read-only");
let record_legacy: Option<String> = options
.iter()
.rev()
.find(|(k, _)| k == "--record-legacy-events")
.map(|(_, v)| v.clone());
let store = Arc::new(
BlockStore::open_read_only(&data_dir)
.with_context(|| format!("opening block store at {data_dir} read-only"))?,
);
let mut opts = Options::default();
let mut bbt = rocksdb::BlockBasedOptions::default();
bbt.set_block_cache(&rocksdb::Cache::new_lru_cache(cache_mb * 1024 * 1024));
bbt.set_cache_index_and_filter_blocks(true);
opts.set_block_based_table_factory(&bbt);
// A segbuild chunk holds only the nodes its producer touched while
// executing its range, so a read we issue that the producer did not can
// return a false absence rather than an error. That is safe here only
// because every block is checked against its header's state root: a block
// computed from an incomplete source cannot match, so it is reported as
// diverged and recorded as a gap, never as a verified decision. Treat a
// divergence from such a source as "this chunk could not serve the block",
// not as a consensus finding.
let source: Arc<dyn TrieStore> = match state_format.as_str() {
"rskj" => Arc::new(RskjUnitrie {
db: DB::open_for_read_only(&opts, &unitrie_dir, false)
.with_context(|| format!("opening rskj unitrie at {unitrie_dir} read-only"))?,
}),
"rustock" => Arc::new(
rustock_storage::RocksDbTrieStore::open_read_only(&unitrie_dir)
.with_context(|| format!("opening rustock trie store at {unitrie_dir}"))?,
),
// The live node's own backend. Useful for the recent chain, where it
// is the only store that has the state at all -- an archival import
// stops at whatever height it was taken.
"epoch" => Arc::new(
rustock_storage::epoch_store::EpochTrieStore::open_read_only_with(
&unitrie_dir,
rustock_storage::epoch_store::EpochConfig::default(),
Arc::new(rustock_storage::epoch_backend::FlatFactory::new(0)),
)
.with_context(|| format!("opening epoch trie store at {unitrie_dir}"))?,
),
other => {
anyhow::bail!("--state-format must be rskj, rustock or epoch, not {other:?}")
}
};
// Somewhere for the replay's own writes that is neither database. Kept on
// disk rather than in memory because a 50,000-block range writes far more
// trie than belongs in RAM.
let tmp;
let scratch_path = match &scratch {
Some(p) => std::path::PathBuf::from(p),
None => {
tmp = tempfile::tempdir().context("creating a scratch directory")?;
tmp.path().to_path_buf()
}
};
// Kept as the concrete type as well as behind the trait: the replay has
// to be able to ask whether its own writes succeeded. They go to a
// scratch directory on whatever filesystem had room, and a full disk
// makes a trie store answer "absent" for nodes it was asked to keep --
// which yields wrong roots, not errors.
anyhow::ensure!(
!(read_only && sequential),
"--read-only cannot be combined with --sequential: a sequential replay's \
parent state IS the previous block's writes, which --read-only discards"
);
// `overlay` is kept alongside the trait object so the per-block clear can
// be called; in read-only mode that is what keeps memory flat.
let (scratch_store, overlay, trie): (
Option<Arc<rustock_storage::RocksDbTrieStore>>,
Option<Arc<OverlayTrieStore>>,
Arc<dyn TrieStore>,
) = if read_only {
let o = Arc::new(OverlayTrieStore::new(source));
eprintln!(" scratch (none: --read-only, writes held in memory per block)");
(None, Some(o.clone()), o)
} else {
let sc = Arc::new(
rustock_storage::RocksDbTrieStore::open(&scratch_path)
.with_context(|| format!("opening scratch trie at {}", scratch_path.display()))?,
);
let spill: Arc<dyn TrieStore> = sc.clone();
let t: Arc<dyn TrieStore> = Arc::new(OverlayTrieStore::with_spill(source, spill));
(Some(sc), None, t)
};
// Genesis state is not read from anywhere -- it is constructed from the
// chain config. Build it into the scratch store and a contiguous replay
// from #1 carries its own state forward, each block's parent root having
// been written by the one before. In sequential mode the state is carried
// here rather than resolved per block from the source, which is why the
// source goes unread in that mode (zero open descriptors on it, measured).
//
// A previous version of this comment also claimed a segbuild chunk
// "captures its producer's read set rather than roots addressable by
// hash", and gave that as a reason the early chain had no usable state
// source. **That is false**, and it is worth saying so plainly because it
// stopped at least one investigation. It came from probing a chunk with
// `rskj_root_probe`, which opens the DEFAULT column family; our own trie
// stores key nodes in `trie_nodes`, so that tool reports ABSENT for every
// hash in a segbuild chunk whatever it holds. Probed through the right
// column family (`trie_probe`), chunk `000-1-1591000` does hold
// pre-wasabi100 unitrie roots with resolvable children -- verified
// against block #6,916's root, taken from rskj's own `stateRoots` table.
//
// So a segbuild chunk is a plausible state source for this era. What
// stops it being used here is not its contents but the composition
// below: `OverlayTrieStore` reads the spill before the base, and
// `--seed-genesis` puts every computed node in the spill, so the base is
// never consulted. Serving reads from a sealed chunk instead would mean
// reading base-first -- safe, since the trie is content-addressed and a
// hash cannot map to two different values -- and has not been measured.
let mut carried: Option<TrieNode> = None;
anyhow::ensure!(
!sequential || blocks_file.is_none(),
"--sequential replays a contiguous range; it cannot take --blocks"
);
if seed_genesis {
use rustock_trie::{account_key, AccountState, TrieKeySlice};
let config = if network_id == support::MAINNET {
rustock_core::config::ChainConfig::mainnet()
} else {
rustock_core::config::ChainConfig::testnet()
};
let mut genesis = TrieNode::empty();
for e in &config.genesis_alloc() {
genesis = genesis.put(
&TrieKeySlice::from_key(&account_key(&e.address)),
&AccountState::new(e.nonce, e.balance).encode(),
trie.as_ref(),
);
}
genesis.save(trie.as_ref(), true);
// The genesis header predates RSKIP126, so its stateRoot is the
// ORCHID root of this trie, not the unitrie node hash. Comparing the
// wrong one of those is how a correct genesis looks broken.
let built = rustock_trie::orchid_state_root(&genesis, trie.as_ref());
let expected = config.genesis_header().state_root;
// Checked, not assumed: a wrong genesis would make every block below
// the first real state source diverge, and look like a consensus bug.
anyhow::ensure!(
built == expected,
"constructed genesis state root {built:?} does not match the genesis header {expected:?}"
);
eprintln!(" seeded genesis state (orchid root {built:?})");
carried = Some(genesis);
}
let hardfork_cfg = support::hardfork_for(network_id);
eprintln!(
"verifying {} block(s) in #{from}..=#{to} on network-id {network_id} (chain_id {})",
targets.len(),
hardfork_cfg.chain_id
);
if let Some(path) = &blocks_file {
eprintln!(" blocks listed in {path}");
}
eprintln!(" blocks {data_dir} (read-only)");
eprintln!(" state {unitrie_dir} (read-only, {state_format} format)");
eprintln!(" scratch {}", scratch_path.display());
// RSKIP-144 parallel sublists. 1 runs every block as one; above 1, a
// block that declares at least two parallel sublists runs them on their
// own threads. Replaying the same range at 1 and at 2 is the check that
// matters, because both must agree with the reference chain.
let execution_threads: usize = options
.iter()
.find(|(k, _)| k == "--execution-threads")
.map(|(_, v)| v.parse())
.transpose()?
.unwrap_or(1);
let processor = BlockProcessor::new(hardfork_cfg.clone(), store.clone())
.with_execution_threads(execution_threads);
let silent = match &record_silent {
Some(dir) => {
let s = rustock_storage::silent_store::SilentStore::open(dir)
.with_context(|| format!("opening the silent-decision store at {dir}"))?;
eprintln!(" silent {dir} (recording Bridge decisions that emit no event)");
Some(s)
}
None => None,
};
// The reconstructed-event store rides on the same decisions as the
// silent store, so either option turns recording on, and both together
// cost one replay rather than two.
let legacy = match &record_legacy {
Some(dir) => {
let s = rustock_storage::legacy_event_store::LegacyEventStore::open(dir)
.with_context(|| format!("opening the legacy event store at {dir}"))?;
eprintln!(" legacy {dir} (reconstructing peg events the Bridge never logged)");
Some(s)
}
None => None,
};
if silent.is_some() || legacy.is_some() {
rustock_execution::silent::start();
}
// Totals for the coverage record, which is what tells a later reader
// that an empty scan means "none here" rather than "never built".
let mut legacy_first: Option<u64> = None;
let mut legacy_last: u64 = 0;
let mut legacy_blocks: u64 = 0;
let mut legacy_verified: u64 = 0;
// The open span, and the last block whose decisions were actually kept.
let mut legacy_run: Option<u64> = None;
let mut legacy_good: u64 = 0;
let mut legacy_events: u64 = 0;
let mut legacy_skipped: u64 = 0;
let mut legacy_skip_kinds: std::collections::BTreeMap<String, u64> =
std::collections::BTreeMap::new();
// The build that produced the records. A replay is only as good as the
// code that ran it, and this is the only way to tell later which rows
// came from which rules.
let build = std::env::var("RUSTOCK_BUILD")
.ok()
.or_else(|| option_env!("RUSTOCK_BUILD").map(str::to_string))
.unwrap_or_else(|| {
// `-C` the crate's own directory rather than the current one: a
// run launched from outside the repo recorded "unknown", which is
// precisely the provenance this field exists to carry.
std::process::Command::new("git")
.args(["-C", env!("CARGO_MANIFEST_DIR"), "rev-parse", "--short", "HEAD"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_else(|| "unknown".to_string())
});
// The first block of the contiguous run being recorded; a divergence or
// an unverifiable block closes the run and starts a new one, so coverage
// describes only blocks that actually verified.
let mut run_start: Option<u64> = None;
let mut silent_rows = 0u64;
if max_failures > 0 {
eprintln!(" stopping after {max_failures} divergent or unverifiable block(s)");
}
// How often to pay for an orchid conversion on pre-RSKIP126 blocks that
// decided nothing. 0 disables the tripwire.
let orchid_every: u64 = std::env::var("RUSTOCK_ORCHID_CHECK_INTERVAL")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(0);
let started = std::time::Instant::now();
let mut checked = 0u64;
let mut diverged = 0u64;
let mut unverifiable = 0u64;
// Executed to carry state forward, but with no root comparison available:
// pre-RSKIP126 blocks that decided nothing. Counted apart from `checked`
// so the summary never calls them verified.
let mut unchecked = 0u64;
let mut first_bad: Option<u64> = None;
let mut stopped_at: Option<u64> = None;
for (done, n) in targets.iter().copied().enumerate() {
// Checked at the top of each block so every exit from the body below
// -- divergence or unverifiable -- is counted before it is consulted.
if max_failures > 0 && diverged + unverifiable >= max_failures {
stopped_at = Some(n - 1);
eprintln!(
"\nstopping at #{}: {max_failures} failures reached. \
Re-run with --max-failures 0 to see the rest.",
n - 1
);
break;
}
let Some(hash) = store.canonical_hash(n)? else {
eprintln!("#{n}: no canonical hash — the metadata rebuild did not reach here");
close_run(&silent, &mut run_start, n, &build, "no canonical hash")?;
unverifiable += 1;
continue;
};
let Some(header) = store.header(hash)? else {
eprintln!("#{n}: header missing");
close_run(&silent, &mut run_start, n, &build, "header missing")?;
unverifiable += 1;
continue;
};
let Some((transactions, ommers)) = store.body(hash)? else {
eprintln!("#{n}: body missing");
close_run(&silent, &mut run_start, n, &build, "body missing")?;
unverifiable += 1;
continue;
};
// The pre-state is rskj's, not ours: that is what makes each block an
// independent check instead of a continuation of the last one.
let Some(parent) = store.header(header.parent_hash)? else {
eprintln!("#{n}: parent header missing");
close_run(&silent, &mut run_start, n, &build, "parent header missing")?;
unverifiable += 1;
continue;
};
let pre = if sequential {
match carried.take() {
Some(r) => r,
None => {
eprintln!("#{n}: no carried state; --sequential needs --seed-genesis \
(or a predecessor that succeeded)");
close_run(&silent, &mut run_start, n, &build, "no carried state")?;
unverifiable += 1;
continue;
}
}
} else {
// Below wasabi100 the header's stateRoot is an orchid root and names
// no unitrie node, so the recorded unitrie root of the parent is used
// instead where one was supplied.
let parent_key: Vec<u8> = match translate(&parent.state_root) {
Some(unitrie) => unitrie.as_slice().to_vec(),
None => parent_roots
.get(&parent.number)
.cloned()
.unwrap_or_else(|| parent.state_root.as_slice().to_vec()),
};
let Some(root_data) = trie.get(&parent_key) else {
// States the fact, not a cause: an absent root means the source
// was pruned below here, or that these blocks and this unitrie
// are from different chains. Naming only one of those sends the
// reader after the wrong thing.
eprintln!(
"#{n}: parent state root {:?} is not in the source unitrie \
(pruned below this height, or not the same chain as these blocks)",
parent.state_root
);
close_run(
&silent,
&mut run_start,
n,
&build,
"parent state root not in the source unitrie",
)?;
close_legacy_run(&legacy, &mut legacy_run, legacy_good, &build,
legacy_blocks, legacy_verified, legacy_events, legacy_skipped, &legacy_skip_kinds)?;
unverifiable += 1;
continue;
};
TrieNode::from_message(&root_data, trie.as_ref())
};
let block = Block { header, transactions, ommers };
let processed = match processor.execute_block(&block, &pre, trie.clone()) {
Ok(p) => p,
Err(e) => {
println!("#{n} EXECUTION ERROR {e}");
// Whatever the failed block recorded before it failed is
// dropped, not kept and not left to be blamed on the next one.
rustock_execution::silent::drain();
close_run(&silent, &mut run_start, n, &build, "execution error")?;
close_legacy_run(&legacy, &mut legacy_run, legacy_good, &build,
legacy_blocks, legacy_verified, legacy_events, legacy_skipped, &legacy_skip_kinds)?;
diverged += 1;
first_bad.get_or_insert(n);
continue;
}
};
// Before the verdict is believed: if the scratch could not keep what
// this block computed, the comparison below is meaningless, and so is
// every block after it. Stop rather than record a plausible lie.
if scratch_store.as_ref().is_some_and(|sc| sc.write_failed()) {
if let Some(store) = &silent {
rustock_execution::silent::drain();
if let Some(start) = run_start.take() {
write_coverage(&silent, start, n - 1, &build)?;
}
store.add_gap(n, "scratch trie write failed (disk full?)")?;
}
anyhow::bail!(
"#{n}: a write to the scratch trie at {} failed, so the replay can no \
longer read back what it computed. Nothing after this point is sound. \
Free space and re-run; completed chunks are recorded as coverage.",
scratch_path.display()
);
}
// Before RSKIP126 the header carries an orchid root, so the unitrie
// hash cannot be compared against it. Converting is expensive -- it
// walks the whole state -- so it is paid for only where the answer
// matters: a block that recorded a decision, or every `orchid_every`
// blocks as a tripwire. Elsewhere state is carried forward unchecked,
// and the coverage records say so by omitting those blocks.
let unitrie_root = hardfork_cfg.has_unitrie_state_root(n);
let decisions_here = if silent.is_some() || legacy.is_some() {
rustock_execution::silent::drain()
} else {
Vec::new()
};
// Below wasabi100 the header's root is an Orchid one and cannot be
// compared with what we compute. There are two ways to get a verdict,
// and they are not equally good:
//
// - rskj's own translation table gives the Unitrie root holding the
// state that Orchid root named. Comparing that against ours is
// like with like, costs a single lookup, and reconstructs no
// Orchid trie. Preferred whenever the table is supplied.
// - Converting our Unitrie into an Orchid root is the fallback, and
// a weak one: Orchid was a different structure, so a mismatch
// cannot be told apart from a limit of the conversion.
let translated = (!unitrie_root)
.then(|| translate(&block.header.state_root))
.flatten();
// Only worth the conversion when nothing better is available, and
// then only where the answer matters: a block that decided something,
// one the caller named, or a periodic tripwire.
let check_orchid = !unitrie_root
&& translated.is_none()
&& (blocks_file.is_some()
|| !decisions_here.is_empty()
|| (orchid_every > 0 && n.is_multiple_of(orchid_every)));
// What was compared, and against what -- so the report shows the two
// numbers the verdict actually rests on.
let (computed_state, expected_state, via_orchid) = if unitrie_root {
(Some(processed.state_root_hash), block.header.state_root, false)
} else if let Some(expected) = translated {
(Some(processed.state_root_hash), expected, false)
} else if check_orchid {
(
Some(rustock_trie::orchid_state_root(&processed.new_state_root, trie.as_ref())),
block.header.state_root,
true,
)
} else {
(None, block.header.state_root, false)
};
let state_ok = computed_state.is_none_or(|c| c == expected_state);
let checked_here = unitrie_root || translated.is_some() || check_orchid;
// A conversion mismatch yields no verdict rather than a divergence.
let orchid_inconclusive = via_orchid && !state_ok;
if sequential {
carried = Some(processed.new_state_root.clone());
}
let verdict = Verdict {
state: !state_ok && !orchid_inconclusive,
receipts: processed.receipts_root != block.header.receipts_root,
gas: processed.gas_used != block.header.gas_used,
};
if orchid_inconclusive {
eprintln!(
"#{n}: pre-wasabi100 orchid comparison did not match; this cannot \
distinguish a conversion limit from an execution fault, so the block \
is left unverified"
);
close_run(
&silent, &mut run_start, n, &build,
"pre-wasabi100: orchid comparison inconclusive",
)?;
rustock_execution::silent::drain();
close_legacy_run(&legacy, &mut legacy_run, legacy_good, &build,
legacy_blocks, legacy_verified, legacy_events, legacy_skipped, &legacy_skip_kinds)?;
unverifiable += 1;
continue;
}
if checked_here {
checked += 1;
} else {
unchecked += 1;
}
// Nothing this block wrote is needed again: the next block resolves
// its own parent from the source. Dropping it here is what keeps a
// read-only run at constant memory over millions of blocks.
if let Some(o) = &overlay {
o.clear_overlay();
}
// Drained above, before the verdict, because the orchid check needs
// to know whether this block decided anything.
let decisions = decisions_here;
// Reconstructed events, written only for blocks that verified: a
// block whose state root disagrees was executed by this code and not
// by the network, so its decisions are ours, not the chain's.
if let Some(lstore) = &legacy {
if verdict.ok() {
legacy_first.get_or_insert(n);
legacy_last = n;
legacy_run.get_or_insert(n);
legacy_good = n;
legacy_blocks += 1;
if checked_here {
legacy_verified += 1;
}
let mut rows = Vec::new();
// One log_index sequence per block, as a receipt would have.
let mut log_index: u32 = 0;
for d in &decisions {
// Never reconstruct what the Bridge really logged: the
// node's own index already has it, and a duplicate would
// double-count in the union of the two. A record that
// does not say is REFUSED, not assumed -- see
// `legacy_events::reconstructable`.
match rustock_execution::bridge::legacy_events::reconstructable(d) {
Ok(true) => {}
Ok(false) => continue,
Err(why) => {
legacy_skipped += 1;
*legacy_skip_kinds
.entry(format!("{}/UNGATED({why})", d.kind.name()))
.or_insert(0) += 1;
continue;
}
}
// A flyover peg-in is the case this whole exercise is
// about, and it is rare enough to name individually.
if matches!(
d.fields.iter().find(|(k, _)| *k == "peginPath").map(|(_, v)| v),
Some(rustock_execution::silent::FieldValue::Text(p)) if p == "flyover"
) {
let f = |name: &str| {
d.fields
.iter()
.find(|(k, _)| *k == name)
.map(|(_, v)| v.to_json())
.unwrap_or_else(|| "null".into())
};
eprintln!(
"FLYOVER #{} tx[{}] lbc={} amountSat={} btcTx={}",
d.block,
d.tx_index,
f("destination"),
f("amountSat"),
f("btcTxHash"),
);
}
let evs = rustock_execution::bridge::legacy_events::events_for(d);
if evs.is_empty() {
legacy_skipped += 1;
*legacy_skip_kinds
.entry(rustock_execution::bridge::legacy_events::skip_label(d))
.or_insert(0) += 1;
continue;
}
// The real index keys each event to its transaction, so
// this does too, taken from the body by position -- the
// ordering the receipts root commits to.
let tx_hash = block
.transactions
.get(d.tx_index as usize)
.map(|t| t.tx_hash())
.unwrap_or_default();
for e in evs {
rows.push(rustock_storage::legacy_event_store::EventRow {
topic0: e.topic0,
block: d.block,
tx_index: d.tx_index,
log_index,
tx_hash,
topics: e.topics,
data: e.data,
});
log_index += 1;
}
}
if !rows.is_empty() {
legacy_events += rows.len() as u64;
lstore.put_events(&rows)?;
}
// Coverage every so often, not only at the end: this runs for
// hours, and a span written only on a clean exit is a span
// lost to any interruption -- leaving events on disk that
// nothing claims, which reads as "never built".
if legacy_blocks.is_multiple_of(LEGACY_COVERAGE_EVERY) {
if let Some(first) = legacy_run {
lstore.add_coverage(