feat(search): Re-search a root fail high a ply shallower per fail high

arche-core/src/bench.rs

@@ -613,8 +613,8 @@ mod tests {
             ],
             [
                 [177_161, 6_475, 22_519, 437, 0, 181, 0],
-                [12_948, 1_021, 3_399, 38, 15, 0, 0],
-                [14_088, 1_084, 3_717, 1, 0, 0, 8],
+                [11_963, 942, 3_125, 36, 15, 0, 0],
+                [12_614, 982, 3_315, 1, 0, 0, 8],
             ],
             "nodes, cutoffs, stores, tainted, tcuts, refused, skipped"
         );
@@ -831,23 +831,23 @@ mod tests {
             counted,
             vec![
                 ("start", 495_700),
-                ("italian", 1_121_074),
-                ("ruy lopez", 366_742),
-                ("kiwipete", 1_575_476),
+                ("italian", 877_695),
+                ("ruy lopez", 405_334),
+                ("kiwipete", 1_288_527),
                 ("perft 4", 2_814_759),
-                ("promotions", 1_983_365),
-                ("middlegame", 532_079),
-                ("sharp middlegame", 449_131),
+                ("promotions", 1_851_773),
+                ("middlegame", 139_793),
+                ("sharp middlegame", 601_307),
                 ("bratko kopec 1", 11_714),
                 ("wac 4", 1_516),
-                ("rook and pawns", 214_156),
-                ("tarrasch", 238_448),
-                ("lucena", 361_398),
+                ("rook and pawns", 130_365),
+                ("tarrasch", 240_269),
+                ("lucena", 173_822),
                 ("philidor", 357_659),
                 ("minor endgame", 156_339),
                 ("queen endgame", 337_681),
                 ("king and pawn", 38_955),
-                ("trebuchet", 32_344),
+                ("trebuchet", 34_982),
             ]
         );
     }

arche-core/src/engine.rs

@@ -548,6 +548,13 @@ pub struct SearchConfig {
     /// the control. Only the deepening loop reads it, so a search asked for
     /// a fixed depth opens full.
     pub aspiration: bool,
+    /// Whether the search after a root fail high runs a ply shallower for
+    /// each fail high since the depth opened or last failed low. Rides on
+    /// `aspiration`, since only a window fails high.
+    ///
+    /// Off in the reference: it changes what a depth answers, where the
+    /// window alone changes only what the depth costs.
+    pub fail_high_reduction: bool,
     /// Whether a deep node whose table move stands a margin above every
     /// other move, shown by a half depth search with that move excluded,
     /// searches the table move a ply deeper.
@@ -657,7 +664,7 @@ impl SearchConfig {
     ///
     /// `taint` is not among them: it is a policy with four values rather
     /// than a switch, and `residuals` already takes it.
-    pub const SWITCHES: [(&'static str, TurnOff); 14] = [
+    pub const SWITCHES: [(&'static str, TurnOff); 15] = [
         ("reverse_futility", |config| config.reverse_futility = false),
         ("null_move", |config| config.null_move = false),
         ("adaptive_null_move", |config| {
@@ -677,6 +684,9 @@ impl SearchConfig {
         ("reduction_table", |config| config.reduction_table = false),
         ("move_memory", |config| config.move_memory = false),
         ("aspiration", |config| config.aspiration = false),
+        ("fail_high_reduction", |config| {
+            config.fail_high_reduction = false
+        }),
         ("singular_extensions", |config| {
             config.singular_extensions = false
         }),
@@ -715,6 +725,7 @@ impl SearchConfig {
             reduction_table: false,
             move_memory: false,
             aspiration: false,
+            fail_high_reduction: false,
             singular_extensions: false,
         }
     }
@@ -770,6 +781,7 @@ impl Default for SearchConfig {
             reduction_table: true,
             move_memory: true,
             aspiration: true,
+            fail_high_reduction: true,
             singular_extensions: true,
         }
     }
@@ -878,7 +890,7 @@ mod switches {
         };
         let one = |name| SearchConfig::without(name).expect(name);
         let default = nodes(SearchConfig::default());
-        let outers: Vec<(&str, Vec<u64>)> = ["null_move", "late_move_reductions"]
+        let outers: Vec<(&str, Vec<u64>)> = ["null_move", "late_move_reductions", "aspiration"]
             .into_iter()
             .map(|outer| (outer, nodes(one(outer).config())))
             .collect();
@@ -887,6 +899,7 @@ mod switches {
             ("late_move_reductions", "deep_reductions"),
             ("late_move_reductions", "late_move_pruning"),
             ("late_move_reductions", "reduction_table"),
+            ("aspiration", "fail_high_reduction"),
         ] {
             assert_ne!(nodes(one(inner).config()), default, "{inner} did nothing");
             let pair = one(outer).and(one(inner)).expect("a pair");
@@ -1080,6 +1093,12 @@ struct Aspiration {
     /// at the edge and stays there.
     low_failures: u8,
     high_failures: u8,
+    /// The plies the next search of the depth is shortened by: one for
+    /// each fail high since the depth opened or last failed low. The move
+    /// a fail high names was the depth's answer after a full depth
+    /// re-search at 89 of 100 such depths on the games suite (d82638a), so
+    /// that re-search was spent mostly confirming it.
+    shallower: u8,
 }
 
 impl Aspiration {
@@ -1102,6 +1121,7 @@ impl Aspiration {
             centre: 0,
             low_failures: ASPIRATION_FAILURES,
             high_failures: ASPIRATION_FAILURES,
+            shallower: 0,
         };
         let Some(centre) = previous else {
             return full;
@@ -1115,19 +1135,29 @@ impl Aspiration {
             centre,
             low_failures: 0,
             high_failures: 0,
+            shallower: 0,
         }
     }
 
+    /// The depth the next search of an iteration at `depth` runs to. A
+    /// side fails at most `ASPIRATION_FAILURES` times and the window opens
+    /// narrow only from `ASPIRATION_MIN_DEPTH`, so this never reaches zero.
+    fn depth(self, depth: u8) -> u8 {
+        depth - self.shallower
+    }
+
     /// The window after an iteration answered `bound` rather than a score
     /// inside this one. Only the side that failed moves, since the other
-    /// proved nothing.
+    /// proved nothing. A fail high also shortens the next search by a ply,
+    /// and a fail low restores the full depth.
     fn widen(self, bound: ScoreBound) -> Self {
         match bound {
             ScoreBound::Upper => {
                 let low_failures = self.low_failures.saturating_add(1);
                 Self {
                     alpha: Self::below(self.centre, low_failures),
                     low_failures,
+                    shallower: 0,
                     ..self
                 }
             }
@@ -1136,6 +1166,7 @@ impl Aspiration {
                 Self {
                     beta: Self::above(self.centre, high_failures),
                     high_failures,
+                    shallower: self.shallower + 1,
                     ..self
                 }
             }
@@ -1213,6 +1244,33 @@ mod aspiration {
         assert_eq!((twice.alpha, twice.beta), (30 - 4 * W, 30 + W));
     }
 
+    #[test]
+    fn each_fail_high_shortens_the_next_search_and_a_fail_low_restores_it() {
+        let opened = Aspiration::open(Some(30), AT + 3);
+        assert_eq!(opened.depth(AT + 3), AT + 3);
+
+        let once = opened.widen(ScoreBound::Lower);
+        assert_eq!(once.depth(AT + 3), AT + 2);
+        let twice = once.widen(ScoreBound::Lower);
+        assert_eq!(twice.depth(AT + 3), AT + 1);
+
+        // a fail low proves nothing about the move that failed high, so the
+        // search is back at the full depth, and the next fail high counts
+        // from there
+        let low = twice.widen(ScoreBound::Upper);
+        assert_eq!(low.depth(AT + 3), AT + 3);
+        assert_eq!(low.widen(ScoreBound::Lower).depth(AT + 3), AT + 2);
+
+        // the fewest plies a depth that opened narrow can be searched to
+        let mut spent = Aspiration::open(Some(30), AT);
+        for _ in 0..3 {
+            spent = spent.widen(ScoreBound::Lower);
+        }
+        assert_eq!(spent.beta, Aspiration::FULL_BETA);
+        assert_eq!(spent.depth(AT), AT - 3);
+        assert_eq!(full().depth(AT), AT);
+    }
+
     #[test]
     fn three_failures_open_that_side_to_the_edge() {
         let mut window = Aspiration::open(Some(30), AT);
@@ -3308,8 +3366,18 @@ impl Engine for AlphaBeta {
             self.root_nodes.clear();
             loop {
                 let (limits, stop) = search_options.for_iteration(best.is_some(), total_nodes);
-                match self.search_root(depth, limits, stop, window) {
+                let searched = if self.config.fail_high_reduction {
+                    window.depth(depth)
+                } else {
+                    depth
+                };
+                match self.search_root(searched, limits, stop, window) {
                     SearchOutcome::Aborted(deeper) => {
+                        // a shortened search follows a fail high, which
+                        // `best` holds. Its partial best comes from a
+                        // shallower search than that floor's and does not
+                        // replace it
+                        let deeper = deeper.filter(|_| searched == depth);
                         // the interrupted search's best outranks what
                         // answers now, whenever it has one. The move it
                         // hands back beat the window's alpha, and the moves

arche-core/src/engine/tests.rs

@@ -1416,6 +1416,47 @@ mod search {
         assert!(swaps > 0, "no budget in the sweep swapped a move in");
     }
 
+    #[test]
+    fn a_stopped_shortened_search_answers_with_the_floor_before_it() {
+        // after a root fail high the depth is searched a ply shallower, and
+        // a stop inside that search answers with the floor rather than with
+        // the shallower search's partial best, move and score alike. Every
+        // floor of a deepening is swept, since which depths fail high is a
+        // fact about the position
+        let mut swept = 0;
+        for fen in [SHARP_MIDDLEGAME, fens::KIWIPETE] {
+            let mut whole = engine(Board::from_fen(fen).unwrap());
+            let mut reports: Vec<(u8, u64, ScoreBound, Play, Score)> = Vec::new();
+            let options = SearchParameters::new(None, nodes_only(150_000));
+            let _ = whole.iterative_deepening_search(options, |depth, result, _, bound| {
+                reports.push((depth, result.nodes, bound, result.best_move, result.score));
+            });
+            for pair in reports.windows(2) {
+                let [(depth, from, bound, play, score), (next_depth, to, _, _, _)] = *pair else {
+                    unreachable!("a window of two");
+                };
+                if bound != ScoreBound::Lower || next_depth != depth || to <= from + 1 {
+                    continue;
+                }
+                for limit in (from + 1..to).step_by(((to - from) / 40).max(1) as usize) {
+                    let mut e = engine(Board::from_fen(fen).unwrap());
+                    let options = SearchParameters::new(None, nodes_only(limit));
+                    let outcome = e.iterative_deepening_search(options, |_, _, _, _| {});
+                    let SearchOutcome::Aborted(Some(result)) = outcome else {
+                        panic!("budget {limit} in {fen}: {outcome:?}");
+                    };
+                    assert_eq!(
+                        (result.best_move, result.score),
+                        (play, score),
+                        "budget {limit} in {fen}: the floor at depth {depth} was replaced"
+                    );
+                    swept += 1;
+                }
+            }
+        }
+        assert!(swept > 0, "no depth failed high before the next search");
+    }
+
     /// The soft line reads the nodes under the move a depth chose against
     /// the nodes under every root move, and whether the move changed from
     /// the depth before. Two root moves, counted twice and once.

arche-core/src/strategy.rs

@@ -32,7 +32,7 @@ pub const TABLE_BYTES: usize = 16 * 1024 * 1024;
 /// How many of the suite's points the search takes at that depth with that
 /// table. Exact, not a floor: a change that moves it updates this number in
 /// the same commit.
-pub const EXPECTED_POINTS: u32 = 104298;
+pub const EXPECTED_POINTS: u32 = 102482;
 
 const SUITE: &str = include_str!("../strategy.epd");
 

arche-core/src/tactics.rs

@@ -33,7 +33,7 @@ pub const TABLE_BYTES: usize = 16 * 1024 * 1024;
 /// is the gate. A change that moves it in either direction updates this
 /// number in the same commit, and a change that lowers it says what the
 /// positions were spent on.
-pub const EXPECTED_PASSES: usize = 228;
+pub const EXPECTED_PASSES: usize = 217;
 
 /// The count the suite may not go under, whatever a commit says it meant to
 /// spend.

docs/ARCHITECTURE.md

@@ -66,9 +66,9 @@ factorization machine term over every pair of pieces.
   worse than one we can force), deepening one ply at a time with each pass
   seeding the next through the transposition table and opening at an
   aspiration window around the last pass's score, widened on the side a
-  pass falls outside it. At the horizon,
-  quiescence search keeps following captures until the position goes
-  quiet, since evaluating in the middle of an exchange scores a hanging
+  pass falls outside it and searched a ply shallower after each fail high.
+  At the horizon, quiescence search keeps following captures until the
+  position goes quiet, since evaluating in the middle of an exchange scores a hanging
   queen as material. Also here: principal variation search (ask a move
   after the first whether it beats the best so far before searching it
   properly), reverse futility pruning (answer a node from its evaluation

docs/INSTRUMENTS.md

@@ -299,7 +299,8 @@ frees more or less than that. A keyword given twice is refused, so a pair is
 one word; the same switch twice and a third switch are refused too. Where one
 switch of a pair is only ever asked under the other (`adaptive_null_move` under
 `null_move`; `deep_reductions`, `late_move_pruning` and `reduction_table`
-under `late_move_reductions`), the pair searches as many nodes as the outer
+under `late_move_reductions`; `fail_high_reduction` under `aspiration`), the
+pair searches as many nodes as the outer
 single, position by position, which a test holds.
 
 **`off` absent means both sides are the default**, which the header says as

tests/effort_session.rs

@@ -180,6 +180,6 @@ fn a_switch_the_engine_does_not_have_is_refused() {
         .strip_prefix("unrecognised effort off: quiet_futilty (a switch is one of ")
         .and_then(|rest| rest.strip_suffix(')'))
         .unwrap_or_else(|| panic!("stderr: {printed}"));
-    assert_eq!(named.split(", ").count(), 14, "stderr: {printed}");
+    assert_eq!(named.split(", ").count(), 15, "stderr: {printed}");
     assert!(output.stdout.is_empty());
 }

tests/uci_session.rs

@@ -368,7 +368,7 @@ fn the_clear_hash_button_empties_the_table() {
 const SHARP_MIDDLEGAME: &str = "r1b2rk1/ppp1qppp/4pn2/6N1/Qn1P4/2NBP3/PP3PPP/R3K2R w KQ - 9 12";
 
 /// WAC.021 of the tactical suite. Its depth five fails low, its depth eight
-/// fails high, and its depth nine changes the root move and reports the new
+/// fails high, and its depth ten changes the root move and reports the new
 /// move as a floor first.
 const WAC_021: &str = "5rk1/1b3p1p/pp3p2/3n1N2/1P6/P1qB1PP1/3Q3P/4R1K1 w - - 0 1";
 
@@ -377,17 +377,19 @@ const WAC_021: &str = "5rk1/1b3p1p/pp3p2/3n1N2/1P6/P1qB1PP1/3Q3P/4R1K1 w - - 0 1
     feature = "machine-test",
     ignore = "pins a search made with the shipped factor table, which the test rank replaces"
 )]
-fn the_move_a_swap_answers_with_opens_the_last_line_said() {
+fn a_stop_in_the_shortened_re_search_answers_the_floor_the_last_line_said() {
     // a node budget, so the cut falls on the same node on every machine.
     // It has to land after an iteration reports its floor and before that
-    // iteration ends: depth seven answers d3e2 and finishes at 41,237
-    // nodes, and depth eight reports d3e2 as a floor from 58,527 nodes on
-    // and finishes at 90,907. The floor names the move depth seven
-    // answered, so the swap is to the same move. The budget moves with the
+    // iteration ends: depth seven answers d3e2 and finishes at 41,216
+    // nodes, and depth eight reports d3e2 as a floor at 58,506 and again
+    // at 64,869 and finishes at 65,727. The cut falls in the re-search a
+    // ply shallower that follows the first floor, so the floor is held
+    // rather than swapped for that search's partial answer. The floor
+    // names the move depth seven answered. The budget moves with the
     // tree, in the commit that moved it
     let mut s = Session::start(&[]);
     s.say(&format!("position fen {}", SHARP_MIDDLEGAME));
-    s.say("go nodes 70000");
+    s.say("go nodes 61000");
     let answer = s.wait_for(|l| l.starts_with("bestmove"));
     let best = move_of(&answer);
     let info = s
@@ -545,16 +547,17 @@ fn a_root_move_that_reaches_beta_is_reported_as_a_floor_and_then_answered_with()
 )]
 fn a_floor_answers_until_the_wider_search_replaces_it() {
     // what the engine plays when the wider search never finishes. WAC.021 is
-    // answered with d2c3 at depth eight. At depth nine d2h6 reaches beta and
-    // is reported as a floor at 87,942 nodes, and again at 89,089 and 91,472
-    // as the window widens, and the search finishes at 189,094. A budget
-    // after d2h6's last floor and inside that is interrupted before anything
-    // beats alpha, so the floor is what is left to answer with; with the
-    // floor not held it answers d2c3, the move the search has just shown
-    // worse
+    // answered with d2c3 at depth nine. At depth ten d2c3 reaches beta at
+    // 37,691 nodes, then d2h6 does in the re-search at nine and is reported
+    // as a floor at 62,711 and again, at eight, at 64,045, and a search at
+    // seven finishes the depth at 67,092. A budget after d2h6's last
+    // floor and inside that stops a shallower search, whose partial answer
+    // never replaces a floor, so the floor is what is left to answer with;
+    // with the floor not held it answers d2c3, the move the search has just
+    // shown worse
     let mut s = Session::start(&[]);
     s.say(&format!("position fen {}", WAC_021));
-    s.say("go nodes 100000");
+    s.say("go nodes 65500");
     let answer = s.wait_for(|l| l.starts_with("bestmove"));
     let best = move_of(&answer);