Preview: fixes for word splitting and escaping issues

src/diff.rs

@@ -1,6 +1,6 @@
 use similar::{Algorithm, ChangeTag, DiffOp, TextDiff};
 
-use crate::{Block, DiffResult, DiffSpan, SpanTag, TypstLabel};
+use crate::{Block, DiffResult, DiffSpan, SpanTag, TypstLabel, TypstRaw};
 
 /// Diff two sequences of blocks, returning a list of diff results.
 ///
@@ -137,8 +137,8 @@ fn process_replace(old_range: &[Block], new_range: &[Block], results: &mut Vec<D
 /// individual single-character token.
 ///
 /// Typst code expressions (`#func[...]`, `#func(...)`), references (`@label`),
-/// and labels (`<label>`) are treated as atomic tokens so the diff never
-/// fragments valid Typst syntax.
+/// labels (`<label>`), raw text (`` `code` ``), and escape sequences are treated
+/// as atomic tokens so the diff never fragments valid Typst syntax.
 ///
 /// This gives word-level granularity for Latin text while allowing
 /// character-level precision for CJK text (which has no whitespace boundaries).
@@ -193,6 +193,24 @@ fn tokenize_mixed(s: &str) -> Vec<&str> {
                 }
             }
             tokens.push(&s[start..i]);
+        } else if c == '\\' {
+            // An escape sequence is one unit. Split in half, the backslash is
+            // left escaping the diff call written next to it, and the
+            // character it guarded becomes loose markup.
+            let start = i;
+            i += c_len;
+            if i < s.len() {
+                i += s[i..].chars().next().unwrap().len_utf8();
+            }
+            tokens.push(&s[start..i]);
+        } else if c == '`' {
+            // Raw text is literal, so it must never be split: a diff call
+            // written inside the backticks is printed as characters rather
+            // than run, and a backtick parted from its partner leaves the raw
+            // element unterminated.
+            let start = i;
+            i += TypstRaw::end(&s[i..]).unwrap_or(c_len);
+            tokens.push(&s[start..i]);
         } else if c == '<' {
             let start = i;
             if let Some(end) = TypstLabel::end(&s[start..]) {

src/lib.rs

@@ -21,6 +21,27 @@ impl TypstLabel {
     }
 }
 
+pub(crate) struct TypstRaw;
+
+impl TypstRaw {
+    /// Byte length of the raw element at the start of `text`, if there is one.
+    ///
+    /// A run of `n` backticks opens raw text that closes on the next run of `n`
+    /// backticks, except that two backticks are an empty element on their own.
+    /// What lies between is literal: escapes are not applied and brackets do
+    /// not nest, so callers have to copy it through untouched.
+    pub(crate) fn end(text: &str) -> Option<usize> {
+        let open = text.len() - text.trim_start_matches('`').len();
+        match open {
+            0 => None,
+            2 => Some(2),
+            _ => text[open..]
+                .find(&"`".repeat(open))
+                .map(|i| open + i + open),
+        }
+    }
+}
+
 /// A block-level element extracted from a Typst document.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Block {
@@ -159,3 +180,34 @@ pub enum SpanTag {
     Deleted,
     Inserted,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::TypstRaw;
+
+    #[test]
+    fn test_raw_end_single_backtick() {
+        assert_eq!(TypstRaw::end("`a`"), Some(3));
+        // A run of one closes on the next backtick, even next to another.
+        assert_eq!(TypstRaw::end("`a``b`"), Some(3));
+    }
+
+    #[test]
+    fn test_raw_end_two_backticks_are_an_empty_element() {
+        assert_eq!(TypstRaw::end("``a``"), Some(2));
+    }
+
+    #[test]
+    fn test_raw_end_matches_the_opening_run_length() {
+        assert_eq!(TypstRaw::end("```rs x```"), Some(10));
+        // Closes after exactly three, leaving the fourth backtick outside.
+        assert_eq!(TypstRaw::end("```a````"), Some(7));
+    }
+
+    #[test]
+    fn test_raw_end_rejects_unterminated_and_non_raw() {
+        assert_eq!(TypstRaw::end("`a"), None);
+        assert_eq!(TypstRaw::end("```a``"), None);
+        assert_eq!(TypstRaw::end("a`b`"), None);
+    }
+}

src/render.rs

@@ -1,6 +1,6 @@
 use std::fmt::Write;
 
-use crate::{Block, BlockKind, DiffResult, DiffSpan, SpanTag, TypstLabel};
+use crate::{Block, BlockKind, DiffResult, DiffSpan, SpanTag, TypstLabel, TypstRaw};
 
 const PREAMBLE: &str = r##"#let diff-added(body) = {
   set text(fill: rgb("#0000ff"))
@@ -239,6 +239,10 @@ fn write_enum_prefix(number: Option<usize>, out: &mut String) {
 ///
 /// - Tracks bracket depth so that balanced `[...]` pairs are left untouched.
 /// - Unbalanced `]` is escaped as `\]`.
+/// - Backslash escapes already in the source are passed through as-is, so an
+///   escaped character is not escaped a second time.
+/// - Raw text is copied through untouched, since none of the above is markup
+///   inside backticks.
 /// - If the content ends with an odd number of backslashes, a trailing space is
 ///   appended to prevent the closing `]` from being interpreted as `\]`.
 /// - When `escape_refs` is true, `@` is escaped as `\@` and `<` is escaped
@@ -249,8 +253,36 @@ fn write_enum_prefix(number: Option<usize>, out: &mut String) {
 fn escape_content(s: &str, escape_refs: bool) -> String {
     let mut result = String::with_capacity(s.len());
     let mut depth: i32 = 0;
-    for (i, ch) in s.char_indices() {
+    let mut chars = s.char_indices();
+    let mut skip_to = 0;
+    while let Some((i, ch)) = chars.next() {
+        if i < skip_to {
+            continue;
+        }
         match ch {
+            // Raw text is literal all the way to its closing backticks, so
+            // nothing inside it needs escaping and escaping it would show the
+            // backslashes on the page.
+            '`' => match TypstRaw::end(&s[i..]) {
+                Some(end) => {
+                    result.push_str(&s[i..i + end]);
+                    skip_to = i + end;
+                }
+                // A backtick parted from its partner would open raw text that
+                // never closes, so keep it literal.
+                None => {
+                    result.push('\\');
+                    result.push(ch);
+                }
+            },
+            // An escape sequence is already literal; copy it through whole so
+            // the escaped character is not treated as markup below.
+            '\\' => {
+                result.push('\\');
+                if let Some((_, escaped)) = chars.next() {
+                    result.push(escaped);
+                }
+            }
             '[' => {
                 depth += 1;
                 result.push(ch);
@@ -317,6 +349,14 @@ mod tests {
         assert_eq!(escape_content("a ] b", false), "a \\] b");
     }
 
+    #[test]
+    fn test_escape_content_preserves_existing_escapes() {
+        // `\]` is already literal; escaping the backslash would hand the `]`
+        // back to Typst and close the caller's content block early.
+        assert_eq!(escape_content("a \\] b", false), "a \\] b");
+        assert_eq!(escape_content("\\[x", false), "\\[x");
+    }
+
     #[test]
     fn test_escape_content_trailing_backslash() {
         assert_eq!(escape_content("text\\", false), "text\\ ");

tests/integration.rs

@@ -2,6 +2,11 @@ use typdiff::diff::diff;
 use typdiff::parse::parse;
 use typdiff::render::render;
 
+/// True when the rendered output is syntactically valid Typst.
+fn parses(rendered: &str) -> bool {
+    !typst_syntax::parse(rendered).erroneous()
+}
+
 fn run_diff(old: &str, new: &str) -> String {
     let old_blocks: Vec<_> = parse(old)
         .into_iter()
@@ -199,6 +204,83 @@ after the field log was reconciled.
     assert!(!output.contains("#ref(\\<sec-bridge-ledger>"));
 }
 
+#[test]
+fn test_existing_backslash_escape_is_not_double_escaped() {
+    // `\*` and `\[` are already literal. Escaping the backslash instead would
+    // hand the `*`/`[` back to Typst as markup and reopen the delimiter.
+    let old = "Foo bar.\n";
+    let new = "Foo \\*baz* and \\[qux bar.\n";
+    let output = run_diff(old, new);
+
+    assert!(
+        !output.contains("\\\\*") && !output.contains("\\\\["),
+        "an existing escape must not be escaped a second time: {output}"
+    );
+}
+
+#[test]
+fn test_escape_sequence_is_not_split_across_a_diff_call() {
+    // The diff boundary would otherwise fall inside `\]`, stranding the
+    // backslash at the end of the unchanged text where it escapes the `#` of
+    // the call that follows, printing "#diff-deleted[...]" as plain text.
+    let old = "\\[x\\] [grp]\n";
+    let new = "\\[x\\]\n";
+    let output = run_diff(old, new);
+
+    assert!(
+        !output.contains("\\#diff-deleted"),
+        "a stranded backslash must not escape the call: {output}"
+    );
+    assert!(output.contains("\\[x\\]#diff-deleted["), "output: {output}");
+    assert!(parses(&output), "output must compile: {output}");
+}
+
+#[test]
+fn test_raw_span_is_replaced_whole() {
+    // A diff call written between the backticks is printed as characters
+    // rather than run, since raw text is literal.
+    let old = "Walk at `ttIdx` until the `last_tile` bit.\n";
+    let new = "Walk at `tpIdx` until the `last` bit.\n";
+    let output = run_diff(old, new);
+
+    assert!(
+        output.contains("#diff-deleted[`ttIdx`]#diff-added[`tpIdx`]"),
+        "the raw span should be wrapped whole: {output}"
+    );
+    assert!(
+        output.contains("#diff-deleted[`last_tile`]#diff-added[`last`]"),
+        "output: {output}"
+    );
+}
+
+#[test]
+fn test_raw_content_is_not_escaped() {
+    // Inside backticks a `]` does not close the caller's content block and a
+    // backslash is not an escape, so escaping there both is needless and puts
+    // stray characters on the page.
+    let old = "Use `a]b` and `*` here.\n";
+    let new = "Use `zzz` and `*` here.\n";
+    let output = run_diff(old, new);
+
+    assert!(output.contains("#diff-deleted[`a]b`]"), "output: {output}");
+    assert!(!output.contains("\\]b"), "output: {output}");
+    assert!(!output.contains("\\*"), "output: {output}");
+}
+
+#[test]
+fn test_escaped_backtick_does_not_open_raw_text() {
+    // `` \` `` is a literal backtick. Reading it as an opening delimiter shifts
+    // every raw boundary after it and splits the next real raw span.
+    let old = "a \\` b `code` c\n";
+    let new = "a \\` b `xxxx` d\n";
+    let output = run_diff(old, new);
+
+    assert!(
+        output.contains("#diff-deleted[`code`]#diff-added[`xxxx`]"),
+        "the raw span after an escaped backtick must stay intact: {output}"
+    );
+}
+
 #[test]
 fn test_indented_line_comment_does_not_split_paragraph() {
     let old = "First sentence.\n  // indented comment\nSecond sentence.\n";