Add helper to reemit original tokens

src/ellipsis.rs

@@ -9,14 +9,14 @@ use std::sync::LazyLock;
 
 use regex::Regex;
 
-use crate::textproc::{Token, process_tokens};
+use crate::textproc::{Token, process_tokens, push_original_token};
 
 static DOT_RE: LazyLock<Regex> = lazy_regex!(r"\.{3,}", "ellipsis pattern regex should compile");
 
 /// Replace `...` with `…` outside code spans and fences.
 #[must_use]
 pub fn replace_ellipsis(lines: &[String]) -> Vec<String> {
-    process_tokens(lines, |token, out| match token {
+    process_tokens(lines, |tok, out| match tok {
         Token::Text(t) => {
             if !DOT_RE.is_match(t) {
                 out.push_str(t);
@@ -30,13 +30,7 @@ pub fn replace_ellipsis(lines: &[String]) -> Vec<String> {
             });
             out.push_str(&replaced);
         }
-        Token::Code(c) => {
-            out.push('`');
-            out.push_str(c);
-            out.push('`');
-        }
-        Token::Fence(f) => out.push_str(f),
-        Token::Newline => out.push('\n'),
+        _ => push_original_token(&tok, out),
     })
 }
 

src/footnotes.rs

@@ -18,7 +18,7 @@ static FOOTNOTE_LINE_RE: LazyLock<Regex> = lazy_regex!(
     "footnote line pattern should compile",
 );
 
-use crate::textproc::{Token, process_tokens};
+use crate::textproc::{Token, process_tokens, push_original_token};
 
 /// Extract the components of an inline footnote reference.
 #[inline]
@@ -96,15 +96,9 @@ fn convert_block(lines: &mut [String]) {
 /// Convert bare numeric footnote references to Markdown footnote syntax.
 #[must_use]
 pub fn convert_footnotes(lines: &[String]) -> Vec<String> {
-    let mut lines = process_tokens(lines, |token, out| match token {
+    let mut lines = process_tokens(lines, |tok, out| match tok {
         Token::Text(t) => out.push_str(&convert_inline(t)),
-        Token::Code(c) => {
-            out.push('`');
-            out.push_str(c);
-            out.push('`');
-        }
-        Token::Fence(f) => out.push_str(f),
-        Token::Newline => out.push('\n'),
+        _ => push_original_token(&tok, out),
     });
     convert_block(&mut lines);
     lines

src/textproc.rs

@@ -8,6 +8,35 @@
 
 pub use crate::wrap::{Token, tokenize_markdown};
 
+/// Append a [`Token`] to an output buffer without modification.
+///
+/// This helper reconstructs a token's original Markdown text. Callers can use
+/// it to forward tokens they do not wish to transform while operating on text
+/// tokens.
+///
+/// # Examples
+///
+/// ```rust
+/// use mdtablefix::textproc::{Token, push_original_token};
+///
+/// let mut buf = String::new();
+/// push_original_token(&Token::Code("x"), &mut buf);
+/// assert_eq!(buf, "`x`");
+/// ```
+#[inline]
+pub fn push_original_token(token: &Token<'_>, out: &mut String) {
+    match token {
+        Token::Text(t) => out.push_str(t),
+        Token::Code(c) => {
+            out.push('`');
+            out.push_str(c);
+            out.push('`');
+        }
+        Token::Fence(f) => out.push_str(f),
+        Token::Newline => out.push('\n'),
+    }
+}
+
 /// Apply a transformation to a sequence of [`Token`]s.
 ///
 /// The `lines` slice is tokenized in order, preserving fence context.
@@ -172,4 +201,24 @@ mod tests {
         ];
         assert_eq!(tokens, expected);
     }
+
+    #[test]
+    fn push_original_token_roundtrips_all_variants() {
+        let mut buf = String::new();
+
+        push_original_token(&Token::Text("a"), &mut buf);
+        assert_eq!(buf, "a");
+
+        buf.clear();
+        push_original_token(&Token::Code("b"), &mut buf);
+        assert_eq!(buf, "`b`");
+
+        buf.clear();
+        push_original_token(&Token::Fence("```"), &mut buf);
+        assert_eq!(buf, "```");
+
+        buf.clear();
+        push_original_token(&Token::Newline, &mut buf);
+        assert_eq!(buf, "\n");
+    }
 }