diff --git a/src/ellipsis.rs b/src/ellipsis.rs index c9db9d82..38b86606 100644 --- a/src/ellipsis.rs +++ b/src/ellipsis.rs @@ -9,14 +9,14 @@ use std::sync::LazyLock; use regex::Regex; -use crate::textproc::{Token, process_tokens}; +use crate::textproc::{Token, process_tokens, push_original_token}; static DOT_RE: LazyLock = lazy_regex!(r"\.{3,}", "ellipsis pattern regex should compile"); /// Replace `...` with `…` outside code spans and fences. #[must_use] pub fn replace_ellipsis(lines: &[String]) -> Vec { - process_tokens(lines, |token, out| match token { + process_tokens(lines, |tok, out| match tok { Token::Text(t) => { if !DOT_RE.is_match(t) { out.push_str(t); @@ -30,13 +30,7 @@ pub fn replace_ellipsis(lines: &[String]) -> Vec { }); out.push_str(&replaced); } - Token::Code(c) => { - out.push('`'); - out.push_str(c); - out.push('`'); - } - Token::Fence(f) => out.push_str(f), - Token::Newline => out.push('\n'), + _ => push_original_token(&tok, out), }) } diff --git a/src/footnotes.rs b/src/footnotes.rs index d4bbd590..e30cba45 100644 --- a/src/footnotes.rs +++ b/src/footnotes.rs @@ -18,7 +18,7 @@ static FOOTNOTE_LINE_RE: LazyLock = lazy_regex!( "footnote line pattern should compile", ); -use crate::textproc::{Token, process_tokens}; +use crate::textproc::{Token, process_tokens, push_original_token}; /// Extract the components of an inline footnote reference. #[inline] @@ -96,15 +96,9 @@ fn convert_block(lines: &mut [String]) { /// Convert bare numeric footnote references to Markdown footnote syntax. #[must_use] pub fn convert_footnotes(lines: &[String]) -> Vec { - let mut lines = process_tokens(lines, |token, out| match token { + let mut lines = process_tokens(lines, |tok, out| match tok { Token::Text(t) => out.push_str(&convert_inline(t)), - Token::Code(c) => { - out.push('`'); - out.push_str(c); - out.push('`'); - } - Token::Fence(f) => out.push_str(f), - Token::Newline => out.push('\n'), + _ => push_original_token(&tok, out), }); convert_block(&mut lines); lines diff --git a/src/textproc.rs b/src/textproc.rs index 74008660..7a30e018 100644 --- a/src/textproc.rs +++ b/src/textproc.rs @@ -8,6 +8,35 @@ pub use crate::wrap::{Token, tokenize_markdown}; +/// Append a [`Token`] to an output buffer without modification. +/// +/// This helper reconstructs a token's original Markdown text. Callers can use +/// it to forward tokens they do not wish to transform while operating on text +/// tokens. +/// +/// # Examples +/// +/// ```rust +/// use mdtablefix::textproc::{Token, push_original_token}; +/// +/// let mut buf = String::new(); +/// push_original_token(&Token::Code("x"), &mut buf); +/// assert_eq!(buf, "`x`"); +/// ``` +#[inline] +pub fn push_original_token(token: &Token<'_>, out: &mut String) { + match token { + Token::Text(t) => out.push_str(t), + Token::Code(c) => { + out.push('`'); + out.push_str(c); + out.push('`'); + } + Token::Fence(f) => out.push_str(f), + Token::Newline => out.push('\n'), + } +} + /// Apply a transformation to a sequence of [`Token`]s. /// /// The `lines` slice is tokenized in order, preserving fence context. @@ -172,4 +201,24 @@ mod tests { ]; assert_eq!(tokens, expected); } + + #[test] + fn push_original_token_roundtrips_all_variants() { + let mut buf = String::new(); + + push_original_token(&Token::Text("a"), &mut buf); + assert_eq!(buf, "a"); + + buf.clear(); + push_original_token(&Token::Code("b"), &mut buf); + assert_eq!(buf, "`b`"); + + buf.clear(); + push_original_token(&Token::Fence("```"), &mut buf); + assert_eq!(buf, "```"); + + buf.clear(); + push_original_token(&Token::Newline, &mut buf); + assert_eq!(buf, "\n"); + } }