Document markdown tokenization

src/wrap.rs

@@ -186,14 +186,16 @@ fn tokenize_inline(text: &str) -> Vec<String> {
     tokens
 }
 
-/// Split the input string into [`Token`]s by analysing whitespace and
-/// backtick delimiters.
+/// Tokenize a Markdown snippet using backtick-delimited code spans.
 ///
-/// The tokenizer groups consecutive whitespace into a single
-/// [`Token::Text`] and recognises backtick sequences as inline code spans.
-/// When a run of backticks is encountered the parser searches forward for an
-/// identical delimiter, allowing nested backticks when the span uses a longer
-/// fence. Unmatched delimiter sequences are treated as literal text.
+/// The function scans the input line by line. Lines matching [`FENCE_RE`]
+/// produce [`Token::Fence`] tokens and toggle fenced mode. Lines inside a
+/// fence are yielded verbatim. Outside fenced regions the scanner searches for
+/// backtick sequences. Text before a backtick becomes [`Token::Text`]. When a
+/// matching sequence of equal length follows, the enclosed portion forms a
+/// [`Token::Code`] span. If no closing sequence is found the backticks and the
+/// remaining text are returned as [`Token::Text`]. Whitespace is preserved
+/// exactly as it appears.
 ///
 /// ```rust,ignore
 /// use mdtablefix::wrap::{Token, tokenize_markdown};