From 3b3dff8a4e1166c21878f691f7f5ac8870a47df4 Mon Sep 17 00:00:00 2001 From: Leynos Date: Mon, 21 Jul 2025 02:58:47 +0100 Subject: [PATCH 1/2] Document tokenisation logic --- README.md | 1 - src/wrap.rs | 19 ++++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 85cc9a2e..3e2432d0 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,6 @@ cargo install --path . ## Command-line usage - ```bash mdtablefix [--wrap] [--renumber] [--breaks] [--ellipsis] [--fences] [--footnotes] [--in-place] [FILE...] ``` diff --git a/src/wrap.rs b/src/wrap.rs index 624d0dc4..f2f572e9 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -118,7 +118,24 @@ fn tokenize_inline(text: &str) -> Vec { tokens } -/// Tokenise Markdown into fences, inline code and plain text. +/// Split the input string into [`Token`]s by analysing whitespace and +/// backtick delimiters. +/// +/// The tokenizer groups consecutive whitespace into a single +/// [`Token::Text`] and recognises backtick sequences as inline code spans. +/// When a run of backticks is encountered the parser searches forward for an +/// identical delimiter, allowing nested backticks when the span uses a longer +/// fence. Unmatched delimiter sequences are treated as literal text. +/// +/// ``` +/// use mdtablefix::wrap::{Token, tokenize_markdown}; +/// +/// let tokens = tokenize_markdown("Example with `code`"); +/// assert_eq!( +/// tokens, +/// vec![Token::Text("Example with "), Token::Code("code")] +/// ); +/// ``` pub(crate) fn tokenize_markdown(input: &str) -> Vec> { let mut out = Vec::new(); let mut in_fence = false; From b566853c6a19d0a9f5da01394a182b6affc42552 Mon Sep 17 00:00:00 2001 From: Leynos Date: Tue, 22 Jul 2025 00:43:12 +0100 Subject: [PATCH 2/2] Document tokenizer logic --- README.md | 9 +++++---- src/wrap.rs | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3e2432d0..82d28942 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,8 @@ A brief intermission for pizza. ## Library usage -The crate exposes helper functions for embedding the table-reflow logic in -Rust projects: +The crate exposes helper functions for embedding the table-reflow logic in Rust +projects: ```rust use mdtablefix::{process_stream_opts, rewrite, Options}; @@ -158,8 +158,9 @@ For an overview of how the crate's internal modules relate to each other, see ## Testing -The test suite is structured using the `rstest` crate. See [Rust testing with -rstest fixtures](docs/rust-testing-with-rstest-fixtures.md) for details. +The test suite is structured using the `rstest` crate. See +[Rust testing with rstest fixtures](docs/rust-testing-with-rstest-fixtures.md) +for details. ## License diff --git a/src/wrap.rs b/src/wrap.rs index f2f572e9..3a5ae7fe 100644 --- a/src/wrap.rs +++ b/src/wrap.rs @@ -127,7 +127,7 @@ fn tokenize_inline(text: &str) -> Vec { /// identical delimiter, allowing nested backticks when the span uses a longer /// fence. Unmatched delimiter sequences are treated as literal text. /// -/// ``` +/// ```rust,ignore /// use mdtablefix::wrap::{Token, tokenize_markdown}; /// /// let tokens = tokenize_markdown("Example with `code`");