From 70d7353b591452799ce50af2cac1aac5e626c6ed Mon Sep 17 00:00:00 2001 From: Keshav Dev Sharma Date: Mon, 17 Nov 2025 03:48:34 +0530 Subject: [PATCH 1/9] Fix comments being removed before fmt:off/on blocks Fixes #4843 Black was incorrectly removing comments that precede fmt:off/on blocks, particularly affecting Jupytext's markdown comments like # %% [markdown]. The issue occurred in two places: 1. _handle_comment_only_fmt_block wasn't preserving pre-fmt comments 2. visit_STANDALONE_COMMENT bypassed prefix processing for fmt blocks This fix: - Improves prefix calculation to preserve all pre-fmt:off content - Enhances STANDALONE_COMMENT handling to process prefix comments while preserving fmt block formatting - Adds comprehensive test coverage All existing tests pass and the original issue is resolved. --- CHANGES.md | 3 ++ src/black/comments.py | 20 ++++++++- src/black/linegen.py | 16 ++++++- tests/data/cases/jupytext_markdown_fmt.py | 51 +++++++++++++++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 tests/data/cases/jupytext_markdown_fmt.py diff --git a/CHANGES.md b/CHANGES.md index fd54b5b8eef..ca47a656758 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -13,6 +13,9 @@ +- Fix bug where comments preceding `# fmt: off`/`# fmt: on` blocks were incorrectly + removed, particularly affecting Jupytext's `# %% [markdown]` comments (#4843) + ### Preview style diff --git a/src/black/comments.py b/src/black/comments.py index 9a89c85e779..f7b1ef0a25b 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -281,9 +281,18 @@ def _handle_comment_only_fmt_block( if hidden_value.endswith("\n"): hidden_value = hidden_value[:-1] - # Build the standalone comment prefix + # Build the standalone comment prefix - preserve all content before fmt:off + # including any comments that precede it + if fmt_off_idx == 0: + # No comments before fmt:off, use previous_consumed + pre_fmt_off_consumed = previous_consumed + else: + # Use the consumed position of the last comment before fmt:off + # This preserves all comments and content before the fmt:off directive + pre_fmt_off_consumed = all_comments[fmt_off_idx - 1].consumed + standalone_comment_prefix = ( - original_prefix[:previous_consumed] + "\n" * comment.newlines + original_prefix[:pre_fmt_off_consumed] + "\n" * comment.newlines ) fmt_off_prefix = original_prefix.split(comment.value)[0] @@ -326,6 +335,13 @@ def convert_one_fmt_off_pair( Returns True if a pair was converted. """ for leaf in node.leaves(): + # Skip STANDALONE_COMMENT nodes that were created by fmt:off/on processing + # to avoid reprocessing them in subsequent iterations + if (leaf.type == STANDALONE_COMMENT and + hasattr(leaf, 'fmt_pass_converted_first_leaf') and + leaf.fmt_pass_converted_first_leaf is None): + continue + previous_consumed = 0 for comment in list_comments(leaf.prefix, is_endmarker=False, mode=mode): should_process, is_fmt_off, is_fmt_skip = _should_process_fmt_comment( diff --git a/src/black/linegen.py b/src/black/linegen.py index cf171c12c62..5cd45ab69b2 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -405,7 +405,21 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: else: is_fmt_off_block = False if is_fmt_off_block: - # This is a fmt:off/on block from normalize_fmt_off - append directly + # This is a fmt:off/on block from normalize_fmt_off - we still need + # to process any prefix comments (like markdown comments) but append + # the fmt block itself directly to preserve its formatting + + # Only process prefix comments if there actually is a prefix with comments + if leaf.prefix and any(line.strip().startswith('#') and + not line.strip().lstrip('#').strip() in ('fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable') + for line in leaf.prefix.split('\n')): + for comment in generate_comments(leaf, mode=self.mode): + yield from self.line() + self.current_line.append(comment) + yield from self.line() + # Clear the prefix since we've processed it as comments above + leaf.prefix = "" + self.current_line.append(leaf) yield from self.line() else: diff --git a/tests/data/cases/jupytext_markdown_fmt.py b/tests/data/cases/jupytext_markdown_fmt.py new file mode 100644 index 00000000000..0dab2fd4ec4 --- /dev/null +++ b/tests/data/cases/jupytext_markdown_fmt.py @@ -0,0 +1,51 @@ +# Test that Jupytext markdown comments are preserved before fmt:off/on blocks +# %% [markdown] + +# fmt: off +# fmt: on + +# Also test with other comments +# Some comment +# %% [markdown] +# Another comment + +# fmt: off +x = 1 +# fmt: on + +# Test multiple markdown comments +# %% [markdown] +# First markdown +# %% [code] +# Code cell + +# fmt: off +y = 2 +# fmt: on + +# output + +# Test that Jupytext markdown comments are preserved before fmt:off/on blocks +# %% [markdown] + +# fmt: off +# fmt: on + +# Also test with other comments +# Some comment +# %% [markdown] +# Another comment + +# fmt: off +x = 1 +# fmt: on + +# Test multiple markdown comments +# %% [markdown] +# First markdown +# %% [code] +# Code cell + +# fmt: off +y = 2 +# fmt: on \ No newline at end of file From bb94598fb1257056c778eb3ac8e3a40bf51ba2af Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 16 Nov 2025 22:30:42 +0000 Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/black/comments.py | 6 +++--- src/black/linegen.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/black/comments.py b/src/black/comments.py index f7b1ef0a25b..b14bec29ce7 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -337,11 +337,11 @@ def convert_one_fmt_off_pair( for leaf in node.leaves(): # Skip STANDALONE_COMMENT nodes that were created by fmt:off/on processing # to avoid reprocessing them in subsequent iterations - if (leaf.type == STANDALONE_COMMENT and - hasattr(leaf, 'fmt_pass_converted_first_leaf') and + if (leaf.type == STANDALONE_COMMENT and + hasattr(leaf, 'fmt_pass_converted_first_leaf') and leaf.fmt_pass_converted_first_leaf is None): continue - + previous_consumed = 0 for comment in list_comments(leaf.prefix, is_endmarker=False, mode=mode): should_process, is_fmt_off, is_fmt_skip = _should_process_fmt_comment( diff --git a/src/black/linegen.py b/src/black/linegen.py index 5cd45ab69b2..081d4546780 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -406,11 +406,11 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: is_fmt_off_block = False if is_fmt_off_block: # This is a fmt:off/on block from normalize_fmt_off - we still need - # to process any prefix comments (like markdown comments) but append + # to process any prefix comments (like markdown comments) but append # the fmt block itself directly to preserve its formatting - + # Only process prefix comments if there actually is a prefix with comments - if leaf.prefix and any(line.strip().startswith('#') and + if leaf.prefix and any(line.strip().startswith('#') and not line.strip().lstrip('#').strip() in ('fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable') for line in leaf.prefix.split('\n')): for comment in generate_comments(leaf, mode=self.mode): @@ -419,7 +419,7 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: yield from self.line() # Clear the prefix since we've processed it as comments above leaf.prefix = "" - + self.current_line.append(leaf) yield from self.line() else: From 40bb99b281c54374e745c26cfbf577427461299f Mon Sep 17 00:00:00 2001 From: Keshav Dev Sharma Date: Mon, 17 Nov 2025 04:10:40 +0530 Subject: [PATCH 3/9] Fix flake8 linting issues - Fix line break and indentation issues in comments.py - Fix line length and formatting issues in linegen.py - Split long condition for better readability --- src/black/comments.py | 8 +++++--- src/black/linegen.py | 9 ++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/black/comments.py b/src/black/comments.py index f7b1ef0a25b..03b7e13f5b9 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -337,9 +337,11 @@ def convert_one_fmt_off_pair( for leaf in node.leaves(): # Skip STANDALONE_COMMENT nodes that were created by fmt:off/on processing # to avoid reprocessing them in subsequent iterations - if (leaf.type == STANDALONE_COMMENT and - hasattr(leaf, 'fmt_pass_converted_first_leaf') and - leaf.fmt_pass_converted_first_leaf is None): + if ( + leaf.type == STANDALONE_COMMENT + and hasattr(leaf, 'fmt_pass_converted_first_leaf') + and leaf.fmt_pass_converted_first_leaf is None + ): continue previous_consumed = 0 diff --git a/src/black/linegen.py b/src/black/linegen.py index 5cd45ab69b2..5d0827e032a 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -410,9 +410,12 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: # the fmt block itself directly to preserve its formatting # Only process prefix comments if there actually is a prefix with comments - if leaf.prefix and any(line.strip().startswith('#') and - not line.strip().lstrip('#').strip() in ('fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable') - for line in leaf.prefix.split('\n')): + fmt_directives = {'fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable'} + if leaf.prefix and any( + line.strip().startswith('#') + and line.strip().lstrip('#').strip() not in fmt_directives + for line in leaf.prefix.split('\n') + ): for comment in generate_comments(leaf, mode=self.mode): yield from self.line() self.current_line.append(comment) From 3e16a3f8cf05f7001628f20781e532c7972c91a7 Mon Sep 17 00:00:00 2001 From: Keshav Dev Sharma Date: Mon, 17 Nov 2025 04:14:13 +0530 Subject: [PATCH 4/9] Fix merge conflicts and syntax errors - Resolve merge conflict markers in linegen.py - Fix indentation issues in comments.py - All syntax errors now resolved --- src/black/comments.py | 6 ------ src/black/linegen.py | 11 ++++------- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/black/comments.py b/src/black/comments.py index 0c18b17cc4f..7c8397229c6 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -337,17 +337,11 @@ def convert_one_fmt_off_pair( for leaf in node.leaves(): # Skip STANDALONE_COMMENT nodes that were created by fmt:off/on processing # to avoid reprocessing them in subsequent iterations -<<<<<<< HEAD if ( leaf.type == STANDALONE_COMMENT and hasattr(leaf, 'fmt_pass_converted_first_leaf') and leaf.fmt_pass_converted_first_leaf is None ): -======= - if (leaf.type == STANDALONE_COMMENT and - hasattr(leaf, 'fmt_pass_converted_first_leaf') and - leaf.fmt_pass_converted_first_leaf is None): ->>>>>>> bb94598fb1257056c778eb3ac8e3a40bf51ba2af continue previous_consumed = 0 diff --git a/src/black/linegen.py b/src/black/linegen.py index c857c4838d0..6d8c4cc6ef6 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -410,18 +410,15 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: # the fmt block itself directly to preserve its formatting # Only process prefix comments if there actually is a prefix with comments -<<<<<<< HEAD - fmt_directives = {'fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable'} + fmt_directives = { + 'fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', + 'yapf: disable', 'yapf: enable' + } if leaf.prefix and any( line.strip().startswith('#') and line.strip().lstrip('#').strip() not in fmt_directives for line in leaf.prefix.split('\n') ): -======= - if leaf.prefix and any(line.strip().startswith('#') and - not line.strip().lstrip('#').strip() in ('fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable') - for line in leaf.prefix.split('\n')): ->>>>>>> bb94598fb1257056c778eb3ac8e3a40bf51ba2af for comment in generate_comments(leaf, mode=self.mode): yield from self.line() self.current_line.append(comment) From b87b7ac6cd9ced0ba8a79f8275f2d2129c40e57b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 16 Nov 2025 22:45:10 +0000 Subject: [PATCH 5/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/black/linegen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black/linegen.py b/src/black/linegen.py index 6d8c4cc6ef6..15fdbcb2b78 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -411,7 +411,7 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: # Only process prefix comments if there actually is a prefix with comments fmt_directives = { - 'fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', + 'fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', 'yapf: disable', 'yapf: enable' } if leaf.prefix and any( From ee9832e9ca815db6cd0e7c8713ddd29c2b86f218 Mon Sep 17 00:00:00 2001 From: Keshav Dev Sharma Date: Mon, 17 Nov 2025 09:57:06 +0530 Subject: [PATCH 6/9] Address review feedback - Update CHANGES.md to reference PR #4845 instead of issue #4843 - Used existing _contains_fmt_directive function instead of custom logic - Import _contains_fmt_directive from black.comments --- CHANGES.md | 2 +- src/black/linegen.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ca47a656758..5344100c563 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,7 +14,7 @@ - Fix bug where comments preceding `# fmt: off`/`# fmt: on` blocks were incorrectly - removed, particularly affecting Jupytext's `# %% [markdown]` comments (#4843) + removed, particularly affecting Jupytext's `# %% [markdown]` comments (#4845) ### Preview style diff --git a/src/black/linegen.py b/src/black/linegen.py index 15fdbcb2b78..cfe6e985f69 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -17,7 +17,7 @@ get_leaves_inside_matching_brackets, max_delimiter_priority_in_atom, ) -from black.comments import FMT_OFF, FMT_ON, generate_comments, list_comments +from black.comments import FMT_OFF, FMT_ON, _contains_fmt_directive, generate_comments, list_comments from black.lines import ( Line, RHSResult, @@ -410,13 +410,9 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: # the fmt block itself directly to preserve its formatting # Only process prefix comments if there actually is a prefix with comments - fmt_directives = { - 'fmt: off', 'fmt:off', 'fmt: on', 'fmt:on', - 'yapf: disable', 'yapf: enable' - } if leaf.prefix and any( line.strip().startswith('#') - and line.strip().lstrip('#').strip() not in fmt_directives + and not _contains_fmt_directive(line.strip()) for line in leaf.prefix.split('\n') ): for comment in generate_comments(leaf, mode=self.mode): From 2ccc86e25a8e156a878f2b9c3094daf60cde5756 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Nov 2025 04:27:38 +0000 Subject: [PATCH 7/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/black/linegen.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/black/linegen.py b/src/black/linegen.py index cfe6e985f69..a5fea484e6d 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -17,7 +17,13 @@ get_leaves_inside_matching_brackets, max_delimiter_priority_in_atom, ) -from black.comments import FMT_OFF, FMT_ON, _contains_fmt_directive, generate_comments, list_comments +from black.comments import ( + FMT_OFF, + FMT_ON, + _contains_fmt_directive, + generate_comments, + list_comments, +) from black.lines import ( Line, RHSResult, From 0c619a72bed1c67b78c98d99be17a21d14b6f1ba Mon Sep 17 00:00:00 2001 From: Keshav Dev Sharma Date: Tue, 18 Nov 2025 15:31:35 +0530 Subject: [PATCH 8/9] Run Black formatter on comments.py and linegen.py --- src/black/comments.py | 2 +- src/black/linegen.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/black/comments.py b/src/black/comments.py index 3a0bbaf67a6..13c9926d03b 100644 --- a/src/black/comments.py +++ b/src/black/comments.py @@ -339,7 +339,7 @@ def convert_one_fmt_off_pair( # to avoid reprocessing them in subsequent iterations if ( leaf.type == STANDALONE_COMMENT - and hasattr(leaf, 'fmt_pass_converted_first_leaf') + and hasattr(leaf, "fmt_pass_converted_first_leaf") and leaf.fmt_pass_converted_first_leaf is None ): continue diff --git a/src/black/linegen.py b/src/black/linegen.py index 5ef87efe6eb..98776a0dbf9 100644 --- a/src/black/linegen.py +++ b/src/black/linegen.py @@ -417,9 +417,9 @@ def visit_STANDALONE_COMMENT(self, leaf: Leaf) -> Iterator[Line]: # Only process prefix comments if there actually is a prefix with comments if leaf.prefix and any( - line.strip().startswith('#') + line.strip().startswith("#") and not _contains_fmt_directive(line.strip()) - for line in leaf.prefix.split('\n') + for line in leaf.prefix.split("\n") ): for comment in generate_comments(leaf, mode=self.mode): yield from self.line() From b0f1a491ebf3d6789e0e94d8d294edfd1fa6e586 Mon Sep 17 00:00:00 2001 From: cobalt <61329810+cobaltt7@users.noreply.github.com> Date: Fri, 21 Nov 2025 16:07:37 -0600 Subject: [PATCH 9/9] Update tests/data/cases/jupytext_markdown_fmt.py --- tests/data/cases/jupytext_markdown_fmt.py | 27 ----------------------- 1 file changed, 27 deletions(-) diff --git a/tests/data/cases/jupytext_markdown_fmt.py b/tests/data/cases/jupytext_markdown_fmt.py index 0dab2fd4ec4..43e94630047 100644 --- a/tests/data/cases/jupytext_markdown_fmt.py +++ b/tests/data/cases/jupytext_markdown_fmt.py @@ -19,33 +19,6 @@ # %% [code] # Code cell -# fmt: off -y = 2 -# fmt: on - -# output - -# Test that Jupytext markdown comments are preserved before fmt:off/on blocks -# %% [markdown] - -# fmt: off -# fmt: on - -# Also test with other comments -# Some comment -# %% [markdown] -# Another comment - -# fmt: off -x = 1 -# fmt: on - -# Test multiple markdown comments -# %% [markdown] -# First markdown -# %% [code] -# Code cell - # fmt: off y = 2 # fmt: on \ No newline at end of file