From fc1956ce75a934eb441b0ec90fadb1e9b01d64ec Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 2 Aug 2022 14:31:47 +0200 Subject: [PATCH 1/3] add test case for pygments + div --- test/tm-cases/fenced_code_blocks_issue462.html | 10 ++++++++++ test/tm-cases/fenced_code_blocks_issue462.opts | 1 + test/tm-cases/fenced_code_blocks_issue462.tags | 1 + test/tm-cases/fenced_code_blocks_issue462.text | 11 +++++++++++ 4 files changed, 23 insertions(+) create mode 100644 test/tm-cases/fenced_code_blocks_issue462.html create mode 100644 test/tm-cases/fenced_code_blocks_issue462.opts create mode 100644 test/tm-cases/fenced_code_blocks_issue462.tags create mode 100644 test/tm-cases/fenced_code_blocks_issue462.text diff --git a/test/tm-cases/fenced_code_blocks_issue462.html b/test/tm-cases/fenced_code_blocks_issue462.html new file mode 100644 index 00000000..54a1f8f2 --- /dev/null +++ b/test/tm-cases/fenced_code_blocks_issue462.html @@ -0,0 +1,10 @@ +

Section 1

+ +
x = 1
+
+ +

Section 2

+ +
+test +
diff --git a/test/tm-cases/fenced_code_blocks_issue462.opts b/test/tm-cases/fenced_code_blocks_issue462.opts new file mode 100644 index 00000000..20052b21 --- /dev/null +++ b/test/tm-cases/fenced_code_blocks_issue462.opts @@ -0,0 +1 @@ +{"extras": ["fenced-code-blocks", "pygments"]} diff --git a/test/tm-cases/fenced_code_blocks_issue462.tags b/test/tm-cases/fenced_code_blocks_issue462.tags new file mode 100644 index 00000000..2c03fb5d --- /dev/null +++ b/test/tm-cases/fenced_code_blocks_issue462.tags @@ -0,0 +1 @@ +extra fenced-code-blocks pygments diff --git a/test/tm-cases/fenced_code_blocks_issue462.text b/test/tm-cases/fenced_code_blocks_issue462.text new file mode 100644 index 00000000..1a01c576 --- /dev/null +++ b/test/tm-cases/fenced_code_blocks_issue462.text @@ -0,0 +1,11 @@ +# Section 1 + +```python +x = 1 +``` + +# Section 2 + +
+test +
From 71c94f5ffe08c57743657734d70e1b69e88e1163 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 2 Aug 2022 14:32:27 +0200 Subject: [PATCH 2/3] make sure that pygments blocks are picked up by HTML block detection --- lib/markdown2.py | 10 +++++-- .../admonitions_with_fenced_code_blocks.html | 30 ++++++++++++------- .../tm-cases/fenced_code_blocks_issue355.html | 6 ++-- .../tm-cases/fenced_code_blocks_issue426.html | 12 +++++--- .../tm-cases/fenced_code_blocks_issue462.html | 6 ++-- ...fenced_code_blocks_leading_lang_space.html | 6 ++-- .../fenced_code_blocks_safe_highlight.html | 12 +++++--- ...enced_code_blocks_syntax_highlighting.html | 12 +++++--- ...fenced_code_blocks_syntax_indentation.html | 6 ++-- .../issue276_fenced_code_blocks_in_lists.html | 6 ++-- test/tm-cases/issue3_bad_code_color_hack.html | 6 ++-- .../pyshell_and_fenced_code_blocks.html | 18 +++++++---- ...ocks_whitespace_around_indented_lines.html | 6 ++-- test/tm-cases/syntax_color.html | 12 +++++--- test/tm-cases/syntax_color_opts.html | 12 +++++--- 15 files changed, 108 insertions(+), 52 deletions(-) diff --git a/lib/markdown2.py b/lib/markdown2.py index 7ea827a4..065188c2 100755 --- a/lib/markdown2.py +++ b/lib/markdown2.py @@ -1864,14 +1864,20 @@ def _wrap_code(self, inner): yield tup yield 0, "" + def _add_newline(self, inner): + # Add newlines around the inner contents so that _strict_tag_block_re matches the outer div. + yield 0, "\n" + yield from inner + yield 0, "\n" + def wrap(self, source, outfile=None): """Return the source with a code, pre, and div.""" if outfile is None: # pygments >= 2.12 - return self._wrap_pre(self._wrap_code(source)) + return self._add_newline(self._wrap_pre(self._wrap_code(source))) else: # pygments < 2.12 - return self._wrap_div(self._wrap_pre(self._wrap_code(source))) + return self._wrap_div(self._add_newline(self._wrap_pre(self._wrap_code(source)))) formatter_opts.setdefault("cssclass", "codehilite") formatter = HtmlCodeFormatter(**formatter_opts) diff --git a/test/tm-cases/admonitions_with_fenced_code_blocks.html b/test/tm-cases/admonitions_with_fenced_code_blocks.html index 6946ba16..7428c571 100644 --- a/test/tm-cases/admonitions_with_fenced_code_blocks.html +++ b/test/tm-cases/admonitions_with_fenced_code_blocks.html @@ -1,25 +1,35 @@ -
# admonitions WITHIN fenced code blocks should NOT be rendered
+

+
# admonitions WITHIN fenced code blocks should NOT be rendered
 .. attention:: title
    body
-

+
+
diff --git a/test/tm-cases/fenced_code_blocks_issue355.html b/test/tm-cases/fenced_code_blocks_issue355.html index 66cc154e..6696ede4 100644 --- a/test/tm-cases/fenced_code_blocks_issue355.html +++ b/test/tm-cases/fenced_code_blocks_issue355.html @@ -1,5 +1,7 @@ -
some code block
-
+
+
some code block
+
+
yet another code block
diff --git a/test/tm-cases/fenced_code_blocks_issue426.html b/test/tm-cases/fenced_code_blocks_issue426.html index 152711ba..66b1cb9c 100644 --- a/test/tm-cases/fenced_code_blocks_issue426.html +++ b/test/tm-cases/fenced_code_blocks_issue426.html @@ -14,18 +14,22 @@

URL PARAMETERS IN THE TEMPLATE

  • All views (except generic.View) from django.forms.generic inherit from ContextMixin

  • ContextMixin defines the method get_context_data:

    -
    def get_context_data(self, **kwargs):
+
    
+
    def get_context_data(self, **kwargs):
     kwargs.setdefault('view', self)
     if self.extra_context is not None:
         kwargs.update(self.extra_context)
     return kwargs
-
    
+
    +

    So when overriding one must be careful to extends super's kwargs:

    -
    def get_context_data(self, **kwargs):
+
    
+
    def get_context_data(self, **kwargs):
     kwargs = super().get_context_data(**kwargs)
     kwargs['page_title'] = "Documentation"
     return kwargs
-
    • + + diff --git a/test/tm-cases/fenced_code_blocks_issue462.html b/test/tm-cases/fenced_code_blocks_issue462.html index 54a1f8f2..6049aa67 100644 --- a/test/tm-cases/fenced_code_blocks_issue462.html +++ b/test/tm-cases/fenced_code_blocks_issue462.html @@ -1,7 +1,9 @@

    Section 1

    -
    x = 1
-
    +
    +
    x = 1
+
    +

    Section 2

    diff --git a/test/tm-cases/fenced_code_blocks_leading_lang_space.html b/test/tm-cases/fenced_code_blocks_leading_lang_space.html index a61524b3..f5815e5e 100644 --- a/test/tm-cases/fenced_code_blocks_leading_lang_space.html +++ b/test/tm-cases/fenced_code_blocks_leading_lang_space.html @@ -1,3 +1,5 @@ -
    if True:
+
    
+
    if True:
     print "hi"
-
    
+
    +
    diff --git a/test/tm-cases/fenced_code_blocks_safe_highlight.html b/test/tm-cases/fenced_code_blocks_safe_highlight.html index a08985a7..105bd8c9 100644 --- a/test/tm-cases/fenced_code_blocks_safe_highlight.html +++ b/test/tm-cases/fenced_code_blocks_safe_highlight.html @@ -1,12 +1,16 @@ -
    if True:
+
    
+
    if True:
     print "hi"
-
    
+
    +

    That's using the fenced-code-blocks extra with Python syntax coloring, if pygments is installed. See http://github.github.com/github-flavored-markdown/.

    -
    def foo
+
    
+
    def foo
     puts "hi"
 end
-
    
+
    +
    diff --git a/test/tm-cases/fenced_code_blocks_syntax_highlighting.html b/test/tm-cases/fenced_code_blocks_syntax_highlighting.html index a08985a7..105bd8c9 100644 --- a/test/tm-cases/fenced_code_blocks_syntax_highlighting.html +++ b/test/tm-cases/fenced_code_blocks_syntax_highlighting.html @@ -1,12 +1,16 @@ -
    if True:
+
    
+
    if True:
     print "hi"
-
    
+
    +

    That's using the fenced-code-blocks extra with Python syntax coloring, if pygments is installed. See http://github.github.com/github-flavored-markdown/.

    -
    def foo
+
    
+
    def foo
     puts "hi"
 end
-
    
+
    +
    diff --git a/test/tm-cases/fenced_code_blocks_syntax_indentation.html b/test/tm-cases/fenced_code_blocks_syntax_indentation.html index 37b5723e..f1a37817 100644 --- a/test/tm-cases/fenced_code_blocks_syntax_indentation.html +++ b/test/tm-cases/fenced_code_blocks_syntax_indentation.html @@ -1,5 +1,7 @@ -
    def foo():
+
    
+
    def foo():
     print "foo"
 
     print "bar"
-
    
+
    +
    diff --git a/test/tm-cases/issue276_fenced_code_blocks_in_lists.html b/test/tm-cases/issue276_fenced_code_blocks_in_lists.html index d69487e9..7154404c 100644 --- a/test/tm-cases/issue276_fenced_code_blocks_in_lists.html +++ b/test/tm-cases/issue276_fenced_code_blocks_in_lists.html @@ -12,8 +12,10 @@

    empty codeblock just for sh*ts and giggles

    -
    test with language set
-
    +
    +
    test with language set
+
    +
    This is a regular code block
 Multiline
diff --git a/test/tm-cases/issue3_bad_code_color_hack.html b/test/tm-cases/issue3_bad_code_color_hack.html
index 8aedb76c..2fff7bfb 100644
--- a/test/tm-cases/issue3_bad_code_color_hack.html
+++ b/test/tm-cases/issue3_bad_code_color_hack.html
@@ -6,7 +6,9 @@ 
    заголовок
    
 
 
    Some python code:
    
 
-
    # комментарий
+
    
+
    # комментарий
 if True:
     print "hi"
-
    
+
    
+
    
diff --git a/test/tm-cases/pyshell_and_fenced_code_blocks.html b/test/tm-cases/pyshell_and_fenced_code_blocks.html
index a9fc2cf6..a21b29bb 100644
--- a/test/tm-cases/pyshell_and_fenced_code_blocks.html
+++ b/test/tm-cases/pyshell_and_fenced_code_blocks.html
@@ -2,20 +2,26 @@ 
    From Recipe 302035
    
 
 
    Some examples:
    
 
-
    >>> nprint(9876543210)
+
    
+
    >>> nprint(9876543210)
 '9 876 543 210'
 >>> nprint(987654321, period=1, delimiter=",")
 '9,8,7,6,5,4,3,2,1,0'
-
    
+
    
+
    
 
 
    Indented a bit:
    
 
-
    >>> 1 + 1
+
    
+
    >>> 1 + 1
 2
-
    
+
    
+
    
 
 
    Cuddled to previous para (and at end of document):
    
 
-
    >>> 2 + 2
+
    
+
    >>> 2 + 2
 4
-
    
+
    
+
    
diff --git a/test/tm-cases/quoted_fenced_code_blocks_whitespace_around_indented_lines.html b/test/tm-cases/quoted_fenced_code_blocks_whitespace_around_indented_lines.html
index 6f344f05..58d5d439 100644
--- a/test/tm-cases/quoted_fenced_code_blocks_whitespace_around_indented_lines.html
+++ b/test/tm-cases/quoted_fenced_code_blocks_whitespace_around_indented_lines.html
@@ -1,7 +1,8 @@
 
    Example:
    
 
 
    
-  
    if True:
+  
    
+
    if True:
     print()
 
     print()
@@ -9,5 +10,6 @@ 
    Example:
    
     print()
 
     print()
-
    
+
    
+  
    
 
    
diff --git a/test/tm-cases/syntax_color.html b/test/tm-cases/syntax_color.html
index ae9ea34d..e2d77d6c 100644
--- a/test/tm-cases/syntax_color.html
+++ b/test/tm-cases/syntax_color.html
@@ -1,15 +1,19 @@
 
    Here is some sample code:
    
 
-
    import sys
+
    
+
    import sys
 def main(argv=sys.argv):
     logging.basicConfig()
     log.info('hi')
-
    
+
    
+
    
 
 
    and:
    
 
-
    use 'zlib'
+
    
+
    use 'zlib'
 sub main(argv)
     puts 'hi'
 end
-
    
+
    
+
    
diff --git a/test/tm-cases/syntax_color_opts.html b/test/tm-cases/syntax_color_opts.html
index a44e49d2..7ac05d15 100644
--- a/test/tm-cases/syntax_color_opts.html
+++ b/test/tm-cases/syntax_color_opts.html
@@ -1,15 +1,19 @@
 
    Here is some sample code:
    
 
-
    import sys
+
    
+
    import sys
 def main(argv=sys.argv):
     logging.basicConfig()
     log.info('hi')
-
    
+
    
+
    
 
 
    and:
    
 
-
    use 'zlib'
+
    
+
    use 'zlib'
 sub main(argv)
     puts 'hi'
 end
-
    
+
    
+
    

From f1634cc0b1e1de6015050860e684eccea06f0f5c Mon Sep 17 00:00:00 2001
From: Maximilian Hils 
Date: Thu, 11 Aug 2022 15:58:00 +0200
Subject: [PATCH 3/3] fix pyshell blocks in blockquotes, refs
 mitmproxy/pdoc#429

---
 CHANGES.md                                        |  2 ++
 lib/markdown2.py                                  |  4 ++--
 test/tm-cases/pyshell_and_fenced_code_blocks.html | 10 ++++++++++
 test/tm-cases/pyshell_and_fenced_code_blocks.text |  5 +++++
 4 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index ddd7490e..f5dd3b47 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -15,6 +15,8 @@
 - [pull #455] Fix code block indentation in lists
 - [pull #434] Fix filter bypass leading to XSS (#362)
 - [pull #464] Fix html-classes extra not applying to code spans
+- [pull #462] Fix pygments block matching
+- [pull #462] Fix pyshell blocks in blockquotes
 
 
 ## python-markdown2 2.4.3
diff --git a/lib/markdown2.py b/lib/markdown2.py
index 065188c2..4ab5e4d7 100755
--- a/lib/markdown2.py
+++ b/lib/markdown2.py
@@ -1056,7 +1056,7 @@ def _pyshell_block_sub(self, match):
         indent = ' ' * self.tab_width
         s = ('\n'  # separate from possible cuddled paragraph
              + indent + ('\n'+indent).join(lines)
-             + '\n\n')
+             + '\n')
         return s
 
     def _prepare_pyshell_blocks(self, text):
@@ -1070,7 +1070,7 @@ def _prepare_pyshell_blocks(self, text):
         _pyshell_block_re = re.compile(r"""
             ^([ ]{0,%d})>>>[ ].*\n  # first line
             ^(\1[^\S\n]*\S.*\n)*    # any number of subsequent lines with at least one character
-            ^\n                     # ends with a blank line
+            (?=^\1?\n|\Z)           # ends with a blank line or end of document
             """ % less_than_tab, re.M | re.X)
 
         return _pyshell_block_re.sub(self._pyshell_block_sub, text)
diff --git a/test/tm-cases/pyshell_and_fenced_code_blocks.html b/test/tm-cases/pyshell_and_fenced_code_blocks.html
index a21b29bb..28b13705 100644
--- a/test/tm-cases/pyshell_and_fenced_code_blocks.html
+++ b/test/tm-cases/pyshell_and_fenced_code_blocks.html
@@ -18,6 +18,16 @@ 
    From Recipe 302035
    +

    Part of blockquote:

    + +
    +
    +
    >>> 1 + 1
+2
+
    +
    +
    +

    Cuddled to previous para (and at end of document):

    diff --git a/test/tm-cases/pyshell_and_fenced_code_blocks.text b/test/tm-cases/pyshell_and_fenced_code_blocks.text index fc1eaae4..f718ec9e 100644 --- a/test/tm-cases/pyshell_and_fenced_code_blocks.text +++ b/test/tm-cases/pyshell_and_fenced_code_blocks.text @@ -12,6 +12,11 @@ Indented a bit: >>> 1 + 1 2 +Part of blockquote: + +> >>> 1 + 1 +> 2 + Cuddled to previous para (and at end of document): >>> 2 + 2 4