Fix pygments block matching

CHANGES.md

@@ -15,6 +15,8 @@
 - [pull #455] Fix code block indentation in lists
 - [pull #434] Fix filter bypass leading to XSS (#362)
 - [pull #464] Fix html-classes extra not applying to code spans
+- [pull #462] Fix pygments block matching
+- [pull #462] Fix pyshell blocks in blockquotes
 - [pull #463] Fix multilevel lists
 
 

lib/markdown2.py

@@ -1056,7 +1056,7 @@ def _pyshell_block_sub(self, match):
         indent = ' ' * self.tab_width
         s = ('\n'  # separate from possible cuddled paragraph
              + indent + ('\n'+indent).join(lines)
-             + '\n\n')
+             + '\n')
         return s
 
     def _prepare_pyshell_blocks(self, text):
@@ -1070,7 +1070,7 @@ def _prepare_pyshell_blocks(self, text):
         _pyshell_block_re = re.compile(r"""
             ^([ ]{0,%d})>>>[ ].*\n  # first line
             ^(\1[^\S\n]*\S.*\n)*    # any number of subsequent lines with at least one character
-            ^\n                     # ends with a blank line
+            (?=^\1?\n|\Z)           # ends with a blank line or end of document
             """ % less_than_tab, re.M | re.X)
 
         return _pyshell_block_re.sub(self._pyshell_block_sub, text)
@@ -1864,14 +1864,20 @@ def _wrap_code(self, inner):
                     yield tup
                 yield 0, "</code>"
 
+            def _add_newline(self, inner):
+                # Add newlines around the inner contents so that _strict_tag_block_re matches the outer div.
+                yield 0, "\n"
+                yield from inner
+                yield 0, "\n"
+
             def wrap(self, source, outfile=None):
                 """Return the source with a code, pre, and div."""
                 if outfile is None:
                     # pygments >= 2.12
-                    return self._wrap_pre(self._wrap_code(source))
+                    return self._add_newline(self._wrap_pre(self._wrap_code(source)))
                 else:
                     # pygments < 2.12
-                    return self._wrap_div(self._wrap_pre(self._wrap_code(source)))
+                    return self._wrap_div(self._add_newline(self._wrap_pre(self._wrap_code(source))))
 
         formatter_opts.setdefault("cssclass", "codehilite")
         formatter = HtmlCodeFormatter(**formatter_opts)

test/tm-cases/admonitions_with_fenced_code_blocks.html

@@ -1,25 +1,35 @@
 <aside class="admonition note">
     <strong>note</strong>
     <p>Admonitions are able to contain fenced code blocks</p>
-    <div class="codehilite"><pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;like so&#39;</span><span class="p">)</span>
-    </code></pre></div>
+    <div class="codehilite">
+    <pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;like so&#39;</span><span class="p">)</span>
+    </code></pre>
+    </div>
 </aside>
 
 <aside class="admonition warning">
     <strong>warning</strong>
-    <div class="codehilite"><pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Consecutive blocks should also be fine&#39;</span><span class="p">)</span>
-    </code></pre></div>
-    <div class="codehilite"><pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Even though fenced code blocks wrap themselves in newlines&#39;</span><span class="p">)</span>
-    </code></pre></div>
+    <div class="codehilite">
+    <pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Consecutive blocks should also be fine&#39;</span><span class="p">)</span>
+    </code></pre>
+    </div>
+    <div class="codehilite">
+    <pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Even though fenced code blocks wrap themselves in newlines&#39;</span><span class="p">)</span>
+    </code></pre>
+    </div>
     <aside class="admonition hint">
         <strong>hint</strong>
         <em>It should also work nested</em>
-        <div class="codehilite"><pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;ok&#39;</span><span class="p">)</span>
-        </code></pre></div>
+        <div class="codehilite">
+        <pre><span></span><code><span class="nb">print</span><span class="p">(</span><span class="s1">&#39;ok&#39;</span><span class="p">)</span>
+        </code></pre>
+        </div>
     </aside>
 </aside>
 
-<div class="codehilite"><pre><span></span><code><span class="c1"># admonitions WITHIN fenced code blocks should NOT be rendered</span>
+<div class="codehilite">
+<pre><span></span><code><span class="c1"># admonitions WITHIN fenced code blocks should NOT be rendered</span>
 <span class="o">..</span> <span class="n">attention</span><span class="p">::</span> <span class="n">title</span>
    <span class="n">body</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/fenced_code_blocks_issue355.html

@@ -1,5 +1,7 @@
-<div class="codehilite"><pre><span></span><code><span class="n">some</span> <span class="n">code</span> <span class="n">block</span>
-</code></pre></div>
+<div class="codehilite">
+<pre><span></span><code><span class="n">some</span> <span class="n">code</span> <span class="n">block</span>
+</code></pre>
+</div>
 
 <pre><code>yet another code block
 </code></pre>

test/tm-cases/fenced_code_blocks_issue426.html

@@ -14,18 +14,22 @@ <h2>URL PARAMETERS IN THE TEMPLATE</h2>
 <li>All views (except <code>generic.View</code>) from <code>django.forms.generic</code> inherit from <code>ContextMixin</code></li>
 <li><p><code>ContextMixin</code> defines the method <code>get_context_data</code>:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="k">def</span> <span class="nf">get_context_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">def</span> <span class="nf">get_context_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
     <span class="n">kwargs</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;view&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span>
     <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">extra_context</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
         <span class="n">kwargs</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">extra_context</span><span class="p">)</span>
     <span class="k">return</span> <span class="n">kwargs</span>
-</code></pre></div>
+</code></pre>
+</div>
 
 <p>So when overriding one must be careful to extends <code>super</code>'s <code>kwargs</code>:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="k">def</span> <span class="nf">get_context_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">def</span> <span class="nf">get_context_data</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
     <span class="n">kwargs</span> <span class="o">=</span> <span class="nb">super</span><span class="p">()</span><span class="o">.</span><span class="n">get_context_data</span><span class="p">(</span><span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
     <span class="n">kwargs</span><span class="p">[</span><span class="s1">&#39;page_title&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;Documentation&quot;</span>
     <span class="k">return</span> <span class="n">kwargs</span>
-</code></pre></div></li>
+</code></pre>
+</div></li>
 </ul>

test/tm-cases/fenced_code_blocks_issue462.html

@@ -0,0 +1,12 @@
+<h1>Section 1</h1>
+
+<div class="codehilite">
+<pre><span></span><code><span class="n">x</span> <span class="o">=</span> <span class="mi">1</span>
+</code></pre>
+</div>
+
+<h1>Section 2</h1>
+
+<div>
+test
+</div>

test/tm-cases/fenced_code_blocks_issue462.opts

@@ -0,0 +1 @@
+{"extras": ["fenced-code-blocks", "pygments"]}

test/tm-cases/fenced_code_blocks_issue462.tags

@@ -0,0 +1 @@
+extra fenced-code-blocks pygments

test/tm-cases/fenced_code_blocks_issue462.text

@@ -0,0 +1,11 @@
+# Section 1
+
+```python
+x = 1
+```
+
+# Section 2
+
+<div>
+test
+</div>

test/tm-cases/fenced_code_blocks_leading_lang_space.html

@@ -1,3 +1,5 @@
-<div class="codehilite"><pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
     <span class="nb">print</span> <span class="s2">&quot;hi&quot;</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/fenced_code_blocks_safe_highlight.html

@@ -1,12 +1,16 @@
-<div class="codehilite"><pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
     <span class="nb">print</span> <span class="s2">&quot;hi&quot;</span>
-</code></pre></div>
+</code></pre>
+</div>
 
 <p>That's using the <em>fenced-code-blocks</em> extra with Python
 syntax coloring, if <code>pygments</code> is installed. See
 <a href="http://github.github.com/github-flavored-markdown/">http://github.github.com/github-flavored-markdown/</a>.</p>
 
-<div class="codehilite"><pre><span></span><code><span class="k">def</span> <span class="nf">foo</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">def</span> <span class="nf">foo</span>
     <span class="nb">puts</span> <span class="s2">&quot;hi&quot;</span>
 <span class="k">end</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/fenced_code_blocks_syntax_highlighting.html

@@ -1,12 +1,16 @@
-<div class="codehilite"><pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
     <span class="nb">print</span> <span class="s2">&quot;hi&quot;</span>
-</code></pre></div>
+</code></pre>
+</div>
 
 <p>That's using the <em>fenced-code-blocks</em> extra with Python
 syntax coloring, if <code>pygments</code> is installed. See
 <a href="http://github.github.com/github-flavored-markdown/">http://github.github.com/github-flavored-markdown/</a>.</p>
 
-<div class="codehilite"><pre><span></span><code><span class="k">def</span> <span class="nf">foo</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">def</span> <span class="nf">foo</span>
     <span class="nb">puts</span> <span class="s2">&quot;hi&quot;</span>
 <span class="k">end</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/fenced_code_blocks_syntax_indentation.html

@@ -1,5 +1,7 @@
-<div class="codehilite"><pre><span></span><code><span class="k">def</span> <span class="nf">foo</span><span class="p">():</span>
+<div class="codehilite">
+<pre><span></span><code><span class="k">def</span> <span class="nf">foo</span><span class="p">():</span>
     <span class="nb">print</span> <span class="s2">&quot;foo&quot;</span>
 
     <span class="nb">print</span> <span class="s2">&quot;bar&quot;</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/issue276_fenced_code_blocks_in_lists.html

@@ -12,8 +12,10 @@
 
 <p>empty codeblock just for sh*ts and giggles</p>
 
-<div class="codehilite"><pre><span></span><code><span class="n">test</span> <span class="k">with</span> <span class="n">language</span> <span class="nb">set</span>
-</code></pre></div>
+<div class="codehilite">
+<pre><span></span><code><span class="n">test</span> <span class="k">with</span> <span class="n">language</span> <span class="nb">set</span>
+</code></pre>
+</div>
 
 <pre><code>This is a regular code block
 Multiline

test/tm-cases/issue3_bad_code_color_hack.html

@@ -6,7 +6,9 @@ <h2>заголовок</h2>
 
 <p>Some python code:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="c1"># комментарий</span>
+<div class="codehilite">
+<pre><span></span><code><span class="c1"># комментарий</span>
 <span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
     <span class="nb">print</span> <span class="s2">&quot;hi&quot;</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/pyshell_and_fenced_code_blocks.html

@@ -2,20 +2,36 @@ <h1>From Recipe 302035</h1>
 
 <p>Some examples:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">nprint</span><span class="p">(</span><span class="mi">9876543210</span><span class="p">)</span>
+<div class="codehilite">
+<pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="n">nprint</span><span class="p">(</span><span class="mi">9876543210</span><span class="p">)</span>
 <span class="go">&#39;9 876 543 210&#39;</span>
 <span class="gp">&gt;&gt;&gt; </span><span class="n">nprint</span><span class="p">(</span><span class="mi">987654321</span><span class="p">,</span> <span class="n">period</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">delimiter</span><span class="o">=</span><span class="s2">&quot;,&quot;</span><span class="p">)</span>
 <span class="go">&#39;9,8,7,6,5,4,3,2,1,0&#39;</span>
-</code></pre></div>
+</code></pre>
+</div>
 
 <p>Indented a bit:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="mi">1</span> <span class="o">+</span> <span class="mi">1</span>
+<div class="codehilite">
+<pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="mi">1</span> <span class="o">+</span> <span class="mi">1</span>
 <span class="go">2</span>
-</code></pre></div>
+</code></pre>
+</div>
+
+<p>Part of blockquote:</p>
+
+<blockquote>
+  <div class="codehilite">
+<pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="mi">1</span> <span class="o">+</span> <span class="mi">1</span>
+<span class="go">2</span>
+</code></pre>
+  </div>
+</blockquote>
 
 <p>Cuddled to previous para (and at end of document):</p>
 
-<div class="codehilite"><pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="mi">2</span> <span class="o">+</span> <span class="mi">2</span>
+<div class="codehilite">
+<pre><span></span><code><span class="gp">&gt;&gt;&gt; </span><span class="mi">2</span> <span class="o">+</span> <span class="mi">2</span>
 <span class="go">4</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/pyshell_and_fenced_code_blocks.text

@@ -12,6 +12,11 @@ Indented a bit:
  >>> 1 + 1
  2
 
+Part of blockquote:
+
+> >>> 1 + 1
+> 2
+
 Cuddled to previous para (and at end of document):
 >>> 2 + 2
 4

test/tm-cases/quoted_fenced_code_blocks_whitespace_around_indented_lines.html

@@ -1,13 +1,15 @@
 <h3>Example:</h3>
 
 <blockquote>
-  <div class="codehilite"><pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
+  <div class="codehilite">
+<pre><span></span><code><span class="k">if</span> <span class="kc">True</span><span class="p">:</span>
     <span class="nb">print</span><span class="p">()</span>
 
     <span class="nb">print</span><span class="p">()</span>
 
     <span class="nb">print</span><span class="p">()</span>
 
     <span class="nb">print</span><span class="p">()</span>
-</code></pre></div>
+</code></pre>
+  </div>
 </blockquote>

test/tm-cases/syntax_color.html

@@ -1,15 +1,19 @@
 <p>Here is some sample code:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="kn">import</span> <span class="nn">sys</span>
+<div class="codehilite">
+<pre><span></span><code><span class="kn">import</span> <span class="nn">sys</span>
 <span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">argv</span><span class="o">=</span><span class="n">sys</span><span class="o">.</span><span class="n">argv</span><span class="p">):</span>
     <span class="n">logging</span><span class="o">.</span><span class="n">basicConfig</span><span class="p">()</span>
     <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;hi&#39;</span><span class="p">)</span>
-</code></pre></div>
+</code></pre>
+</div>
 
 <p>and:</p>
 
-<div class="codehilite"><pre><span></span><code><span class="n">use</span> <span class="s1">&#39;zlib&#39;</span>
+<div class="codehilite">
+<pre><span></span><code><span class="n">use</span> <span class="s1">&#39;zlib&#39;</span>
 <span class="nb">sub</span> <span class="n">main</span><span class="p">(</span><span class="n">argv</span><span class="p">)</span>
     <span class="nb">puts</span> <span class="s1">&#39;hi&#39;</span>
 <span class="k">end</span>
-</code></pre></div>
+</code></pre>
+</div>

test/tm-cases/syntax_color_opts.html

@@ -1,15 +1,19 @@
 <p>Here is some sample code:</p>
 
-<div class="codehilite" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><code><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sys</span>
+<div class="codehilite" style="background: #f8f8f8">
+<pre style="line-height: 125%;"><span></span><code><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">sys</span>
 <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">main</span>(argv<span style="color: #666666">=</span>sys<span style="color: #666666">.</span>argv):
     logging<span style="color: #666666">.</span>basicConfig()
     log<span style="color: #666666">.</span>info(<span style="color: #BA2121">&#39;hi&#39;</span>)
-</code></pre></div>
+</code></pre>
+</div>
 
 <p>and:</p>
 
-<div class="codehilite" style="background: #f8f8f8"><pre style="line-height: 125%;"><span></span><code>use <span style="color: #BA2121">&#39;zlib&#39;</span>
+<div class="codehilite" style="background: #f8f8f8">
+<pre style="line-height: 125%;"><span></span><code>use <span style="color: #BA2121">&#39;zlib&#39;</span>
 <span style="color: #008000">sub</span> main(argv)
     <span style="color: #008000">puts</span> <span style="color: #BA2121">&#39;hi&#39;</span>
 <span style="color: #008000; font-weight: bold">end</span>
-</code></pre></div>
+</code></pre>
+</div>