Merge PR #59 - Switch Markdown parser to Mistune for speed

#59

Author: encukou

.travis.yml

@@ -3,7 +3,10 @@ python:
 - '3.5'
 cache:
 - pip
+install:
+- pip install -r requirements.txt pytest
 script:
+- python -m pytest test_naucse
 - python -m naucse freeze
 deploy:
   provider: script

lessons/beginners/cmdline/index.md

@@ -9,7 +9,7 @@
 ```
 {%- endfilter -%}
             </div>
-        {% endfor %}
+        {%- endfor -%}
     </div>
 {%- endmacro -%}
 

lessons/beginners/install-editor/_linux_base.md

@@ -5,10 +5,14 @@
 Na Linuxu se {{ editor_name }} instaluje jako ostatní programy:
 
 Fedora
-:   `sudo dnf install {{ editor_cmd }}`
+:   ```console
+    $ sudo dnf install {{ editor_cmd }}
+    ```
 
 Ubuntu
-:   `sudo apt-get install {{ editor_cmd }}`
+:   ```console
+    $ sudo apt-get install {{ editor_cmd }}
+    ```
 
 Používáš-li jiný Linux, předpokládám že programy instalovat umíš :)
 

lessons/beginners/install-editor/index.md

@@ -53,17 +53,18 @@ Obarvování
 
     Pro ilustraci, takhle může v editoru vypadat kousek kódu:
 
-        :::python
-         1  @app.route('/courses/<course:course>/')
-         2  def course_page(course):
-         3      try:
-         4          return render_template(
-         5              'course.html',
-         6              course=course,
-         7              plan=course.sessions,
-         8          )
-         9      except TemplateNotFound:
-        10         abort(404)
+    ```python
+        1  @app.route('/courses/<course:course>/')
+        2  def course_page(course):
+        3      try:
+        4          return render_template(
+        5              'course.html',
+        6              course=course,
+        7              plan=course.sessions,
+        8          )
+        9      except TemplateNotFound:
+       10          abort(404)
+    ```
 
 
 ## Volba a nastavení editoru

lessons/beginners/variables/index.md

@@ -155,18 +155,21 @@ pro teď to budou kouzelná zaříkadla:
 
 * Chceš-li načíst **řetězec**, použij:
 
-        :::python
-        promenna = input('Zadej řetězec: ')
+  ```python
+  promenna = input('Zadej řetězec: ')
+  ```
 
 * Chceš-li načíst **celé číslo**, použij:
 
-        :::python
-        promenna = int(input('Zadej číslo: '))
+  ```python
+  promenna = int(input('Zadej číslo: '))
+  ```
 
 * Chceš-li načíst **desetinné číslo**, použij:
 
-        :::python
-        promenna = float(input('Zadej číslo: '))
+  ```python
+  promenna = float(input('Zadej číslo: '))
+  ```
 
 Místo řetězce `'Zadej …'` se dá napsat i jiná výzva. 
 A výsledek se samozřejmě dá uložit i do jiné proměnné než `promenna`.

naucse/markdown_util.py

@@ -1,26 +1,131 @@
 from textwrap import dedent
+import re
 
-from markdown import Markdown
-from markdown.extensions.admonition import AdmonitionExtension
-from markdown.extensions.codehilite import CodeHiliteExtension
-from markdown.extensions.fenced_code import FencedCodeExtension
-from markdown.extensions.def_list import DefListExtension
+import mistune
 from jinja2 import Markup
+import pygments
+import pygments.lexers
+import pygments.formatters.html
+
+pygments_formatter = pygments.formatters.html.HtmlFormatter(
+    cssclass='codehilite'
+)
+
+
+class BlockGrammar(mistune.BlockGrammar):
+    admonition = re.compile(r'^!!! *(\S+) *"([^"]*)"\n((\n| .*)+)')
+    deflist = re.compile(r'^(([^\n: ][^\n]*\n)+)((:( {0,3})[^\n]*\n)( \5[^\n]*\n|\n)+)')
+
+
+class BlockLexer(mistune.BlockLexer):
+    grammar_class = BlockGrammar
+
+    default_rules = [
+        'admonition',
+        'deflist',
+    ] + mistune.BlockLexer.default_rules
+
+    def parse_admonition(self, m):
+        self.tokens.append({
+            'type': 'admonition_start',
+            'name': m.group(1),
+            'title': m.group(2),
+        })
+        self.parse(dedent(m.group(3)))
+        self.tokens.append({
+            'type': 'admonition_end',
+        })
+
+    def parse_deflist(self, m):
+        self.tokens.append({
+            'type': 'deflist_term_start',
+        })
+        self.parse(dedent(m.group(1)))
+        self.tokens.append({
+            'type': 'deflist_term_end',
+        })
+        self.tokens.append({
+            'type': 'deflist_def_start',
+        })
+        self.parse(dedent(' ' + m.group(3)[1:]))
+        self.tokens.append({
+            'type': 'deflist_def_end',
+        })
+
+
+class Renderer(mistune.Renderer):
+    def admonition(self, name, content):
+        return '<div class="admonition {}">{}</div>'.format(name, content)
+
+    def block_code(self, code, lang):
+        if lang is not None:
+            lang = lang.strip()
+        if not lang or lang == 'plain':
+            escaped = mistune.escape(code)
+            return '<div class="codehilite"><pre><code>{}</code></pre></div>'.format(escaped)
+        lexer = pygments.lexers.get_lexer_by_name(lang)
+        return pygments.highlight(code, lexer, pygments_formatter).strip()
+
+    def deflist(self, items):
+        tags = {'term': 'dt', 'def': 'dd'}
+        return '<dl>\n{}</dl>'.format(''.join(
+            '<{tag}>{text}</{tag}>'.format(tag=tags[type], text=text)
+            for type, text in items
+        ))
+
+
+class Markdown(mistune.Markdown):
+    def output_admonition(self):
+        name = self.token['name']
+        body = self.renderer.placeholder()
+        if self.token['title']:
+            template = '<p class="admonition-title">{}</p>\n'
+            body += template.format(self.token['title'])
+        while self.pop()['type'] != 'admonition_end':
+            body += self.tok()
+        return self.renderer.admonition(name, body)
+
+    def output_deflist_term(self):
+        items = [['term', self.renderer.placeholder()]]
+        while True:
+            end_token = 'deflist_{}_end'.format(items[-1][0])
+            while self.pop()['type'] not in (end_token, 'paragraph'):
+                items[-1][1] += self.tok()
+            if self.token['type'] == 'paragraph':
+                if items[-1][0] == 'term':
+                    items.append(['term', self.renderer.placeholder()])
+                    items[-1][1] += self.token['text']
+                else:
+                    items[-1][1] += self.output_paragraph()
+            elif self.peek()['type'] == 'deflist_term_start':
+                self.pop()
+                items.append(['term', self.renderer.placeholder()])
+            elif self.peek()['type'] == 'deflist_def_start':
+                self.pop()
+                items.append(['def', self.renderer.placeholder()])
+            else:
+                break
+        return self.renderer.deflist(items)
+
 
 markdown = Markdown(
-    extensions=[
-        AdmonitionExtension(),
-        FencedCodeExtension(),
-        CodeHiliteExtension(guess_lang=False),
-        DefListExtension(),
-    ],
+    escape = False,
+    block = BlockLexer(),
+    renderer = Renderer(),
 )
 
+
 def convert_markdown(text, *, inline=False):
+    # Workaround for https://github.com/lepture/mistune/issues/125
+    NBSP_REPLACER = '\uf8ff'
+    text = text.replace('\N{NO-BREAK SPACE}', NBSP_REPLACER)
+
     text = dedent(text)
-    result = Markup(markdown.convert(text))
+    result = Markup(markdown(text))
 
     if inline and result.startswith('<p>') and result.endswith('</p>'):
         result = result[len('<p>'):-len('</p>')]
 
+    # Workaround for https://github.com/lepture/mistune/issues/125
+    result = result.replace(NBSP_REPLACER, '\N{NO-BREAK SPACE}')
     return result

naucse/templates.py

@@ -76,11 +76,11 @@ def __str__(self):
 
 @template_function
 def figure(img, alt):
-    t = Markup('''
+    t = Markup(''.join(p.strip() for p in """
         <span class="figure">
             <a href="{img}">
                 <img src="{img}" alt="{alt}">
             </a>
         </span>
-    ''')
+    """.splitlines()))
     return t.strip().format(img=img, alt=alt)

requirements.txt

@@ -2,7 +2,7 @@ PyYAML
 flask
 elsa>=0.1.2
 frozen-flask
-markdown
+mistune
 jinja2
 werkzeug
 pygments

test_naucse/test_markdown.py

@@ -0,0 +1,96 @@
+from textwrap import dedent
+
+from naucse.markdown_util import convert_markdown
+
+
+def test_markdown_admonition():
+    src = dedent("""
+        !!! note ""
+            Foo *bar*
+    """)
+    expected = '<div class="admonition note"><p>Foo <em>bar</em></p>\n</div>'
+    assert convert_markdown(src) == expected
+
+
+def test_markdown_admonition_paragraphs():
+    src = dedent("""
+        !!! note ""
+
+            Foo *fi*
+
+            fo
+
+            fum
+    """)
+    expected = dedent("""
+        <div class="admonition note"><p>Foo <em>fi</em></p>
+        <p>fo</p>
+        <p>fum</p>
+        </div>
+    """).strip()
+    assert convert_markdown(src) == expected
+
+
+def test_markdown_admonition_name():
+    src = dedent("""
+        !!! note "NB!"
+
+            foo
+    """)
+    expected = dedent("""
+        <div class="admonition note"><p class="admonition-title">NB!</p>
+        <p>foo</p>
+        </div>
+    """).strip()
+    assert convert_markdown(src) == expected
+
+
+def test_markdown_definition_list():
+    src = dedent("""
+        Bla Bla
+
+        The Term
+        :    Its Definition
+
+        More Text
+    """)
+    expected = dedent("""
+        <p>Bla Bla</p>
+        <dl>
+        <dt></dt><dt>The Term</dt><dd><p>Its Definition</p>
+        </dd></dl><p>More Text</p>
+    """).strip()
+    assert convert_markdown(src).strip() == expected
+
+
+def test_markdown_definition_list_advanced():
+    src = dedent("""
+        Bla Bla
+
+        The Term
+        :   Its Definition
+            More Definition
+
+            Even More
+
+        Another Term
+        :   Define this
+
+        More Text
+    """)
+    expected = dedent("""
+        <p>Bla Bla</p>
+        <dl>
+        <dt></dt><dt>The Term</dt><dd><p>Its Definition
+        More Definition</p>
+        <p>Even More</p>
+        </dd><dt></dt><dt>Another Term</dt><dd><p>Define this</p>
+        </dd></dl><p>More Text</p>
+    """).strip()
+    print(convert_markdown(src))
+    assert convert_markdown(src).strip() == expected
+
+
+def test_markdown_keeps_nbsp():
+    text = 'Some text\N{NO-BREAK SPACE}more text'
+    assert convert_markdown(text).strip() == '<p>{}</p>'.format(text)