From a89d647639eb118f142eecdd35f83fa03b73e8de Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 14 Mar 2017 14:18:26 +0100 Subject: [PATCH 1/6] Switch Markdown handler to Mistune; implement admonitions --- naucse/markdown_util.py | 68 +++++++++++++++++++++++++++++------- naucse/templates.py | 6 ++-- requirements.txt | 2 +- test_naucse/test_markdown.py | 45 ++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 test_naucse/test_markdown.py diff --git a/naucse/markdown_util.py b/naucse/markdown_util.py index de2fd7fff9..de8f8574ba 100644 --- a/naucse/markdown_util.py +++ b/naucse/markdown_util.py @@ -1,24 +1,68 @@ from textwrap import dedent +import re -from markdown import Markdown -from markdown.extensions.admonition import AdmonitionExtension -from markdown.extensions.codehilite import CodeHiliteExtension -from markdown.extensions.fenced_code import FencedCodeExtension -from markdown.extensions.def_list import DefListExtension +import mistune from jinja2 import Markup + +class BlockGrammar(mistune.BlockGrammar): + admonition = re.compile(r'^!!! *(\S+) *"([^"]*)"\n((\n| .*)+)') + #admonition = re.compile(r'^!!!') + + +class BlockLexer(mistune.BlockLexer): + grammar_class = BlockGrammar + + default_rules = [ + 'admonition', + ] + mistune.BlockLexer.default_rules + + def parse_admonition(self, m): + self.tokens.append({ + 'type': 'admonition_start', + 'name': m.group(1), + 'title': m.group(2), + }) + self.parse(dedent(m.group(3))) + self.tokens.append({ + 'type': 'admonition_end', + }) + + +class Renderer(mistune.Renderer): + def admonition(self, name, content): + return '
{}
'.format(name, content) + + +class Markdown(mistune.Markdown): + def output_admonition(self): + name = self.token['name'] + body = self.renderer.placeholder() + if self.token['title']: + template = '

{}

\n' + body += template.format(self.token['title']) + while self.pop()['type'] != 'admonition_end': + body += self.tok() + return self.renderer.admonition(name, body) + + markdown = Markdown( - extensions=[ - AdmonitionExtension(), - FencedCodeExtension(), - CodeHiliteExtension(guess_lang=False), - DefListExtension(), - ], + escape = False, + block = BlockLexer(), + renderer = Renderer(), + #extensions=[ + #XXX: CodeHiliteExtension(guess_lang=False), + #XXX: DefListExtension(), + #], ) + def convert_markdown(text, *, inline=False): text = dedent(text) - result = Markup(markdown.convert(text)) + result = Markup(markdown(text)) + + print(text) + print(result) if inline and result.startswith('

') and result.endswith('

'): result = result[len('

'):-len('

')] diff --git a/naucse/templates.py b/naucse/templates.py index 8817ca918c..9417769692 100644 --- a/naucse/templates.py +++ b/naucse/templates.py @@ -1,3 +1,5 @@ +import textwrap + from flask import url_for, g from jinja2 import Markup @@ -76,11 +78,11 @@ def __str__(self): @template_function def figure(img, alt): - t = Markup(''' + t = Markup(textwrap.dedent(''' {alt} - ''') + ''')) return t.strip().format(img=img, alt=alt) diff --git a/requirements.txt b/requirements.txt index 45c5a2ce48..ed5511a50a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML flask elsa>=0.1.2 frozen-flask -markdown +mistune jinja2 werkzeug pygments diff --git a/test_naucse/test_markdown.py b/test_naucse/test_markdown.py new file mode 100644 index 0000000000..69b7a9beef --- /dev/null +++ b/test_naucse/test_markdown.py @@ -0,0 +1,45 @@ +from textwrap import dedent + +from naucse.markdown_util import convert_markdown + + +def test_markdown_admonition(): + src = dedent(""" + !!! note "" + Foo *bar* + """) + expected = '

Foo bar

\n
' + assert convert_markdown(src) == expected + + +def test_markdown_admonition_paragraphs(): + src = dedent(""" + !!! note "" + + Foo *fi* + + fo + + fum + """) + expected = dedent(""" +

Foo fi

+

fo

+

fum

+
+ """).strip() + assert convert_markdown(src) == expected + + +def test_markdown_admonition_name(): + src = dedent(""" + !!! note "NB!" + + foo + """) + expected = dedent(""" +

NB!

+

foo

+
+ """).strip() + assert convert_markdown(src) == expected From f3a2d537505597cb781a27145bb0ba74091f416a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 14 Mar 2017 16:35:40 +0100 Subject: [PATCH 2/6] Markdown: Fix figures and side-by-side consoles for Mistune --- lessons/beginners/cmdline/index.md | 2 +- naucse/markdown_util.py | 3 --- naucse/templates.py | 6 ++---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/lessons/beginners/cmdline/index.md b/lessons/beginners/cmdline/index.md index 1d33e816b2..032efc0abf 100644 --- a/lessons/beginners/cmdline/index.md +++ b/lessons/beginners/cmdline/index.md @@ -9,7 +9,7 @@ ``` {%- endfilter -%} - {% endfor %} + {%- endfor -%} {%- endmacro -%} diff --git a/naucse/markdown_util.py b/naucse/markdown_util.py index de8f8574ba..4aebc213ad 100644 --- a/naucse/markdown_util.py +++ b/naucse/markdown_util.py @@ -61,9 +61,6 @@ def convert_markdown(text, *, inline=False): text = dedent(text) result = Markup(markdown(text)) - print(text) - print(result) - if inline and result.startswith('

') and result.endswith('

'): result = result[len('

'):-len('

')] diff --git a/naucse/templates.py b/naucse/templates.py index 9417769692..ee40d2b58b 100644 --- a/naucse/templates.py +++ b/naucse/templates.py @@ -1,5 +1,3 @@ -import textwrap - from flask import url_for, g from jinja2 import Markup @@ -78,11 +76,11 @@ def __str__(self): @template_function def figure(img, alt): - t = Markup(textwrap.dedent(''' + t = Markup(''.join(p.strip() for p in """ {alt} - ''')) + """.splitlines())) return t.strip().format(img=img, alt=alt) From 49abafab76eca8f4a8181a2998d525e7ca7a2bf9 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Mar 2017 09:34:49 +0100 Subject: [PATCH 3/6] Add Pygments code highlighting --- lessons/beginners/install-editor/index.md | 23 ++++++++++++----------- lessons/beginners/variables/index.md | 15 +++++++++------ naucse/markdown_util.py | 18 ++++++++++++++++-- 3 files changed, 37 insertions(+), 19 deletions(-) diff --git a/lessons/beginners/install-editor/index.md b/lessons/beginners/install-editor/index.md index 655fcdb04f..101177580f 100644 --- a/lessons/beginners/install-editor/index.md +++ b/lessons/beginners/install-editor/index.md @@ -53,17 +53,18 @@ Obarvování Pro ilustraci, takhle může v editoru vypadat kousek kódu: - :::python - 1 @app.route('/courses//') - 2 def course_page(course): - 3 try: - 4 return render_template( - 5 'course.html', - 6 course=course, - 7 plan=course.sessions, - 8 ) - 9 except TemplateNotFound: - 10 abort(404) + ```python + 1 @app.route('/courses//') + 2 def course_page(course): + 3 try: + 4 return render_template( + 5 'course.html', + 6 course=course, + 7 plan=course.sessions, + 8 ) + 9 except TemplateNotFound: + 10 abort(404) + ``` ## Volba a nastavení editoru diff --git a/lessons/beginners/variables/index.md b/lessons/beginners/variables/index.md index 872be96d95..cd27183c6b 100644 --- a/lessons/beginners/variables/index.md +++ b/lessons/beginners/variables/index.md @@ -155,18 +155,21 @@ pro teď to budou kouzelná zaříkadla: * Chceš-li načíst **řetězec**, použij: - :::python - promenna = input('Zadej řetězec: ') + ```python + promenna = input('Zadej řetězec: ') + ``` * Chceš-li načíst **celé číslo**, použij: - :::python - promenna = int(input('Zadej číslo: ')) + ```python + promenna = int(input('Zadej číslo: ')) + ``` * Chceš-li načíst **desetinné číslo**, použij: - :::python - promenna = float(input('Zadej číslo: ')) + ```python + promenna = float(input('Zadej číslo: ')) + ``` Místo řetězce `'Zadej …'` se dá napsat i jiná výzva. A výsledek se samozřejmě dá uložit i do jiné proměnné než `promenna`. diff --git a/naucse/markdown_util.py b/naucse/markdown_util.py index 4aebc213ad..53226efb06 100644 --- a/naucse/markdown_util.py +++ b/naucse/markdown_util.py @@ -3,11 +3,17 @@ import mistune from jinja2 import Markup +import pygments +import pygments.lexers +import pygments.formatters.html + +pygments_formatter = pygments.formatters.html.HtmlFormatter( + cssclass='codehilite' +) class BlockGrammar(mistune.BlockGrammar): admonition = re.compile(r'^!!! *(\S+) *"([^"]*)"\n((\n| .*)+)') - #admonition = re.compile(r'^!!!') class BlockLexer(mistune.BlockLexer): @@ -33,6 +39,15 @@ class Renderer(mistune.Renderer): def admonition(self, name, content): return '
{}
'.format(name, content) + def block_code(self, code, lang): + if lang is not None: + lang = lang.strip() + if not lang or lang == 'plain': + escaped = mistune.escape(code) + return '
{}
'.format(escaped) + lexer = pygments.lexers.get_lexer_by_name(lang) + return pygments.highlight(code, lexer, pygments_formatter).strip() + class Markdown(mistune.Markdown): def output_admonition(self): @@ -51,7 +66,6 @@ def output_admonition(self): block = BlockLexer(), renderer = Renderer(), #extensions=[ - #XXX: CodeHiliteExtension(guess_lang=False), #XXX: DefListExtension(), #], ) From 66ab8adfb64b496457ae17e7c092651c03f1ed5c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Mar 2017 11:03:53 +0100 Subject: [PATCH 4/6] Re-add an extension for definition lists --- .../beginners/install-editor/_linux_base.md | 8 ++- naucse/markdown_util.py | 50 +++++++++++++++++-- test_naucse/test_markdown.py | 46 +++++++++++++++++ 3 files changed, 99 insertions(+), 5 deletions(-) diff --git a/lessons/beginners/install-editor/_linux_base.md b/lessons/beginners/install-editor/_linux_base.md index 04f9571497..1a529af332 100644 --- a/lessons/beginners/install-editor/_linux_base.md +++ b/lessons/beginners/install-editor/_linux_base.md @@ -5,10 +5,14 @@ Na Linuxu se {{ editor_name }} instaluje jako ostatní programy: Fedora -: `sudo dnf install {{ editor_cmd }}` +: ```console + $ sudo dnf install {{ editor_cmd }} + ``` Ubuntu -: `sudo apt-get install {{ editor_cmd }}` +: ```console + $ sudo apt-get install {{ editor_cmd }} + ``` Používáš-li jiný Linux, předpokládám že programy instalovat umíš :) diff --git a/naucse/markdown_util.py b/naucse/markdown_util.py index 53226efb06..fd2521f0e9 100644 --- a/naucse/markdown_util.py +++ b/naucse/markdown_util.py @@ -14,6 +14,7 @@ class BlockGrammar(mistune.BlockGrammar): admonition = re.compile(r'^!!! *(\S+) *"([^"]*)"\n((\n| .*)+)') + deflist = re.compile(r'^(([^\n: ][^\n]*\n)+)((:( {0,3})[^\n]*\n)( \5[^\n]*\n|\n)+)') class BlockLexer(mistune.BlockLexer): @@ -21,6 +22,7 @@ class BlockLexer(mistune.BlockLexer): default_rules = [ 'admonition', + 'deflist', ] + mistune.BlockLexer.default_rules def parse_admonition(self, m): @@ -34,6 +36,22 @@ def parse_admonition(self, m): 'type': 'admonition_end', }) + def parse_deflist(self, m): + self.tokens.append({ + 'type': 'deflist_term_start', + }) + self.parse(dedent(m.group(1))) + self.tokens.append({ + 'type': 'deflist_term_end', + }) + self.tokens.append({ + 'type': 'deflist_def_start', + }) + self.parse(dedent(' ' + m.group(3)[1:])) + self.tokens.append({ + 'type': 'deflist_def_end', + }) + class Renderer(mistune.Renderer): def admonition(self, name, content): @@ -48,6 +66,13 @@ def block_code(self, code, lang): lexer = pygments.lexers.get_lexer_by_name(lang) return pygments.highlight(code, lexer, pygments_formatter).strip() + def deflist(self, items): + tags = {'term': 'dt', 'def': 'dd'} + return '
\n{}
'.format(''.join( + '<{tag}>{text}'.format(tag=tags[type], text=text) + for type, text in items + )) + class Markdown(mistune.Markdown): def output_admonition(self): @@ -60,14 +85,33 @@ def output_admonition(self): body += self.tok() return self.renderer.admonition(name, body) + def output_deflist_term(self): + items = [['term', self.renderer.placeholder()]] + while True: + end_token = 'deflist_{}_end'.format(items[-1][0]) + while self.pop()['type'] not in (end_token, 'paragraph'): + items[-1][1] += self.tok() + if self.token['type'] == 'paragraph': + if items[-1][0] == 'term': + items.append(['term', self.renderer.placeholder()]) + items[-1][1] += self.token['text'] + else: + items[-1][1] += self.output_paragraph() + elif self.peek()['type'] == 'deflist_term_start': + self.pop() + items.append(['term', self.renderer.placeholder()]) + elif self.peek()['type'] == 'deflist_def_start': + self.pop() + items.append(['def', self.renderer.placeholder()]) + else: + break + return self.renderer.deflist(items) + markdown = Markdown( escape = False, block = BlockLexer(), renderer = Renderer(), - #extensions=[ - #XXX: DefListExtension(), - #], ) diff --git a/test_naucse/test_markdown.py b/test_naucse/test_markdown.py index 69b7a9beef..94fa3c7b3e 100644 --- a/test_naucse/test_markdown.py +++ b/test_naucse/test_markdown.py @@ -43,3 +43,49 @@ def test_markdown_admonition_name(): """).strip() assert convert_markdown(src) == expected + + +def test_markdown_definition_list(): + src = dedent(""" + Bla Bla + + The Term + : Its Definition + + More Text + """) + expected = dedent(""" +

Bla Bla

+
+
The Term

Its Definition

+

More Text

+ """).strip() + assert convert_markdown(src).strip() == expected + + +def test_markdown_definition_list_advanced(): + src = dedent(""" + Bla Bla + + The Term + : Its Definition + More Definition + + Even More + + Another Term + : Define this + + More Text + """) + expected = dedent(""" +

Bla Bla

+
+
The Term

Its Definition + More Definition

+

Even More

+
Another Term

Define this

+

More Text

+ """).strip() + print(convert_markdown(src)) + assert convert_markdown(src).strip() == expected From e2c0ea93799e64e760dc51f5de948b1cbb3a5145 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Mar 2017 14:46:39 +0100 Subject: [PATCH 5/6] Don't squash non-breaking whitespace Workaround for: https://github.com/lepture/mistune/issues/125 --- naucse/markdown_util.py | 6 ++++++ test_naucse/test_markdown.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/naucse/markdown_util.py b/naucse/markdown_util.py index fd2521f0e9..fbb9f2115a 100644 --- a/naucse/markdown_util.py +++ b/naucse/markdown_util.py @@ -116,10 +116,16 @@ def output_deflist_term(self): def convert_markdown(text, *, inline=False): + # Workaround for https://github.com/lepture/mistune/issues/125 + NBSP_REPLACER = '\uf8ff' + text = text.replace('\N{NO-BREAK SPACE}', NBSP_REPLACER) + text = dedent(text) result = Markup(markdown(text)) if inline and result.startswith('

') and result.endswith('

'): result = result[len('

'):-len('

')] + # Workaround for https://github.com/lepture/mistune/issues/125 + result = result.replace(NBSP_REPLACER, '\N{NO-BREAK SPACE}') return result diff --git a/test_naucse/test_markdown.py b/test_naucse/test_markdown.py index 94fa3c7b3e..1a7c2d3179 100644 --- a/test_naucse/test_markdown.py +++ b/test_naucse/test_markdown.py @@ -89,3 +89,8 @@ def test_markdown_definition_list_advanced(): """).strip() print(convert_markdown(src)) assert convert_markdown(src).strip() == expected + + +def test_markdown_keeps_nbsp(): + text = 'Some text\N{NO-BREAK SPACE}more text' + assert convert_markdown(text).strip() == '

{}

'.format(text) From 1fbed3d96bcd47f7e3923d3bf5989c9e7990d690 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 15 Mar 2017 15:08:31 +0100 Subject: [PATCH 6/6] Add pytest to Travis script --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index fd455a5ce6..de8c0b52ef 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,10 @@ python: - '3.5' cache: - pip +install: +- pip install -r requirements.txt pytest script: +- python -m pytest test_naucse - python -m naucse freeze deploy: provider: script