From 4b50311403ae294336628f64789662b747cbcd23 Mon Sep 17 00:00:00 2001 From: Zack Buhman Date: Thu, 25 Dec 2014 20:50:35 -0500 Subject: [PATCH 1/3] add vanity pastes --- pb/paste/model.py | 39 +++++++++++++++++++++-------- pb/paste/views.py | 42 +++++++++++++++++++------------ pb/pb.py | 18 +++++++++++++ pb/util.py | 8 ++++++ schema.sql | 64 +++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 142 insertions(+), 29 deletions(-) diff --git a/pb/paste/model.py b/pb/paste/model.py index 4f559b2..dc1af84 100644 --- a/pb/paste/model.py +++ b/pb/paste/model.py @@ -14,6 +14,8 @@ from uuid import uuid4 from hashlib import sha1 +from mysql.connector import errors + def insert(content): secret = uuid4().bytes args = (secret, content, None) @@ -25,22 +27,32 @@ def insert_private(content): args = (secret, content, None) (_, _, digest) = request.cur.callproc('paste_insert_private', args) return bytes(digest) if digest else None, secret - + +def insert_vanity(label, content): + secret = uuid4().bytes + args = (label, secret, content) + + try: + request.cur.callproc('paste_insert_vanity', args) + return secret + except errors.IntegrityError: + return None + def put(secret, content): - args = (secret, content, None, None) - (_, _, id, digest) = request.cur.callproc('paste_put', args) - return int(id) if id else None, bytes(digest) if digest else None + args = (secret, content, None, None, None) + (_, _, id, digest, label) = request.cur.callproc('paste_put', args) + return int(id) if id else None, bytes(digest) if digest else None, bytes(label) if label else None def delete(uuid): - args = (uuid, None, None) - (_, id, digest) = request.cur.callproc('paste_delete', args) - return int(id) if id else None, bytes(digest) if digest else None + args = (uuid, None, None, None) + (_, id, digest, label) = request.cur.callproc('paste_delete', args) + return int(id) if id else None, bytes(digest) if digest else None, bytes(label) if label else None def get_digest(content): digest = sha1(content).digest() - args = (digest, None, None) - (_, id, exists) = request.cur.callproc('paste_get_digest', args) - return int(id) if id else None, digest if exists else None + args = (digest, None, None, None) + (_, id, label, exists) = request.cur.callproc('paste_get_digest', args) + return int(id) if id else None, digest if exists else None, bytes(label) if label else None def get_content(id): args = (id,) + (None,) @@ -52,7 +64,12 @@ def get_content_digest(digest): (_, content) = request.cur.callproc('paste_get_content_digest', args) return content +def get_content_vanity(label): + args = (label, None) + (_, content) = request.cur.callproc('paste_get_content_vanity', args) + return content + def get_stats(): args = (None, None) (count, length) = request.cur.callproc('paste_get_stats', args) - return int(count), int(length) + return int(count) if count else 0, int(length) if length else 0 diff --git a/pb/paste/views.py b/pb/paste/views.py index 7da2ee8..1a56825 100644 --- a/pb/paste/views.py +++ b/pb/paste/views.py @@ -20,7 +20,7 @@ from pb.db import cursor from pb.paste import model, handler as _handler -from pb.util import highlight, redirect, request_content, id_url, publish_parts +from pb.util import highlight, redirect, request_content, id_url, publish_parts, any_url paste = Blueprint('paste', __name__) @@ -43,21 +43,25 @@ def form(): return Response(render_template("form.html"), mimetype='text/html') @paste.route('/', methods=['POST']) +@paste.route('/', methods=['POST']) @cursor -def post(): +def post(vanity=None): content, filename = request_content() if not content: return "Nope.\n", 400 uuid = None - id, digest = model.get_digest(content) - if not id and not digest: - if request.form.get('p'): + id, digest, label = model.get_digest(content) + if not any((id, digest, label)): + if vanity: + label, name = vanity + uuid = model.insert_vanity(label, content) + elif request.form.get('p'): digest, uuid = model.insert_private(content) else: id, uuid = model.insert(content) - url = id_url(b66=(id, filename)) if id else id_url(sha1=(digest, filename)) + url = any_url(id, digest, label, filename=filename) uuid = str(UUID(bytes=uuid)) if uuid else '' return redirect(url, safe_dump(dict(url=url, uuid=uuid), default_flow_style=False)) @@ -68,14 +72,14 @@ def put(uuid): if not content: return "Nope.\n", 400 - id, digest = model.get_digest(content) - if id or digest: - url = id_url(b66=id) if id else id_url(sha1=digest) + args = model.get_digest(content) + if any(args): + url = any_url(*args) return redirect(url, "Paste already exists.\n", 409) - id, digest = model.put(uuid.bytes, content) - if id or digest: - url = id_url(b66=(id, filename)) if id else id_url(sha1=digest) + args = model.put(uuid.bytes, content) + if any(args): + url = any_url(*args) return redirect(url, "{} updated.\n".format(url), 200) return "Not found.\n", 404 @@ -83,9 +87,9 @@ def put(uuid): @paste.route('/', methods=['DELETE']) @cursor def delete(uuid): - id, digest = model.delete(uuid.bytes) - if id or digest: - url = id_url(b66=id) if id else id_url(sha1=digest) + args = model.delete(uuid.bytes) + if any(args): + url = any_url(*args) return redirect(url, "{} deleted.\n".format(url), 200) return "Not found.\n", 404 @@ -95,8 +99,11 @@ def delete(uuid): @paste.route('/') @paste.route('//') @paste.route('//') +@paste.route('/') +@paste.route('//') +@paste.route('//') @cursor -def get(b66=None, sha1=None, lexer=None, handler=None): +def get(b66=None, sha1=None, label=None, lexer=None, handler=None): content = None if b66: id, name = b66 @@ -104,6 +111,9 @@ def get(b66=None, sha1=None, lexer=None, handler=None): if sha1: digest, name = sha1 content = model.get_content_digest(digest) + if label: + label, name = label + content = model.get_content_vanity(label) if not content: return "Not found.\n", 404 diff --git a/pb/pb.py b/pb/pb.py index 9b36473..06e9d47 100644 --- a/pb/pb.py +++ b/pb/pb.py @@ -60,6 +60,23 @@ def to_url(self, value): ext = path.splitext(filename)[1] if filename else '' return '{}{}'.format(hexlify(digest).decode('utf-8'), ext) +class LabelConverter(BaseConverter): + def __init__(self, map): + super().__init__(map) + self.regex = '(([^/.]{6,39})([.][^/]*)?)' + self.sre = re.compile(self.regex) + + def to_python(self, value): + (name, label, _) = self.sre.match(value).groups() + return label.encode('utf-8'), name + + def to_url(self, value): + if isinstance(value, bytes): + return value.decode('utf-8') + label, filename = value + ext = path.splitext(filename)[1] if filename else '' + return '{}{}'.format(label.decode('utf-8'), ext) + def load_yaml(app, filename): for filename in BaseDirectory.load_config_paths('pb', filename): with open(filename) as f: @@ -71,6 +88,7 @@ def create_app(config_filename='config.yaml'): app.response_class = TextResponse app.url_map.converters['id'] = IDConverter app.url_map.converters['sha1'] = SHA1Converter + app.url_map.converters['label'] = LabelConverter load_yaml(app, config_filename) init_db(app) diff --git a/pb/util.py b/pb/util.py index ba9c692..f4fb2de 100644 --- a/pb/util.py +++ b/pb/util.py @@ -73,6 +73,14 @@ def id_url(**kwargs): scheme = proto if proto else request.scheme return url_for('.get', _external=True, _scheme=scheme, **kwargs) +def any_url(id, digest, label, **kwargs): + if id: + return id_url(b66=id, **kwargs) + if digest: + return id_url(sha1=digest, **kwargs) + if label: + return id_url(label=label, **kwargs) + def publish_parts(source): overrides = {'syntax_highlight': 'short'} parts = core.publish_parts(source, writer_name='html', settings_overrides=overrides) diff --git a/schema.sql b/schema.sql index 7373407..0437da4 100644 --- a/schema.sql +++ b/schema.sql @@ -25,6 +25,18 @@ CREATE TABLE private ( ) ENGINE = InnoDB; +DROP TABLE IF EXISTS vanity; +CREATE TABLE vanity ( + label TINYBLOB NOT NULL, + digest BINARY(20) NOT NULL, + secret BINARY(16) NOT NULL, + content BLOB NOT NULL, + PRIMARY KEY (label(39)), + UNIQUE KEY (digest), + UNIQUE KEY (secret) +) +ENGINE = InnoDB; + DELIMITER @@ DROP PROCEDURE IF EXISTS paste_insert@@ @@ -57,12 +69,25 @@ BEGIN END; @@ +DROP PROCEDURE IF EXISTS paste_insert_vanity@@ +CREATE PROCEDURE paste_insert_vanity ( + p_label TINYBLOB, + p_secret BINARY(16), + p_content MEDIUMBLOB +) +BEGIN + INSERT vanity (label, digest, secret, content) + VALUES (p_label, UNHEX(SHA1(p_content)), p_secret, p_content); +END; +@@ + DROP PROCEDURE IF EXISTS paste_put@@ CREATE PROCEDURE paste_put ( p_secret BINARY(16), p_content MEDIUMBLOB, OUT p_id MEDIUMINT, - OUT p_digest BINARY(20) + OUT p_digest BINARY(20), + OUT p_label TINYBLOB ) BEGIN START TRANSACTION; @@ -72,6 +97,9 @@ BEGIN UPDATE private SET digest = UNHEX(SHA1(p_content)), content = p_content WHERE secret = p_secret; + UPDATE private + SET digest = UNHEX(SHA1(p_content)), content = p_content + WHERE secret = p_secret; /* .. */ SELECT id INTO p_id FROM paste @@ -79,6 +107,9 @@ BEGIN SELECT digest INTO p_digest FROM private WHERE secret = p_secret; + SELECT label INTO p_label + FROM vanity + WHERE secret = p_secret; COMMIT; END; @@ @@ -87,7 +118,8 @@ DROP PROCEDURE IF EXISTS paste_delete@@ CREATE PROCEDURE paste_delete ( p_secret BINARY(16), OUT p_id MEDIUMINT, - OUT p_digest BINARY(20) + OUT p_digest BINARY(20), + OUT p_label TINYBLOB ) BEGIN START TRANSACTION; @@ -97,6 +129,9 @@ BEGIN SELECT digest INTO p_digest FROM private WHERE secret = p_secret; + SELECT label INTO p_label + FROM vanity + WHERE secret = p_secret; /* .. */ DELETE FROM paste @@ -104,6 +139,9 @@ BEGIN DELETE FROM private WHERE secret = p_secret; + DELETE + FROM vanity + WHERE secret = p_secret; COMMIT; END; @@ @@ -119,6 +157,8 @@ BEGIN SELECT content FROM paste UNION SELECT content FROM private + UNION + SELECT content FROM vanity ) AS p; END; @@ @@ -127,12 +167,16 @@ DROP PROCEDURE IF EXISTS paste_get_digest@@ CREATE PROCEDURE paste_get_digest ( p_digest BINARY(20), OUT p_id MEDIUMINT, + OUT p_label TINYBLOB, OUT p_exists BIT(1) ) BEGIN SELECT id INTO p_id FROM paste WHERE digest = p_digest; + SELECT label INTO p_label + FROM vanity + WHERE digest = p_digest; SELECT 1 INTO p_exists FROM private WHERE digest = p_digest; @@ -166,10 +210,26 @@ BEGIN SELECT content FROM private WHERE digest = p_digest + UNION + SELECT content + FROM vanity + WHERE digest = p_digest ) AS p; END; @@ +DROP PROCEDURE IF EXISTS paste_get_content_vanity@@ +CREATE PROCEDURE paste_get_content_vanity ( + p_label TINYBLOB, + OUT p_content MEDIUMBLOB +) +BEGIN + SELECT content INTO p_content + FROM vanity + WHERE label = p_label; +END; +@@ + /* END PASTE SCHEMA */ From feed2c8edef300b0af4339f1927e629b0ec12861 Mon Sep 17 00:00:00 2001 From: Zack Buhman Date: Thu, 25 Dec 2014 20:52:37 -0500 Subject: [PATCH 2/3] add vanity paste tests --- tests/test_paste_vanity.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tests/test_paste_vanity.py diff --git a/tests/test_paste_vanity.py b/tests/test_paste_vanity.py new file mode 100644 index 0000000..3eab021 --- /dev/null +++ b/tests/test_paste_vanity.py @@ -0,0 +1,33 @@ +from time import time +from yaml import load + +from flask import url_for + +from pb.pb import create_app + +def test_paste_vanity(): + app = create_app() + + c = str(time()) + rv = app.test_client().post('/foo123', data=dict( + c = c + )) + location = rv.headers.get('Location') + assert 'foo123' in location + data = load(rv.get_data()) + + rv = app.test_client().get(location) + assert rv.status_code == 200 + assert rv.get_data() == c.encode('utf-8') + + with app.test_request_context(): + url = url_for('paste.put', uuid=data.get('uuid')) + + rv = app.test_client().put(url, data=dict( + c = str(time()) + )) + assert rv.status_code == 200 + + rv = app.test_client().delete(url) + assert rv.status_code == 200 + From 2182580820481ccc3fdae193dc2d94bf6ee874ac Mon Sep 17 00:00:00 2001 From: Zack Buhman Date: Thu, 25 Dec 2014 21:05:07 -0500 Subject: [PATCH 3/3] update documentation --- pb/templates/index.rst | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/pb/templates/index.rst b/pb/templates/index.rst index 4623b1d..f653385 100644 --- a/pb/templates/index.rst +++ b/pb/templates/index.rst @@ -30,16 +30,25 @@ id One of: - a four character base66 paste id -- a four character base66 paste id, followed by a period-delimiter and a - mimetype extension +- a four character base66 paste id, followed by a period-delimiter and + a mimetype extension - a 40 character sha1 hexdigest - a 40 character sha1 hexdigest, followed by a period-delimiter and a mimetype extension +- a 'vanity' label +- a 'vanity' label, followed by a period-delimiter + and a mimetype extension - a three character base66 url redirect id A mimetype extension, when specified, is first matched with a matching mimetype known to the system, then returned in the HTTP response headers. +vanity +^^^^^^ + +Any unicode string excluding the characters '/' and '.' of 5 < length +< 40. + lexer ^^^^^ @@ -110,6 +119,12 @@ Unless the 'filename' disposition extension parameter is specified, the form data is decoded. The value of the 'filename' parameter is split by period-delimited extension, and appended to the location in the response. +``POST /`` +^^^^^^^^^^^^^^^^^^ + +Same as above, except the paste is a 'vanity' paste, where the GET URL +path is identical to the POST path. + ``PUT /`` ^^^^^^^^^^^^^^^ @@ -153,8 +168,8 @@ examples No really, how in the name of Gandalf's beard does this actually work? Show me! -normal paste -^^^^^^^^^^^^ +creating pastes +^^^^^^^^^^^^^^^ Create a paste from the output of 'dmesg': @@ -235,6 +250,19 @@ base66 id: url: http://localhost:10002/1c5dd062b6a3359cf60989d0e1c8746944608304 uuid: e5860f7a-b074-4e5d-88d4-747cfacc1fcd +vanity pastes +^^^^^^^^^^^^^ + +Witness the gloriousness: + +.. code:: console + + $ curl -F c=@- https://ptpb.pw/polyzen <<< boats and hoes + url: https://ptpb.pw/polyzen + uuid: + $ curl https://ptpb.pw/polyzen + boats and hoes + shell functions ---------------