From d2ecbde84966f4c3232dfb84b66c716eea307764 Mon Sep 17 00:00:00 2001 From: Dick Marinus Date: Fri, 18 Aug 2017 20:40:16 +0200 Subject: [PATCH] Preliminary work for a future change in outputting results that uses less memory --- changelog.rst | 4 +++ pgcli/main.py | 17 ++++++++-- tests/features/steps/expanded.py | 1 - tests/test_main.py | 58 ++++++++++++++++---------------- tests/test_pgexecute.py | 20 +++++------ 5 files changed, 57 insertions(+), 43 deletions(-) diff --git a/changelog.rst b/changelog.rst index ed0a49909..7f0e0dbdf 100644 --- a/changelog.rst +++ b/changelog.rst @@ -10,6 +10,10 @@ Features: * Remove the ``...`` in the continuation prompt and use empty space instead. (Thanks: `Amjith Ramanujam`_) * Add \conninfo and handle more parameters with \c (issue #716) (Thanks: `François Pietka`_) +Internal changes: +----------------- +* Preliminary work for a future change in outputting results that uses less memory. (Thanks: `Dick Marinus`_) + Bug Fixes: ---------- diff --git a/pgcli/main.py b/pgcli/main.py index 57e0befff..65588474d 100755 --- a/pgcli/main.py +++ b/pgcli/main.py @@ -12,6 +12,7 @@ import functools import humanize import datetime as dt +import itertools from time import time, sleep from codecs import open @@ -1011,16 +1012,26 @@ def format_arrays(data, headers, **_): headers = [case_function(utf8tounicode(x)) for x in headers] rows = list(cur) formatted = formatter.format_output(rows, headers, **output_kwargs) - first_line = formatted[:formatted.find('\n')] + + if isinstance(formatted, (text_type)): + formatted = iter(formatted.splitlines()) + + first_line = next(formatted) + formatted = itertools.chain([first_line], formatted) + + if max_width: + formatted = list(formatted) if (not expanded and max_width and len(first_line) > max_width and headers): formatted = formatter.format_output( rows, headers, format_name='vertical', **output_kwargs) + if isinstance(formatted, (text_type)): + formatted = iter(formatted.splitlines()) - output.append(formatted) + output = itertools.chain(output, formatted) if status: # Only print the status if it's not None. - output.append(status) + output = itertools.chain(output, [status]) return output diff --git a/tests/features/steps/expanded.py b/tests/features/steps/expanded.py index 31622edf1..5360ea22d 100644 --- a/tests/features/steps/expanded.py +++ b/tests/features/steps/expanded.py @@ -43,7 +43,6 @@ def step_see_data(context, which): x | 1\r y | 1.0\r z | 1.0000\r - \r SELECT 1\r '''), timeout=1) diff --git a/tests/test_main.py b/tests/test_main.py index c35853ee9..df99eb45d 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -57,14 +57,14 @@ def test_format_output(): 'test status', settings) expected = [ 'Title', - '+---------+---------+\n' - '| head1 | head2 |\n' - '|---------+---------|\n' - '| abc | def |\n' + '+---------+---------+', + '| head1 | head2 |', + '|---------+---------|', + '| abc | def |', '+---------+---------+', 'test status' ] - assert results == expected + assert list(results) == expected def test_format_array_output(executor): @@ -78,15 +78,15 @@ def test_format_array_output(executor): """ results = run(executor, statement) expected = [ - '+----------------+------------------------+--------------+\n' - '| bigint_array | nested_numeric_array | 配列 |\n' - '|----------------+------------------------+--------------|\n' - '| {1,2,3} | {{1,2},{3,4}} | {å,魚,текст} |\n' - '| {} | | {} |\n' + '+----------------+------------------------+--------------+', + '| bigint_array | nested_numeric_array | 配列 |', + '|----------------+------------------------+--------------|', + '| {1,2,3} | {{1,2},{3,4}} | {å,魚,текст} |', + '| {} | | {} |', '+----------------+------------------------+--------------+', 'SELECT 2' ] - assert results == expected + assert list(results) == expected def test_format_array_output_expanded(executor): @@ -100,17 +100,17 @@ def test_format_array_output_expanded(executor): """ results = run(executor, statement, expanded=True) expected = [ - '-[ RECORD 1 ]-------------------------\n' - 'bigint_array | {1,2,3}\n' - 'nested_numeric_array | {{1,2},{3,4}}\n' - '配列 | {å,魚,текст}\n' - '-[ RECORD 2 ]-------------------------\n' - 'bigint_array | {}\n' - 'nested_numeric_array | \n' - '配列 | {}\n', + '-[ RECORD 1 ]-------------------------', + 'bigint_array | {1,2,3}', + 'nested_numeric_array | {{1,2},{3,4}}', + '配列 | {å,魚,текст}', + '-[ RECORD 2 ]-------------------------', + 'bigint_array | {}', + 'nested_numeric_array | ', + '配列 | {}', 'SELECT 2' ] - assert results == expected + assert list(results) == expected def test_format_output_auto_expand(): @@ -120,14 +120,14 @@ def test_format_output_auto_expand(): ['head1', 'head2'], 'test status', settings) table = [ 'Title', - '+---------+---------+\n' - '| head1 | head2 |\n' - '|---------+---------|\n' - '| abc | def |\n' + '+---------+---------+', + '| head1 | head2 |', + '|---------+---------|', + '| abc | def |', '+---------+---------+', 'test status' ] - assert table_results == table + assert list(table_results) == table expanded_results = format_output( 'Title', [('abc', 'def')], @@ -137,12 +137,12 @@ def test_format_output_auto_expand(): ) expanded = [ 'Title', - '-[ RECORD 1 ]-------------------------\n' - 'head1 | abc\n' - 'head2 | def\n', + '-[ RECORD 1 ]-------------------------', + 'head1 | abc', + 'head2 | def', 'test status' ] - assert expanded_results == expanded + assert list(expanded_results) == expanded @dbtest diff --git a/tests/test_pgexecute.py b/tests/test_pgexecute.py index 63f652902..3141dc0e5 100644 --- a/tests/test_pgexecute.py +++ b/tests/test_pgexecute.py @@ -181,25 +181,25 @@ def test_unicode_support_in_output(executor, expanded): @dbtest def test_multiple_queries_same_line(executor): result = run(executor, "select 'foo'; select 'bar'") - assert len(result) == 4 # 2 * (output+status) - assert "foo" in result[0] - assert "bar" in result[2] + assert len(result) == 12 # 2 * (output+status) * 3 lines + assert "foo" in result[3] + assert "bar" in result[9] @dbtest def test_multiple_queries_with_special_command_same_line(executor, pgspecial): result = run(executor, "select 'foo'; \d", pgspecial=pgspecial) - assert len(result) == 4 # 2 * (output+status) - assert "foo" in result[0] + assert len(result) == 11 # 2 * (output+status) * 3 lines + assert "foo" in result[3] # This is a lame check. :( - assert "Schema" in result[2] + assert "Schema" in result[7] @dbtest def test_multiple_queries_same_line_syntaxerror(executor, exception_formatter): result = run(executor, u"select 'fooé'; invalid syntax é", exception_formatter=exception_formatter) - assert u'fooé' in result[0] + assert u'fooé' in result[3] assert 'syntax error at or near "invalid"' in result[-1] @@ -210,9 +210,9 @@ def pgspecial(): @dbtest def test_special_command_help(executor, pgspecial): - result = run(executor, '\\?', pgspecial=pgspecial)[0].split('|') - assert(result[1].find(u'Command') != -1) - assert(result[2].find(u'Description') != -1) + result = run(executor, '\\?', pgspecial=pgspecial)[1].split('|') + assert u'Command' in result[1] + assert u'Description' in result[2] @dbtest