From ad15c0cab1fb8582aa451e392c08f56c7f138422 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Mon, 16 Jan 2023 10:58:12 -0300 Subject: [PATCH 1/2] Fixes for BigQuery, REPL --- sqeleton/databases/base.py | 6 +++--- sqeleton/databases/bigquery.py | 14 ++++++++------ sqeleton/repl.py | 4 +++- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/sqeleton/databases/base.py b/sqeleton/databases/base.py index cc6abc9..ea705ab 100644 --- a/sqeleton/databases/base.py +++ b/sqeleton/databases/base.py @@ -487,7 +487,7 @@ def _normalize_table_path(self, path: DbPath) -> DbPath: def parse_table_name(self, name: str) -> DbPath: return parse_table_name(name) - def _query_cursor(self, c, sql_code: str): + def _query_cursor(self, c, sql_code: str) -> QueryResult: assert isinstance(sql_code, str), sql_code try: c.execute(sql_code) @@ -499,7 +499,7 @@ def _query_cursor(self, c, sql_code: str): # logger.error(f'Caused by SQL: {sql_code}') raise - def _query_conn(self, conn, sql_code: Union[str, ThreadLocalInterpreter]) -> list: + def _query_conn(self, conn, sql_code: Union[str, ThreadLocalInterpreter]) -> QueryResult: c = conn.cursor() callback = partial(self._query_cursor, c) return apply_query(callback, sql_code) @@ -542,7 +542,7 @@ def set_conn(self): except Exception as e: self._init_error = e - def _query(self, sql_code: Union[str, ThreadLocalInterpreter]): + def _query(self, sql_code: Union[str, ThreadLocalInterpreter]) -> QueryResult: r = self._queue.submit(self._query_in_worker, sql_code) return r.result() diff --git a/sqeleton/databases/bigquery.py b/sqeleton/databases/bigquery.py index 5826fc8..0b4dc66 100644 --- a/sqeleton/databases/bigquery.py +++ b/sqeleton/databases/bigquery.py @@ -19,7 +19,7 @@ ) from ..abcs import Compilable from ..queries import this, table, SKIP, code -from .base import BaseDialect, Database, import_helper, parse_table_name, ConnectError, apply_query +from .base import BaseDialect, Database, import_helper, parse_table_name, ConnectError, apply_query, QueryResult from .base import TIMESTAMP_PRECISION_POS, ThreadLocalInterpreter, Mixin_RandomSample @@ -161,16 +161,18 @@ def _query_atom(self, sql_code: str): from google.cloud import bigquery try: - res = list(self._client.query(sql_code)) + result = self._client.query(sql_code).result() + columns = [c.name for c in result.schema] + rows = list(result) except Exception as e: msg = "Exception when trying to execute SQL code:\n %s\n\nGot error: %s" raise ConnectError(msg % (sql_code, e)) - if res and isinstance(res[0], bigquery.table.Row): - res = [tuple(self._normalize_returned_value(v) for v in row.values()) for row in res] - return res + if rows and isinstance(rows[0], bigquery.table.Row): + rows = [tuple(self._normalize_returned_value(v) for v in row.values()) for row in rows] + return QueryResult(rows, columns) - def _query(self, sql_code: Union[str, ThreadLocalInterpreter]): + def _query(self, sql_code: Union[str, ThreadLocalInterpreter]) -> QueryResult: return apply_query(self._query_atom, sql_code) def close(self): diff --git a/sqeleton/repl.py b/sqeleton/repl.py index cea1bbd..58f6471 100644 --- a/sqeleton/repl.py +++ b/sqeleton/repl.py @@ -47,7 +47,9 @@ def repl(uri): help() continue try: - schema = db.query_table_schema((table_name,)) + path = db.parse_table_name(table_name) + print('->', path) + schema = db.query_table_schema(path) except Exception as e: logging.error(e) else: From 0117d8b4cca698356b9e3dee00509db3f927b346 Mon Sep 17 00:00:00 2001 From: Erez Shinan Date: Thu, 2 Feb 2023 10:58:52 -0300 Subject: [PATCH 2/2] Improved documentation --- README.md | 27 ++++++++++++++++++--- docs/conn_editor.md | 37 +++++++++++++++++++++++++++++ docs/index.rst | 4 ++++ docs/install.md | 58 +++++++++++++++++++++++++++++++++++++++++++++ docs/intro.md | 8 +++++-- sqeleton/utils.py | 10 ++++++-- 6 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 docs/conn_editor.md create mode 100644 docs/install.md diff --git a/README.md b/README.md index 27c6751..54e4519 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,8 @@ It consists of - It is comparable to other libraries such as SQLAlchemy or PyPika, in terms of API and intended audience. However there are several notable ways in which it is different. +## Overview + ### Built for performance - Multi-threaded by default - @@ -31,11 +33,13 @@ This feature can be also used to inform the query-builder, either as an alternat The schema is used for validation when building expressions, making sure the names are correct, and that the data-types align. +(Still WIP) + ### Multi-database access Sqeleton is designed to work with several databases at the same time. Its API abstracts away as many implementation details as possible. -Databases we support: +Databases we fully support: - PostgreSQL >=10 - MySQL @@ -51,11 +55,28 @@ Databases we support: - DuckDB >=0.6 - SQLite (coming soon) - -### Documentation +## Documentation [Read the docs!](https://sqeleton.readthedocs.io) +Or jump straight to the [introduction](https://sqeleton.readthedocs.io/en/latest/intro.html). + +### Install + +Install using pip: + +```bash +pip install sqeleton +``` + +It is recommended to install the driver dependencies using pip's `[]` syntax: + +```bash +pip install 'sqeleton[mysql, postgresql]' +``` + +Read more in [install / getting started.](https://sqeleton.readthedocs.io/en/latest/install.html) + ### Basic usage ```python diff --git a/docs/conn_editor.md b/docs/conn_editor.md new file mode 100644 index 0000000..db1181c --- /dev/null +++ b/docs/conn_editor.md @@ -0,0 +1,37 @@ +# Connection Editor + +A common complaint among new users was the difficulty in setting up the connections. + +Connection URLs are admittedly confusing, and editing `.toml` files isn't always straight-forward either. + +To ease this initial difficulty, we added a `textual`-based TUI tool to sqeleton, that allows users to edit configuration files and test the connections while editing them. + +## Install + +This tool needs `textual` to run. You can install it using: + +```bash +pip install 'sqeleton[tui]' +``` + +Make sure you also have drivers for whatever database connection you're going to edit! + +## Run + +Once everything is installed, you can run the editor with the following command: + +```bash +sqeleton conn-editor +``` + +Example: + +```bash +sqeleton conn-editor ~/dbs.toml +``` + +The available actions and hotkeys will be listed in the status bar. + +Note: using the connection editor will delete comments and reformat the file! + +We recommend backing up the configuration file before editing it. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 2296231..c840b4a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,8 +3,10 @@ :caption: API Reference :hidden: + install intro supported-databases + conn_editor python-api Sqeleton @@ -26,8 +28,10 @@ For more information, `See our README `_ diff --git a/docs/install.md b/docs/install.md new file mode 100644 index 0000000..2731846 --- /dev/null +++ b/docs/install.md @@ -0,0 +1,58 @@ +# Install / Get started + +Sqeleton can be installed using pip: + +``` +pip install sqeleton +``` + +## Database drivers + +To ensure that the database drivers are compatible with sqeleton, we recommend installing them along with sqeleton, using pip's `[]` syntax: + +- `pip install 'sqeleton[mysql]'` + +- `pip install 'sqeleton[postgresql]'` + +- `pip install 'sqeleton[snowflake]'` + +- `pip install 'sqeleton[presto]'` + +- `pip install 'sqeleton[oracle]'` + +- `pip install 'sqeleton[trino]'` + +- `pip install 'sqeleton[clickhouse]'` + +- `pip install 'sqeleton[vertica]'` + +- For BigQuery, see: https://pypi.org/project/google-cloud-bigquery/ + +_Some drivers have dependencies that cannot be installed using `pip` and still need to be installed manually._ + + +It is also possible to install several databases at once. For example: + +```bash +pip install 'sqeleton[mysql, postgresql]' +``` + +Note: Some shells use `"` for escaping instead, like: + +```bash +pip install "sqeleton[mysql, postgresql]" +``` + +## Connection editor + +Sqeleton provides a TUI connection editor, that can be installed using: + +```bash +pip install 'sqeleton[tui]' +``` + +Read more [here](conn_editor.md). + +## What's next? + +Read the [introduction](intro.md) and start coding! \ No newline at end of file diff --git a/docs/intro.md b/docs/intro.md index 8543ceb..25dddf9 100644 --- a/docs/intro.md +++ b/docs/intro.md @@ -440,12 +440,12 @@ from sqeleton.abcs.mixins import AbstractMixin_NormalizeValue, AbstractMixin_Ran connect = sqeleton.connect.load_mixins(AbstractMixin_NormalizeValue) ddb = connect("duckdb://:memory:") -print(ddb.dialect.normalize_boolean("bool", None) == "bool::INTEGER::VARCHAR") +print(ddb.dialect.normalize_boolean("bool", None)) # Outputs: # bool::INTEGER::VARCHAR ``` -Each database is already aware of the available mixin implementation, because it was defined with the `MIXINS` attribute. We're only using the abstract mixins to select the mixins we want to use. +Each database is already aware of the available mixin implementations, because it was defined with the `MIXINS` attribute. We're only using the abstract mixins to select the mixins we want to use. #### List of mixins @@ -463,6 +463,10 @@ List of available abstract mixins: - `AbstractMixin_TimeTravel` - Only snowflake & bigquery +More will be added in the future. + +Note that it's still possible to use user-defined mixins that aren't on this list. + #### Unimplemented Mixins Trying to load a mixin that isn't implemented by all databases, will fail: diff --git a/sqeleton/utils.py b/sqeleton/utils.py index af6bd75..895b02f 100644 --- a/sqeleton/utils.py +++ b/sqeleton/utils.py @@ -1,4 +1,4 @@ -from typing import Iterable, Iterator, MutableMapping, Union, Any, Sequence, Dict, Hashable, TypeVar, TYPE_CHECKING +from typing import Iterable, Iterator, MutableMapping, Union, Any, Sequence, Dict, Hashable, TypeVar, TYPE_CHECKING, List from abc import abstractmethod from weakref import ref import math @@ -251,6 +251,12 @@ def __lt__(self, other): return NotImplemented return self._str < other._str + def __eq__(self, other): + if not isinstance(other, type(self)): + return NotImplemented + return self._str == other._str + + def new(self, *args, **kw): return type(self)(*args, **kw, max_len=self._max_len) @@ -266,7 +272,7 @@ def number_to_human(n): return "{:.0f}{}".format(n / 10 ** (3 * millidx), millnames[millidx]) -def split_space(start, end, count): +def split_space(start, end, count) -> List[int]: size = end - start assert count <= size, (count, size) return list(range(start, end, (size + 1) // (count + 1)))[1 : count + 1]