diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..8c6bd68 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,15 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**Describe the environment** + +Describe which OS you're using, which sqeleton version, and any other information that might be relevant to this bug. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..19f8813 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/request-support-for-a-database.md b/.github/ISSUE_TEMPLATE/request-support-for-a-database.md new file mode 100644 index 0000000..71a72ad --- /dev/null +++ b/.github/ISSUE_TEMPLATE/request-support-for-a-database.md @@ -0,0 +1,10 @@ +--- +name: Request support for a database +about: 'Request a driver to support a new database ' +title: 'Add support for ' +labels: new-db-driver +assignees: '' + +--- + + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..c2c3984 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI-COVER-VERSIONS + +on: +# push: +# paths: +# - '**.py' +# - '.github/workflows/**' +# - '!dev/**' + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + unit_tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + + name: Check Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Build the stack + run: docker-compose up -d mysql postgres presto trino clickhouse vertica + + - name: Install Poetry + run: pip install poetry + + - name: Install package + run: "poetry install" + + - name: Run unit tests + env: + PRESTO_URI: 'presto://presto@127.0.0.1/postgresql/public' + TRINO_URI: 'trino://postgres@127.0.0.1:8081/postgresql/public' + CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' + VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica' + SNOWFLAKE_URI: '${{ secrets.SNOWFLAKE_URI }}' + REDSHIFT_URI: '${{ secrets.REDSHIFT_URI }}' + run: | + chmod +x tests/waiting_for_stack_up.sh + ./tests/waiting_for_stack_up.sh && TEST_ACROSS_ALL_DBS=0 poetry run unittest-parallel -j 16 diff --git a/.github/workflows/ci_full.yml b/.github/workflows/ci_full.yml new file mode 100644 index 0000000..85c4333 --- /dev/null +++ b/.github/workflows/ci_full.yml @@ -0,0 +1,51 @@ +name: CI-COVER-DATABASES + +on: +# push: +# paths: +# - '**.py' +# - '.github/workflows/**' +# - '!dev/**' + pull_request: + branches: [ master ] + workflow_dispatch: + +jobs: + unit_tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: + - "3.10" + + name: Check Python ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Build the stack + run: docker-compose up -d mysql postgres presto trino clickhouse vertica + + - name: Install Poetry + run: pip install poetry + + - name: Install package + run: "poetry install" + + - name: Run unit tests + env: + PRESTO_URI: 'presto://presto@127.0.0.1/postgresql/public' + TRINO_URI: 'trino://postgres@127.0.0.1:8081/postgresql/public' + CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' + VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica' + SNOWFLAKE_URI: '${{ secrets.SNOWFLAKE_URI }}' + REDSHIFT_URI: '${{ secrets.REDSHIFT_URI }}' + run: | + chmod +x tests/waiting_for_stack_up.sh + ./tests/waiting_for_stack_up.sh && poetry run unittest-parallel -j 16 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1aa1541 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Mac +.DS_Store + +# IntelliJ +.idea + +# VSCode +.vscode diff --git a/dev/Dockerfile.prestosql.340 b/dev/Dockerfile.prestosql.340 new file mode 100644 index 0000000..0b21113 --- /dev/null +++ b/dev/Dockerfile.prestosql.340 @@ -0,0 +1,25 @@ +FROM openjdk:11-jdk-slim-buster + +ENV PRESTO_VERSION=340 +ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/io/prestosql/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz +ENV PRESTO_CLI_URL=https://repo1.maven.org/maven2/io/prestosql/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar +ENV PRESTO_HOME=/opt/presto +ENV PATH=${PRESTO_HOME}/bin:${PATH} + +WORKDIR $PRESTO_HOME + +RUN set -xe \ + && apt-get update \ + && apt-get install -y curl less python \ + && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \ + && curl -sSL $PRESTO_CLI_URL > ./bin/presto \ + && chmod +x ./bin/presto \ + && apt-get remove -y curl \ + && rm -rf /var/lib/apt/lists/* + +VOLUME /data + +EXPOSE 8080 + +ENTRYPOINT ["launcher"] +CMD ["run"] diff --git a/dev/dev.env b/dev/dev.env new file mode 100644 index 0000000..cb5eea1 --- /dev/null +++ b/dev/dev.env @@ -0,0 +1,23 @@ +POSTGRES_USER=postgres +POSTGRES_PASSWORD=Password1 +POSTGRES_DB=postgres + +MYSQL_DATABASE=mysql +MYSQL_USER=mysql +MYSQL_PASSWORD=Password1 +MYSQL_ROOT_PASSWORD=RootPassword1 + +CLICKHOUSE_USER=clickhouse +CLICKHOUSE_PASSWORD=Password1 +CLICKHOUSE_DB=clickhouse +CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + +# Vertica credentials +APP_DB_USER=vertica +APP_DB_PASSWORD=Password1 +VERTICA_DB_NAME=vertica + +# To prevent generating sample demo VMart data (more about it here https://www.vertica.com/docs/9.2.x/HTML/Content/Authoring/GettingStartedGuide/IntroducingVMart/IntroducingVMart.htm), +# leave VMART_DIR and VMART_ETL_SCRIPT empty. +VMART_DIR= +VMART_ETL_SCRIPT= diff --git a/dev/presto-conf/standalone/catalog/jmx.properties b/dev/presto-conf/standalone/catalog/jmx.properties new file mode 100644 index 0000000..b6e0372 --- /dev/null +++ b/dev/presto-conf/standalone/catalog/jmx.properties @@ -0,0 +1 @@ +connector.name=jmx diff --git a/dev/presto-conf/standalone/catalog/memory.properties b/dev/presto-conf/standalone/catalog/memory.properties new file mode 100644 index 0000000..833abd3 --- /dev/null +++ b/dev/presto-conf/standalone/catalog/memory.properties @@ -0,0 +1 @@ +connector.name=memory diff --git a/dev/presto-conf/standalone/catalog/postgresql.properties b/dev/presto-conf/standalone/catalog/postgresql.properties new file mode 100644 index 0000000..ef47996 --- /dev/null +++ b/dev/presto-conf/standalone/catalog/postgresql.properties @@ -0,0 +1,5 @@ +connector.name=postgresql +connection-url=jdbc:postgresql://postgres:5432/postgres +connection-user=postgres +connection-password=Password1 +allow-drop-table=true diff --git a/dev/presto-conf/standalone/catalog/tpcds.properties b/dev/presto-conf/standalone/catalog/tpcds.properties new file mode 100644 index 0000000..ba8147d --- /dev/null +++ b/dev/presto-conf/standalone/catalog/tpcds.properties @@ -0,0 +1 @@ +connector.name=tpcds diff --git a/dev/presto-conf/standalone/catalog/tpch.properties b/dev/presto-conf/standalone/catalog/tpch.properties new file mode 100644 index 0000000..75110c5 --- /dev/null +++ b/dev/presto-conf/standalone/catalog/tpch.properties @@ -0,0 +1 @@ +connector.name=tpch diff --git a/dev/presto-conf/standalone/combined.pem b/dev/presto-conf/standalone/combined.pem new file mode 100644 index 0000000..a2132a1 --- /dev/null +++ b/dev/presto-conf/standalone/combined.pem @@ -0,0 +1,52 @@ +-----BEGIN CERTIFICATE----- +MIIERDCCAyygAwIBAgIUBxO/CflDP+0yZAXt5FKm/WQ4538wDQYJKoZIhvcNAQEL +BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X +DTIyMDgyNDA4NTI1N1oXDTMyMDgyMTA4NTI1N1owWTELMAkGA1UEBhMCQVUxEzAR +BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5 +IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEA3lNywkj/eGPGoFA3Lcx++98l17CRy+uzZMtJsr6lYAg1p/n1vPw0 +BQXI5TSBJ6vM/axtwgwrfXQsjQ/GYJKQkb6eEBCc3xb+Rk5HNBiBBZsIjYm0U1zz +7dKnNwAznjx3j72s2ZQiqkoxcu7Bctw28ynbg0rjNkuUk3QESKuOgaTltpWKZiiu +XwWasREeH6MH7ROy8db6cz+MwGaig0mUvGPmD97bPRD/X683RyOiXzEaogl/rpGK +qZ3jRsmS8ZwawzKxx16kqPsX8/01EruGIoubMttr3YoZG044zq7nQqdAAz6wXx6V +mgzToCHI+/g+8JS/bgqJTyb2Y6aGXExiuQIDAQABo4IBAjCB/zAdBgNVHQ4EFgQU +5i1F8pTnwjFxw6W/0RjwpaJaK9MwgZYGA1UdIwSBjjCBi4AU5i1F8pTnwjFxw6W/ +0RjwpaJaK9OhXaRbMFkxCzAJBgNVBAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRl +MSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxEjAQBgNVBAMMCWxv +Y2FsaG9zdIIUBxO/CflDP+0yZAXt5FKm/WQ4538wDAYDVR0TBAUwAwEB/zALBgNV +HQ8EBAMCAvwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MBQGA1UdEgQNMAuCCWxvY2Fs +aG9zdDANBgkqhkiG9w0BAQsFAAOCAQEAjBQLl/UFSd9TH2VLM1GH8bixtEJ9+rm7 +x+Jw665+XLjW107dJ33qxy9zjd3cZ2fynKg2Tb7+9QAvSlqpt2YMGP9jr4W2w16u +ngbNB+kfoOotcUChk90aHmdHLZgOOve/ArFIvbr8douLOn0NAJBrj+iX4zC1pgEC +9hsMUekkAPIcCGc0rEkEc8r8uiUBWNAdEWpBt0X2fE1ownLuB/E/+3HutLTw8Lv0 +b+jNt/vogVixcw/FF4atoO+F7S5FYzAb0U7YXaNISfVPVBsA89oPy7PlxULHDUIF +Iq+vVqKdj1EXR+Iec0TMiMsa3MnIGkpL7ZuUXaG+xGBaVhGrUp67lQ== +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIIEogIBAAKCAQEA3lNywkj/eGPGoFA3Lcx++98l17CRy+uzZMtJsr6lYAg1p/n1 +vPw0BQXI5TSBJ6vM/axtwgwrfXQsjQ/GYJKQkb6eEBCc3xb+Rk5HNBiBBZsIjYm0 +U1zz7dKnNwAznjx3j72s2ZQiqkoxcu7Bctw28ynbg0rjNkuUk3QESKuOgaTltpWK +ZiiuXwWasREeH6MH7ROy8db6cz+MwGaig0mUvGPmD97bPRD/X683RyOiXzEaogl/ +rpGKqZ3jRsmS8ZwawzKxx16kqPsX8/01EruGIoubMttr3YoZG044zq7nQqdAAz6w +Xx6VmgzToCHI+/g+8JS/bgqJTyb2Y6aGXExiuQIDAQABAoIBAD3pKwnjXhDeaA94 +hwUf7zSgfV9E8jTBHCGzYoB+CntljduLBd1stee4Jqt9JYIwm1MA00e4L9wtn8Jg +ZDO8XLnZRRbgKW8ObhyR684cDMHM3GLdt/OG7P6LLLlqOvWTjQ/gF+Q3FjgplP+W +cRRVMpAgVdqH3iHehi9RnWfHLlX3WkBC97SumWFzWqBnqUQAC8AvFHiUCqA9qIeA +8ieOEoE17yv2nkmu+A5OZoCXtVfc2lQ90Fj9QZiv4rIVXBtTRRURJuvi2iX3nOPl +MsjAUIBK1ndzpJ7wuLICSR1U3/npPC6Va06lTm0H/Q6DEqZjEHbx9TGY3pTgVXuA ++G0C5GkCgYEA/spvoDUMZrH2JE43TT/qMEHPtHW4qT6fTmzu08Gx++8nFmddNgSD +zrdjxqPMUGV7Q5smwoHaQyqFxHMM2jh8icnV6VoBDrDdZM0eGFAs6HUjKmyaAdQO +dC4kPiy3LX5pJUnQnmwq1fVsgXWGQF/LhD0L/y6xOiqdhZp/8nv6SFMCgYEA32GR +gWJQdgWXTXxSEDn0twKPevgAT0s778/7h5osCLG82Q7ab2+Fc1qTleiwiQ2SAuOl +mWvtz0Eg4dYe/q6jugqkEgAYZvdHGL7CSmC28O98fTLapgKQC5GUUan90sCbRec4 +kjbyx5scICNBYJVchdFg6UUSNz5czORUVgQEF0MCgYB1toUX2Spfj7yOTWyTTgIe +RWl2kCS+XGYxT3aPcp+OK5E9cofH2xIiQOvh6+8K/beTJm0j0+ZIva6LcjPv5cTz +y8H+S0zNwrymQ3Wx+eilhOi4QvBsA9KhrmekKfh/FjXxukadyo+HxhlZPjjGKPvX +nnSacrICk4mvHhAasViSbQKBgD7mZGiAXJO/I0moVhtHlobp66j+qGerkacHc5ZN +bVTNZ5XfPtbeGj/PI3u01/Dfp1u06m53G7GebznoZzXjyyqZ0HVZHYXw304yeNck +wJ67cNx4M2VHl3QKfC86pMRxg8d9Qkq5ukdGf/b0tnYR2Mm9mYJV9rkjkFIJgU3v +N4+tAoGABOlVGuRx2cSQ9QeC0AcqKlxXygdrzyadA7i0KNBZGGyrMSpJDrl2rrRn +ylzAgGjvfilwQzZuqTm6Vo2yvaX+TTGS44B+DnxCZvuviftea++sNMjuEkBLTCpF +xk2yOzsOnx652kWO4L+dVrDAxl65f3v0YaKWZI504LFYl18uS/E= +-----END RSA PRIVATE KEY----- diff --git a/dev/presto-conf/standalone/config.properties b/dev/presto-conf/standalone/config.properties new file mode 100644 index 0000000..d54e2a1 --- /dev/null +++ b/dev/presto-conf/standalone/config.properties @@ -0,0 +1,8 @@ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +query.max-memory=5GB +query.max-memory-per-node=1GB +query.max-total-memory-per-node=2GB +discovery-server.enabled=true +discovery.uri=http://127.0.0.1:8080 diff --git a/dev/presto-conf/standalone/jvm.config b/dev/presto-conf/standalone/jvm.config new file mode 100644 index 0000000..afd8e8a --- /dev/null +++ b/dev/presto-conf/standalone/jvm.config @@ -0,0 +1,9 @@ +-server +-Xmx16G +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+UseGCOverheadLimit +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-XX:OnOutOfMemoryError=kill -9 %p diff --git a/dev/presto-conf/standalone/log.properties b/dev/presto-conf/standalone/log.properties new file mode 100644 index 0000000..1c52627 --- /dev/null +++ b/dev/presto-conf/standalone/log.properties @@ -0,0 +1 @@ +com.facebook.presto=INFO diff --git a/dev/presto-conf/standalone/node.properties b/dev/presto-conf/standalone/node.properties new file mode 100644 index 0000000..f2cf0e0 --- /dev/null +++ b/dev/presto-conf/standalone/node.properties @@ -0,0 +1,3 @@ +node.environment=production +node.data-dir=/data +node.id=standalone diff --git a/dev/presto-conf/standalone/password-authenticator.properties b/dev/presto-conf/standalone/password-authenticator.properties new file mode 100644 index 0000000..68ced47 --- /dev/null +++ b/dev/presto-conf/standalone/password-authenticator.properties @@ -0,0 +1,2 @@ +password-authenticator.name=file +file.password-file=/opt/presto/etc/password.db diff --git a/dev/presto-conf/standalone/password.db b/dev/presto-conf/standalone/password.db new file mode 100644 index 0000000..6d49930 --- /dev/null +++ b/dev/presto-conf/standalone/password.db @@ -0,0 +1 @@ +test:$2y$10$877iU3J5a26SPDjFSrrz2eFAq2DwMDsBAus92Dj0z5A5qNMNlnpHa diff --git a/dev/trino-conf/etc/catalog/jms.properties b/dev/trino-conf/etc/catalog/jms.properties new file mode 100644 index 0000000..b6e0372 --- /dev/null +++ b/dev/trino-conf/etc/catalog/jms.properties @@ -0,0 +1 @@ +connector.name=jmx diff --git a/dev/trino-conf/etc/catalog/memory.properties b/dev/trino-conf/etc/catalog/memory.properties new file mode 100644 index 0000000..56b3b5e --- /dev/null +++ b/dev/trino-conf/etc/catalog/memory.properties @@ -0,0 +1,2 @@ +connector.name=memory +memory.max-data-per-node=128MB diff --git a/dev/trino-conf/etc/catalog/postgresql.properties b/dev/trino-conf/etc/catalog/postgresql.properties new file mode 100644 index 0000000..b6bc300 --- /dev/null +++ b/dev/trino-conf/etc/catalog/postgresql.properties @@ -0,0 +1,4 @@ +connector.name=postgresql +connection-url=jdbc:postgresql://postgres:5432/postgres +connection-user=postgres +connection-password=Password1 diff --git a/dev/trino-conf/etc/catalog/tpcds.properties b/dev/trino-conf/etc/catalog/tpcds.properties new file mode 100644 index 0000000..ba8147d --- /dev/null +++ b/dev/trino-conf/etc/catalog/tpcds.properties @@ -0,0 +1 @@ +connector.name=tpcds diff --git a/dev/trino-conf/etc/catalog/tpch.properties b/dev/trino-conf/etc/catalog/tpch.properties new file mode 100644 index 0000000..75110c5 --- /dev/null +++ b/dev/trino-conf/etc/catalog/tpch.properties @@ -0,0 +1 @@ +connector.name=tpch diff --git a/dev/trino-conf/etc/config.properties b/dev/trino-conf/etc/config.properties new file mode 100644 index 0000000..6553add --- /dev/null +++ b/dev/trino-conf/etc/config.properties @@ -0,0 +1,5 @@ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://localhost:8080 +discovery-server.enabled=true diff --git a/dev/trino-conf/etc/jvm.config b/dev/trino-conf/etc/jvm.config new file mode 100644 index 0000000..34ee130 --- /dev/null +++ b/dev/trino-conf/etc/jvm.config @@ -0,0 +1,12 @@ +-server +-Xmx1G +-XX:-UseBiasedLocking +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+UseGCOverheadLimit +-XX:+ExitOnOutOfMemoryError +-XX:ReservedCodeCacheSize=256M +-Djdk.attach.allowAttachSelf=true +-Djdk.nio.maxCachedBufferSize=2000000 \ No newline at end of file diff --git a/dev/trino-conf/etc/node.properties b/dev/trino-conf/etc/node.properties new file mode 100644 index 0000000..aaf7398 --- /dev/null +++ b/dev/trino-conf/etc/node.properties @@ -0,0 +1,3 @@ +node.environment=docker +node.data-dir=/data/trino +plugin.dir=/usr/lib/trino/plugin diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..60bab06 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,130 @@ +version: "3.8" + +services: + postgres: + container_name: dd-postgresql + image: postgres:14.1-alpine + # work_mem: less tmp files + # maintenance_work_mem: improve table-level op perf + # max_wal_size: allow more time before merging to heap + command: > + -c work_mem=1GB + -c maintenance_work_mem=1GB + -c max_wal_size=8GB + restart: always + volumes: + - postgresql-data:/var/lib/postgresql/data:delegated + ports: + - '5432:5432' + expose: + - '5432' + env_file: + - dev/dev.env + tty: true + networks: + - local + + mysql: + container_name: dd-mysql + image: mysql:oracle + # fsync less aggressively for insertion perf for test setup + command: > + --default-authentication-plugin=mysql_native_password + --binlog-cache-size=16M + --key_buffer_size=0 + --max_connections=1000 + --innodb_flush_log_at_trx_commit=2 + --innodb_flush_log_at_timeout=10 + --innodb_log_compressed_pages=OFF + --sync_binlog=0 + restart: always + volumes: + - mysql-data:/var/lib/mysql:delegated + user: mysql + ports: + - '3306:3306' + expose: + - '3306' + env_file: + - dev/dev.env + tty: true + networks: + - local + + clickhouse: + container_name: dd-clickhouse + image: clickhouse/clickhouse-server:21.12.3.32 + restart: always + volumes: + - clickhouse-data:/var/lib/clickhouse:delegated + ulimits: + nproc: 65535 + nofile: + soft: 262144 + hard: 262144 + ports: + - '8123:8123' + - '9000:9000' + expose: + - '8123' + - '9000' + env_file: + - dev/dev.env + tty: true + networks: + - local + + # prestodb.dbapi.connect(host="127.0.0.1", user="presto").cursor().execute('SELECT * FROM system.runtime.nodes') + presto: + container_name: dd-presto + build: + context: ./dev + dockerfile: ./Dockerfile.prestosql.340 + volumes: + - ./dev/presto-conf/standalone:/opt/presto/etc:ro + ports: + - '8080:8080' + tty: true + networks: + - local + + trino: + container_name: dd-trino + image: 'trinodb/trino:389' + hostname: trino + ports: + - '8081:8080' + volumes: + - ./dev/trino-conf/etc:/etc/trino:ro + networks: + - local + + vertica: + container_name: dd-vertica + image: vertica/vertica-ce:12.0.0-0 + restart: always + volumes: + - vertica-data:/data:delegated + ports: + - '5433:5433' + - '5444:5444' + expose: + - '5433' + - '5444' + env_file: + - dev/dev.env + tty: true + networks: + - local + + + +volumes: + postgresql-data: + mysql-data: + clickhouse-data: + vertica-data: + +networks: + local: + driver: bridge diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..0f98ab4 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,1143 @@ +[[package]] +name = "asn1crypto" +version = "1.5.1" +description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "backports.zoneinfo" +version = "0.2.1" +description = "Backport of the standard library zoneinfo module" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +tzdata = ["tzdata"] + +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +pycparser = "*" + +[[package]] +name = "charset-normalizer" +version = "2.1.1" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.6.0" + +[package.extras] +unicode_backport = ["unicodedata2"] + +[[package]] +name = "click" +version = "8.1.3" +description = "Composable command line interface toolkit" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "clickhouse-driver" +version = "0.2.5" +description = "Python driver with native interface for ClickHouse" +category = "main" +optional = false +python-versions = ">=3.6, <4" + +[package.dependencies] +pytz = "*" +tzlocal = "*" + +[package.extras] +lz4 = ["clickhouse-cityhash (>=1.0.2.1)", "lz4", "lz4 (<=3.0.1)"] +numpy = ["numpy (>=1.12.0)", "pandas (>=0.24.0)"] +zstd = ["clickhouse-cityhash (>=1.0.2.1)", "zstd"] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "coverage" +version = "6.5.0" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "cryptography" +version = "38.0.4" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cffi = ">=1.12" + +[package.extras] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] +pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] +sdist = ["setuptools-rust (>=0.11.4)"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] + +[[package]] +name = "dsnparse" +version = "0.1.15" +description = "parse dsn urls" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "duckdb" +version = "0.6.1" +description = "DuckDB embedded database" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +numpy = ">=1.14" + +[[package]] +name = "filelock" +version = "3.8.2" +description = "A platform independent file lock." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo (>=2022.9.29)", "sphinx (>=5.3)", "sphinx-autodoc-typehints (>=1.19.5)"] +testing = ["covdefaults (>=2.2.2)", "coverage (>=6.5)", "pytest (>=7.2)", "pytest-cov (>=4)", "pytest-timeout (>=2.1)"] + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "importlib-metadata" +version = "5.1.0" +description = "Read metadata from Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +perf = ["ipython"] +testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] + +[[package]] +name = "mysql-connector-python" +version = "8.0.29" +description = "MySQL driver written in Python" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +protobuf = ">=3.0.0" + +[package.extras] +compression = ["lz4 (>=2.1.6)", "zstandard (>=0.12.0)"] +dns-srv = ["dnspython (>=1.16.0)"] +gssapi = ["gssapi (>=1.6.9)"] + +[[package]] +name = "numpy" +version = "1.21.1" +description = "NumPy is the fundamental package for array computing with Python." +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "oscrypto" +version = "1.3.0" +description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +asn1crypto = ">=1.5.1" + +[[package]] +name = "parameterized" +version = "0.8.1" +description = "Parameterized testing with any Python test framework" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +dev = ["jinja2"] + +[[package]] +name = "presto-python-client" +version = "0.8.3" +description = "Client for the Presto distributed SQL Engine" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +click = "*" +requests = "*" +six = "*" + +[package.extras] +all = ["google-auth", "requests-kerberos"] +google_auth = ["google-auth"] +kerberos = ["requests-kerberos"] +tests = ["google-auth", "httpretty", "pytest", "pytest-runner", "requests-kerberos"] + +[[package]] +name = "protobuf" +version = "4.21.12" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "psycopg2" +version = "2.9.5" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" + +[[package]] +name = "pycryptodomex" +version = "3.16.0" +description = "Cryptographic library for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "Pygments" +version = "2.13.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +plugins = ["importlib-metadata"] + +[[package]] +name = "PyJWT" +version = "2.6.0" +description = "JSON Web Token implementation in Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + +[[package]] +name = "pyOpenSSL" +version = "22.1.0" +description = "Python wrapper module around the OpenSSL library" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cryptography = ">=38.0.0,<39" + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0)", "sphinx-rtd-theme"] +test = ["flaky", "pretend", "pytest (>=3.0.1)"] + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pytz" +version = "2022.6" +description = "World timezone definitions, modern and historical" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "pytz-deprecation-shim" +version = "0.1.0.post0" +description = "Shims to make deprecation of pytz easier" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" + +[package.dependencies] +"backports.zoneinfo" = {version = "*", markers = "python_version >= \"3.6\" and python_version < \"3.9\""} +tzdata = {version = "*", markers = "python_version >= \"3.6\""} + +[[package]] +name = "requests" +version = "2.28.1" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<3" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "rich" +version = "12.6.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "runtype" +version = "0.2.7" +description = "Type dispatch and validation for run-time Python" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[[package]] +name = "setuptools" +version = "65.6.3" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "snowflake-connector-python" +version = "2.9.0" +description = "Snowflake Connector for Python" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +certifi = ">=2017.4.17" +cffi = ">=1.9,<2.0.0" +charset-normalizer = ">=2,<3" +cryptography = ">=3.1.0,<41.0.0" +filelock = ">=3.5,<4" +idna = ">=2.5,<4" +oscrypto = "<2.0.0" +pycryptodomex = ">=3.2,<3.5.0 || >3.5.0,<4.0.0" +pyjwt = "<3.0.0" +pyOpenSSL = ">=16.2.0,<23.0.0" +pytz = "*" +requests = "<3.0.0" +setuptools = ">34.0.0" +typing-extensions = ">=4.3,<5" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +development = ["Cython", "coverage", "more-itertools", "numpy (<1.24.0)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.3.0)", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytzdata"] +pandas = ["pandas (>=1.0.0,<1.6.0)", "pyarrow (>=8.0.0,<8.1.0)"] +secure-local-storage = ["keyring (!=16.1.0,<24.0.0)"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "trino" +version = "0.314.0" +description = "Client for the Trino distributed SQL Engine" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +pytz = "*" +requests = "*" + +[package.extras] +all = ["requests-kerberos", "sqlalchemy (>=1.3,<2.0)"] +external-authentication-token-cache = ["keyring"] +kerberos = ["requests-kerberos"] +sqlalchemy = ["sqlalchemy (>=1.3,<2.0)"] +tests = ["click", "httpretty (<1.1)", "pytest", "pytest-runner", "requests-kerberos", "sqlalchemy (>=1.3,<2.0)"] + +[[package]] +name = "typing-extensions" +version = "4.4.0" +description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "tzdata" +version = "2022.7" +description = "Provider of IANA time zone data" +category = "main" +optional = false +python-versions = ">=2" + +[[package]] +name = "tzlocal" +version = "4.2" +description = "tzinfo object for the local timezone" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +"backports.zoneinfo" = {version = "*", markers = "python_version < \"3.9\""} +pytz-deprecation-shim = "*" +tzdata = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +devenv = ["black", "pyroma", "pytest-cov", "zest.releaser"] +test = ["pytest (>=4.3)", "pytest-mock (>=3.3)"] + +[[package]] +name = "unittest-parallel" +version = "1.5.3" +description = "Parallel unit test runner with coverage support" +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +coverage = ">=5.1" + +[[package]] +name = "urllib3" +version = "1.26.13" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[[package]] +name = "vertica-python" +version = "1.1.1" +description = "Official native Python client for the Vertica database." +category = "dev" +optional = false +python-versions = "*" + +[package.dependencies] +python-dateutil = ">=1.5" +six = ">=1.10.0" + +[[package]] +name = "zipp" +version = "3.11.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)"] +testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] + +[extras] +clickhouse = ["clickhouse-driver"] +duckdb = ["duckdb"] +mysql = ["mysql-connector-python"] +oracle = [] +postgresql = ["psycopg2"] +presto = ["presto-python-client"] +snowflake = ["snowflake-connector-python", "cryptography"] +trino = ["trino"] +vertica = [] + +[metadata] +lock-version = "1.1" +python-versions = "^3.7" +content-hash = "93d803ff31252129545c881c7ef4e65cbcba1668f7513c776273b5fe719bf774" + +[metadata.files] +asn1crypto = [ + {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, + {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, +] +"backports.zoneinfo" = [ + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win32.whl", hash = "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08"}, + {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win32.whl", hash = "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b"}, + {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-win32.whl", hash = "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328"}, + {file = "backports.zoneinfo-0.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6"}, + {file = "backports.zoneinfo-0.2.1.tar.gz", hash = "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"}, +] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +cffi = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] +charset-normalizer = [ + {file = "charset-normalizer-2.1.1.tar.gz", hash = "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845"}, + {file = "charset_normalizer-2.1.1-py3-none-any.whl", hash = "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f"}, +] +click = [ + {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, + {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, +] +clickhouse-driver = [ + {file = "clickhouse-driver-0.2.5.tar.gz", hash = "sha256:403239e5820d07d35a380f1abcd45ea0397fe8059984abf0594783d1474d56f5"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:413c7116a0f35bfc12c22f616b4cc9673d74b293c90361c748be03a987c753c4"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc7d48fc72ddb5a24c59b521c7ff56e7bcc8520a9b4afe50d7efaec1aa4ae616"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5fbaef87d62bdf932c6d2f86c3a77d4675d309b9dd8c579b94776ec3da044bdf"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fda807b08ed23c6388c734486dcd218adefe78cac298f08dbc3a62d27d20462a"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec2eb78e361ff0f4f45b694e2d08e58c00329ee512471b23f53a23815dc34660"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c70a2f0c8bd8165e63788aa64b583a2c92ff73ca0a07b60ef7980250cee24a3d"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:77c541961f29f17880bece8c5cf22f52564579c086fe859568189ccb36cef7cb"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3c0f5567e935b09ac6836e2a015306a6233538111be2a1067997fd7fdb865bed"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:36077623489c1558495cecfb682a777d33cb3f90b0d1e61558fe323d39cd736b"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ff24c5425b926eb18e434aa977f2ac1dfa6f907245fc1989633c07b67973843c"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d4fa637d702aca6993b5b03c20c910743e2a2e6a38661b8e9449b898d836b049"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-win32.whl", hash = "sha256:319c4ac6e063ca6786ca25aac86685841523d421f5483dc890589d221d36d562"}, + {file = "clickhouse_driver-0.2.5-cp310-cp310-win_amd64.whl", hash = "sha256:f0cb4b9b1dd316611e4de858eb82a4f70fd1807cce9919aeed5b7bfd320ca3dc"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4605f2f197c07c6db91b9636d29e50322f095b006b6aca0f5f958bbbfe3a4381"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dc5543eb45e556a45ed57328446971ad3f786f8230fbb96d736ad51ca6a0f085"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ab396a5ad5c54d463bac1b268373b7977d076c3c7ec474fc5fc908629b7e989"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:758c3d0c09d6c23f9014045aeadf3a2620c3763fc0b2875a1a8e213092afb92b"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe9781724eaf2a0cd85a1f1b2906867d548399f4ecc6b8b7460f20410eb036b5"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e8aa79ddfb67ae2b6bf844d1493ab83321c4f2052686c6aa93f6658c737bd12"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c8092518f839e01edbbe74db4c3d902028f9f4af5410571a131dac40f190edd4"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:019da690c9b82034ff7ebe2a2ac2af87b715a2c010f9a65fefc08c4bd792b0d6"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:fc300a72f0a09c2c286a55723466315a1b307f94a2ed81d1b7802598cd451059"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:937b663c400233d5ddd8b50fc260c3e374b7803dd401cb194ff64ddce9825aac"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e577b20814021e6851a7a3c075b1b4498c07ee5c1863481a6a1558a85739e145"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-win32.whl", hash = "sha256:d0d8f06a9264e1279c0aefc5dd7a6c6cdc4ce2d3aac3565865250db92694c9e7"}, + {file = "clickhouse_driver-0.2.5-cp311-cp311-win_amd64.whl", hash = "sha256:3a0741b32cf2ed019aba7c65e198d33a74d91b4ca67d5918860b7a32b7a5c158"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:1edf78ff50587bde5775bfa98a17764d9fa2588e7841d63fadfca169277438d8"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e225d9261fbd35024b87b83c9d703331247d64583aedeb1210a700058422548e"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5db4eccf1f9ff685b6a92b3755cfea612f58868c513344f71927697e93e7103"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2b4fe92b120faae255354d6cf4a45639236583add948401e2b72600be793a1a"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85e1b9618dbd0772497556e7b6dcf8063191c0e2ea9d7ba8e447e70aea0a88c8"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:02c540813d644f1de73b0c033c10671b6c2ce97b46a233ac1936d71cc527d48f"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:d9d27f28fe2d3ca3e9ef995466b44ee7863de61a671e1e305853e235c4f4b36e"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:ee77062908490d111f65cc795cad9db2d3952630905f44ad17cb0b71374cffad"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:2a3e9d7dc9e58c7fee174516d00ad00b81fa6e4da56461d8ac206c28a16ad44c"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:d73c9dbc146d1456ea26c87f8b245f7ca69e5bc1883521fbc9074c3dd74a439a"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:14ce9ea8e4270307563bb95a1e39d8e9d6130cda926b33abbfb17d826b23acf0"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-win32.whl", hash = "sha256:d1cc49e8579b22d7fd2a8550b2da294bc21cc00189fa71ecf46b2f939011654c"}, + {file = "clickhouse_driver-0.2.5-cp36-cp36m-win_amd64.whl", hash = "sha256:e6693c1ee9c6c248fe31db716f4f0bdeff84ea8f1b4262c4d941f58d4a3a58d5"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5e1d7b6d22dd5efd771aa925a586185a51a64eeb908d61d5bfdcbb1416608d89"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20c8eb194897c7826d82aa67ac2f92496214cdac3a3713df5c48d94fff494593"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0631980b28267f1137b08e6982f244597c0809478edeaa9d6b8046f38e1369ed"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:86c4b3bf77d3f81394b05dc4795485a95076bbe85708d14de261d7f961b0eff5"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:704a68cec8ce9997d2a8d43678852cbcc734b77fbe8c9b4b6c3aee9e09fed9ee"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8884f12911202c7b22502d1224cffb84559c91f93bcb1deb6a28251f7a82e98e"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:45852b9c90fa75a70c2a98556ad0a031a2a1c40e5bffe6dcf252c8cbace1f2b4"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:780843107ad370fdf1742aa68196158e395345ef72bb10f12a63ab82ea6f823d"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f8ceb503bacd0234d28c2e99ad61ca0d64d3282d8531c3607a3d7a849aedaf01"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:fd9116181a645c9393e5ef9287fc8747ce25ce5a1d8d8cf079223de8bdb00236"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:efcb67627233f377f6057388b88b9f68a98f06588da24d073166fed9b5e4e875"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-win32.whl", hash = "sha256:6876827eaf7b1c9711a54a1241cc7be9417ba4885c89e6dd49205417e870a261"}, + {file = "clickhouse_driver-0.2.5-cp37-cp37m-win_amd64.whl", hash = "sha256:82835a6dca0016ac50d0b29eb02e9609f87f99af83e99fb0c5f15f893404c8bf"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e577d3e6d83cf71cf3dd29f5914448eb4d37adb61d9585a06642bd21c0b4bf6a"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7e95ad1e7d8fa42b79b44a8fad5db1a92c6e09534f1a72209e0e6a00fdf2c148"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a61ec8e789f47933efb88c2d34af27e664a15f064d53411bfa2a56a34c1c8c39"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cf671bac0d8f5e83736547c5ee3935ec859a77d5ce75cdfbbb5313e87cb3cb0b"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:44f01d4a257b928e5981b8a0a0193c28a48ce3aed8b06f7a32f717ad2224836d"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:64bf7aad1d2b2c22dbccfaebcca55469558b0a754911b25e3a8506c4cc15cf01"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0e3762a866321af6a7ece560ed89c072e7bd6381fe642343e9d47c47ff191e23"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:9f152bfe68af34468280f780b622a0c5dc0a5f19fd352542bf3491be4654cc13"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:9954ca8b56e81f077d3f3845f88fabbf223eb4260b09691c3d957db7ff31f0d5"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:5b011ee7340a73fda603e009f5f5463aabb4584d92aa93e96969908e9befb967"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ef2545823cfff38a61255651373b4705946accac38b5ffe817caaf9147a42ec4"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-win32.whl", hash = "sha256:0d1f4545e0de52774b512d943783529109b088b8716f16c75072a8af7b11ce22"}, + {file = "clickhouse_driver-0.2.5-cp38-cp38-win_amd64.whl", hash = "sha256:420b8be690e678c86ba57e446c98716598dc29a128759cc50bb1a6dc75e6b619"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b75463226330fd5d37bb2590f47cfa43c6694efffe516d598bd663b81a1d09dc"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f40176af385fc8d69e48a1e66d79349b0cdc05aeb83a45bc655d3fa5a7c6c36"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4e2a0e059b5b2886c2ada313799dafddab5052f3f2209f0429cc738c3d7e55f8"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fd78c78de4a74ac9fc3bcce3916b735539d34caaefb3c0fdcb39fea763078367"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70a70e46df56d966aed8423d5870a6b7f6137e20bb8ec2572723efb4bd1c80ca"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78062269ae40b9ba2d2f5bb4318c705d5b9543d268982f4fbf39f80f1325207c"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ba975dbc89444ac69075007a97dfae5c7d51cd807372259e0051c723cc75fd21"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e61f72a6615477f3577714f5ba657be649e9c348a2d20864d16eb41b4446882d"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:0254a800731469c00fa46d8f25cfab565703d07ae34ebe9bceb64ebfb0d8e99b"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:115abd0113e78dcc618b219c5539942792d121070b40f4e4b4bdbc03a23f6630"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c7311c46274f480f2bdc19d42151054eae8e3b9045624d4ca3182afd2fb7c5df"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-win32.whl", hash = "sha256:852b2d57890db87248c60f05809df3ce8a3737fc37b3908b23b6a914212cc2b9"}, + {file = "clickhouse_driver-0.2.5-cp39-cp39-win_amd64.whl", hash = "sha256:55f989eba98bfcf28da1dcbf4b7bb8322cd4a98df7855a68d4090160da2058a1"}, + {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5f8585d6e1023779da471d37a3ea8623017f90115a85fef51d49a0c619010b44"}, + {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:59c07f648edbff11fdb5fc57f5d56f585acc0381bfda2cbbbc2ea04f6adf1767"}, + {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd8e223a434fb7bffdd10e4541a3f756b58d15eac5e6bb701d0ad37719b20298"}, + {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d092c89725c1c616ba9c1624a1ea5abbfc13d671ef2197b6c075b2d051f8bba2"}, + {file = "clickhouse_driver-0.2.5-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f4587b47b8651f7c0576cfb374f318c758c420883fe8fbb97b2a6503ce525150"}, + {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:6083afaa4bb2691e42c588e635cc7bdfae3060d0a627e5a49f6bda2cb7bb3468"}, + {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:585decb371c6b5cd3e9607c703925c49e96e7b872bbd676c71a0acf85397eeef"}, + {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08aa10e63a911513fbe4ea7168a9fe2c55ce47693094e0e49d3db98ab1da9b1d"}, + {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:81c639203ae0c0491af6794dd88a7ee0329298cc76389e5475c89df732a649b4"}, + {file = "clickhouse_driver-0.2.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5afb33e39d2ac0765e6e5014d57056bf1a783b7938a6d231eeeb049fe1967c27"}, + {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:378d4a641842b891387d9a577eccbecdc0ac3dbafc29f7a6fa273cab2e3591f4"}, + {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6321f328d79c1986706db57109ec80ac7c78b3782f261509e5441ba0ff8fce18"}, + {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7ec4a6a8aba19c5b5945ad10ac45224a1203d00939d6382105f69c6367599ed"}, + {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ac0d8c2f1d54fcca45393d6dbe40f0b11500ee07471671345361ca259ffd726"}, + {file = "clickhouse_driver-0.2.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f380c0c7c598c22ed7c9ee0f6ecc133af959ba359999b09ec42dbbc97e7482dc"}, +] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +coverage = [ + {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, + {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, + {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, + {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, + {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, + {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, + {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, + {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, + {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, + {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, + {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, + {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, + {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, + {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, + {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, + {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, +] +cryptography = [ + {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70"}, + {file = "cryptography-38.0.4-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b"}, + {file = "cryptography-38.0.4-cp36-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"}, + {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00"}, + {file = "cryptography-38.0.4-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0"}, + {file = "cryptography-38.0.4-cp36-abi3-win32.whl", hash = "sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744"}, + {file = "cryptography-38.0.4-cp36-abi3-win_amd64.whl", hash = "sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7"}, + {file = "cryptography-38.0.4-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876"}, + {file = "cryptography-38.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee"}, + {file = "cryptography-38.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9"}, + {file = "cryptography-38.0.4.tar.gz", hash = "sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290"}, +] +dsnparse = [ + {file = "dsnparse-0.1.15.tar.gz", hash = "sha256:2ac5705b17cb28e8b115053c2d51cf3321dc2041b1d75e2db6157e05146d0fba"}, +] +duckdb = [ + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e566514f9327f89264e98ac14ee7a84fbd9857328028258422c3e8375ee19d25"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b31c2883de5b19591a2852165e6b3f9821f77af649835f27bc146b26e4aa30cb"}, + {file = "duckdb-0.6.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:998165b2fb1f1d2b0ad742096015ea70878f7d40304643c7424c3ed3ddf07bfc"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3941b3a1e8a1cdb7b90ab3917b87af816e71f9692e5ada7f19b6b60969f731e5"}, + {file = "duckdb-0.6.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:143611bd1b7c13343f087d4d423a7a8a4f33a114c5326171e867febf3f0fcfe1"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:125ba45e8b08f28858f918ec9cbd3a19975e5d8d9e8275ef4ad924028a616e14"}, + {file = "duckdb-0.6.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:e609a65b31c92f2f7166831f74b56f5ed54b33d8c2c4b4c3974c26fdc50464c5"}, + {file = "duckdb-0.6.1-cp310-cp310-win32.whl", hash = "sha256:b39045074fb9a3f068496475a5d627ad4fa572fa3b4980e3b479c11d0b706f2d"}, + {file = "duckdb-0.6.1-cp310-cp310-win_amd64.whl", hash = "sha256:16fa96ffaa3d842a9355a633fb8bc092d119be08d4bc02013946d8594417bc14"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b4bbe2f6c1b109c626f9318eee80934ad2a5b81a51409c6b5083c6c5f9bdb125"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cfea36b58928ce778d17280d4fb3bf0a2d7cff407667baedd69c5b41463ac0fd"}, + {file = "duckdb-0.6.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0b64eb53d0d0695814bf1b65c0f91ab7ed66b515f89c88038f65ad5e0762571c"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35b01bc724e1933293f4c34f410d2833bfbb56d5743b515d805bbfed0651476e"}, + {file = "duckdb-0.6.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fec2c2466654ce786843bda2bfba71e0e4719106b41d36b17ceb1901e130aa71"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:82cd30f5cf368658ef879b1c60276bc8650cf67cfe3dc3e3009438ba39251333"}, + {file = "duckdb-0.6.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a782bbfb7f5e97d4a9c834c9e78f023fb8b3f6687c22ca99841e6ed944b724da"}, + {file = "duckdb-0.6.1-cp311-cp311-win32.whl", hash = "sha256:e3702d4a9ade54c6403f6615a98bbec2020a76a60f5db7fcf085df1bd270e66e"}, + {file = "duckdb-0.6.1-cp311-cp311-win_amd64.whl", hash = "sha256:93b074f473d68c944b0eeb2edcafd91ad11da8432b484836efaaab4e26351d48"}, + {file = "duckdb-0.6.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:adae183924d6d479202c39072e37d440b511326e84525bcb7432bca85f86caba"}, + {file = "duckdb-0.6.1-cp36-cp36m-win32.whl", hash = "sha256:546a1cd17595bd1dd009daf6f36705aa6f95337154360ce44932157d353dcd80"}, + {file = "duckdb-0.6.1-cp36-cp36m-win_amd64.whl", hash = "sha256:87b0d00eb9d1a7ebe437276203e0cdc93b4a2154ba9688c65e8d2a8735839ec6"}, + {file = "duckdb-0.6.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8442e074de6e1969c3d2b24363a5a6d7f866d5ac3f4e358e357495b389eff6c1"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a6bf2ae7bec803352dade14561cb0b461b2422e70f75d9f09b36ba2dad2613b"}, + {file = "duckdb-0.6.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5054792f22733f89d9cbbced2bafd8772d72d0fe77f159310221cefcf981c680"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:21cc503dffc2c68bb825e4eb3098e82f40e910b3d09e1b3b7f090d39ad53fbea"}, + {file = "duckdb-0.6.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:54b3da77ad893e99c073087ff7f75a8c98154ac5139d317149f12b74367211db"}, + {file = "duckdb-0.6.1-cp37-cp37m-win32.whl", hash = "sha256:f1d709aa6a26172a3eab804b57763d5cdc1a4b785ac1fc2b09568578e52032ee"}, + {file = "duckdb-0.6.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f4edcaa471d791393e37f63e3c7c728fa6324e3ac7e768b9dc2ea49065cd37cc"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d218c2dd3bda51fb79e622b7b2266183ac9493834b55010aa01273fa5b7a7105"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c7155cb93ab432eca44b651256c359281d26d927ff43badaf1d2276dd770832"}, + {file = "duckdb-0.6.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0925778200090d3d5d8b6bb42b4d05d24db1e8912484ba3b7e7b7f8569f17dcb"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8b544dd04bb851d08bc68b317a7683cec6091547ae75555d075f8c8a7edb626e"}, + {file = "duckdb-0.6.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2c37d5a0391cf3a3a66e63215968ffb78e6b84f659529fa4bd10478f6203071"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ce376966260eb5c351fcc6af627a979dbbcae3efeb2e70f85b23aa45a21e289d"}, + {file = "duckdb-0.6.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:73c974b09dd08dff5e8bdedba11c7d0aa0fc46ca93954ee7d19e1e18c9883ac1"}, + {file = "duckdb-0.6.1-cp38-cp38-win32.whl", hash = "sha256:bfe39ed3a03e8b1ed764f58f513b37b24afe110d245803a41655d16d391ad9f1"}, + {file = "duckdb-0.6.1-cp38-cp38-win_amd64.whl", hash = "sha256:afa97d982dbe6b125631a17e222142e79bee88f7a13fc4cee92d09285e31ec83"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c35ff4b1117096ef72d101524df0079da36c3735d52fcf1d907ccffa63bd6202"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5c54910fbb6de0f21d562e18a5c91540c19876db61b862fc9ffc8e31be8b3f03"}, + {file = "duckdb-0.6.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99a7172563a3ae67d867572ce27cf3962f58e76f491cb7f602f08c2af39213b3"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7363ffe857d00216b659116647fbf1e925cb3895699015d4a4e50b746de13041"}, + {file = "duckdb-0.6.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06c1cef25f896b2284ba048108f645c72fab5c54aa5a6f62f95663f44ff8a79b"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e92dd6aad7e8c29d002947376b6f5ce28cae29eb3b6b58a64a46cdbfc5cb7943"}, + {file = "duckdb-0.6.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b280b2d8a01ecd4fe2feab041df70233c534fafbe33a38565b52c1e017529c7"}, + {file = "duckdb-0.6.1-cp39-cp39-win32.whl", hash = "sha256:d9212d76e90b8469743924a4d22bef845be310d0d193d54ae17d9ef1f753cfa7"}, + {file = "duckdb-0.6.1-cp39-cp39-win_amd64.whl", hash = "sha256:00b7be8f67ec1a8edaa8844f521267baa1a795f4c482bfad56c72c26e1862ab2"}, + {file = "duckdb-0.6.1.tar.gz", hash = "sha256:6d26e9f1afcb924a6057785e506810d48332d4764ddc4a5b414d0f2bf0cacfb4"}, +] +filelock = [ + {file = "filelock-3.8.2-py3-none-any.whl", hash = "sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"}, + {file = "filelock-3.8.2.tar.gz", hash = "sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +importlib-metadata = [ + {file = "importlib_metadata-5.1.0-py3-none-any.whl", hash = "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313"}, + {file = "importlib_metadata-5.1.0.tar.gz", hash = "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b"}, +] +mysql-connector-python = [ + {file = "mysql-connector-python-8.0.29.tar.gz", hash = "sha256:29ec05ded856b4da4e47239f38489c03b31673ae0f46a090d0e4e29c670e6181"}, + {file = "mysql_connector_python-8.0.29-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:bed43ea3a11f8d4e7c2e3f20c891214e68b45451314f91fddf9ca701de7a53ac"}, + {file = "mysql_connector_python-8.0.29-cp310-cp310-manylinux1_i686.whl", hash = "sha256:6e2267ad75b37b5e1c480cde77cdc4f795427a54266ead30aabcdbf75ac70064"}, + {file = "mysql_connector_python-8.0.29-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:d5afb766b379111942d4260f29499f93355823c7241926471d843c9281fe477c"}, + {file = "mysql_connector_python-8.0.29-cp310-cp310-win_amd64.whl", hash = "sha256:4de5959e27038cbd11dfccb1afaa2fd258c013e59d3e15709dd1992086103050"}, + {file = "mysql_connector_python-8.0.29-cp37-cp37m-macosx_11_0_x86_64.whl", hash = "sha256:895135cde57622edf48e1fce3beb4ed85f18332430d48f5c1d9630d49f7712b0"}, + {file = "mysql_connector_python-8.0.29-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:fdd262d8538aa504475f8860cfda939a297d3b213c8d15f7ceed52508aeb2aa3"}, + {file = "mysql_connector_python-8.0.29-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:89597c091c4f25b6e023cbbcd32be73affbb0b44256761fe3b8e1d4b14d14d02"}, + {file = "mysql_connector_python-8.0.29-cp37-cp37m-win_amd64.whl", hash = "sha256:ab0e9d9b5fc114b78dfa9c74e8bfa30b48fcfa17dbb9241ad6faada08a589900"}, + {file = "mysql_connector_python-8.0.29-cp38-cp38-macosx_11_0_x86_64.whl", hash = "sha256:245087999f081b389d66621f2abfe2463e3927f63c7c4c0f70ce0f82786ccb93"}, + {file = "mysql_connector_python-8.0.29-cp38-cp38-manylinux1_i686.whl", hash = "sha256:5eef51e48b22aadd633563bbdaf02112d98d954a4ead53f72fde283ea3f88152"}, + {file = "mysql_connector_python-8.0.29-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:b7dccd7f72f19c97b58428ebf8e709e24eb7e9b67a408af7e77b60efde44bea4"}, + {file = "mysql_connector_python-8.0.29-cp38-cp38-win_amd64.whl", hash = "sha256:7be3aeff73b85eab3af2a1e80c053a98cbcb99e142192e551ebd4c1e41ce2596"}, + {file = "mysql_connector_python-8.0.29-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a7fd6a71df824f5a7d9a94060598d67b3a32eeccdc9837ee2cd98a44e2536cae"}, + {file = "mysql_connector_python-8.0.29-cp39-cp39-manylinux1_i686.whl", hash = "sha256:fd608c288f596c4c8767d9a8e90f129385bd19ee6e3adaf6974ad8012c6138b8"}, + {file = "mysql_connector_python-8.0.29-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:f353893481476a537cca7afd4e81e0ed84dd2173932b7f1721ab3e3351cbf324"}, + {file = "mysql_connector_python-8.0.29-cp39-cp39-win_amd64.whl", hash = "sha256:1bef2a4a2b529c6e9c46414100ab7032c252244e8a9e017d2b6a41bb9cea9312"}, + {file = "mysql_connector_python-8.0.29-py2.py3-none-any.whl", hash = "sha256:047420715bbb51d3cba78de446c8a6db4666459cd23e168568009c620a3f5b90"}, +] +numpy = [ + {file = "numpy-1.21.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671"}, + {file = "numpy-1.21.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e"}, + {file = "numpy-1.21.1-cp37-cp37m-win32.whl", hash = "sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172"}, + {file = "numpy-1.21.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267"}, + {file = "numpy-1.21.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68"}, + {file = "numpy-1.21.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8"}, + {file = "numpy-1.21.1-cp38-cp38-win32.whl", hash = "sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd"}, + {file = "numpy-1.21.1-cp38-cp38-win_amd64.whl", hash = "sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b"}, + {file = "numpy-1.21.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1"}, + {file = "numpy-1.21.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a"}, + {file = "numpy-1.21.1-cp39-cp39-win32.whl", hash = "sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2"}, + {file = "numpy-1.21.1-cp39-cp39-win_amd64.whl", hash = "sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33"}, + {file = "numpy-1.21.1-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4"}, + {file = "numpy-1.21.1.zip", hash = "sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd"}, +] +oscrypto = [ + {file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"}, + {file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"}, +] +parameterized = [ + {file = "parameterized-0.8.1-py2.py3-none-any.whl", hash = "sha256:9cbb0b69a03e8695d68b3399a8a5825200976536fe1cb79db60ed6a4c8c9efe9"}, + {file = "parameterized-0.8.1.tar.gz", hash = "sha256:41bbff37d6186430f77f900d777e5bb6a24928a1c46fb1de692f8b52b8833b5c"}, +] +presto-python-client = [ + {file = "presto-python-client-0.8.3.tar.gz", hash = "sha256:260249efefbcf56f407f93d39d92fb9963164ca3d7abffa4493fd127346d2732"}, + {file = "presto_python_client-0.8.3-py3-none-any.whl", hash = "sha256:c45fbb8be71cdbca75b37836c47878eb6333733cc65ceff73b7aa116c410b5bc"}, +] +protobuf = [ + {file = "protobuf-4.21.12-cp310-abi3-win32.whl", hash = "sha256:b135410244ebe777db80298297a97fbb4c862c881b4403b71bac9d4107d61fd1"}, + {file = "protobuf-4.21.12-cp310-abi3-win_amd64.whl", hash = "sha256:89f9149e4a0169cddfc44c74f230d7743002e3aa0b9472d8c28f0388102fc4c2"}, + {file = "protobuf-4.21.12-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:299ea899484ee6f44604deb71f424234f654606b983cb496ea2a53e3c63ab791"}, + {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:d1736130bce8cf131ac7957fa26880ca19227d4ad68b4888b3be0dea1f95df97"}, + {file = "protobuf-4.21.12-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:78a28c9fa223998472886c77042e9b9afb6fe4242bd2a2a5aced88e3f4422aa7"}, + {file = "protobuf-4.21.12-cp37-cp37m-win32.whl", hash = "sha256:3d164928ff0727d97022957c2b849250ca0e64777ee31efd7d6de2e07c494717"}, + {file = "protobuf-4.21.12-cp37-cp37m-win_amd64.whl", hash = "sha256:f45460f9ee70a0ec1b6694c6e4e348ad2019275680bd68a1d9314b8c7e01e574"}, + {file = "protobuf-4.21.12-cp38-cp38-win32.whl", hash = "sha256:6ab80df09e3208f742c98443b6166bcb70d65f52cfeb67357d52032ea1ae9bec"}, + {file = "protobuf-4.21.12-cp38-cp38-win_amd64.whl", hash = "sha256:1f22ac0ca65bb70a876060d96d914dae09ac98d114294f77584b0d2644fa9c30"}, + {file = "protobuf-4.21.12-cp39-cp39-win32.whl", hash = "sha256:27f4d15021da6d2b706ddc3860fac0a5ddaba34ab679dc182b60a8bb4e1121cc"}, + {file = "protobuf-4.21.12-cp39-cp39-win_amd64.whl", hash = "sha256:237216c3326d46808a9f7c26fd1bd4b20015fb6867dc5d263a493ef9a539293b"}, + {file = "protobuf-4.21.12-py2.py3-none-any.whl", hash = "sha256:a53fd3f03e578553623272dc46ac2f189de23862e68565e83dde203d41b76fc5"}, + {file = "protobuf-4.21.12-py3-none-any.whl", hash = "sha256:b98d0148f84e3a3c569e19f52103ca1feacdac0d2df8d6533cf983d1fda28462"}, + {file = "protobuf-4.21.12.tar.gz", hash = "sha256:7cd532c4566d0e6feafecc1059d04c7915aec8e182d1cf7adee8b24ef1e2e6ab"}, +] +psycopg2 = [ + {file = "psycopg2-2.9.5-cp310-cp310-win32.whl", hash = "sha256:d3ef67e630b0de0779c42912fe2cbae3805ebaba30cda27fea2a3de650a9414f"}, + {file = "psycopg2-2.9.5-cp310-cp310-win_amd64.whl", hash = "sha256:4cb9936316d88bfab614666eb9e32995e794ed0f8f6b3b718666c22819c1d7ee"}, + {file = "psycopg2-2.9.5-cp36-cp36m-win32.whl", hash = "sha256:b9ac1b0d8ecc49e05e4e182694f418d27f3aedcfca854ebd6c05bb1cffa10d6d"}, + {file = "psycopg2-2.9.5-cp36-cp36m-win_amd64.whl", hash = "sha256:fc04dd5189b90d825509caa510f20d1d504761e78b8dfb95a0ede180f71d50e5"}, + {file = "psycopg2-2.9.5-cp37-cp37m-win32.whl", hash = "sha256:922cc5f0b98a5f2b1ff481f5551b95cd04580fd6f0c72d9b22e6c0145a4840e0"}, + {file = "psycopg2-2.9.5-cp37-cp37m-win_amd64.whl", hash = "sha256:1e5a38aa85bd660c53947bd28aeaafb6a97d70423606f1ccb044a03a1203fe4a"}, + {file = "psycopg2-2.9.5-cp38-cp38-win32.whl", hash = "sha256:f5b6320dbc3cf6cfb9f25308286f9f7ab464e65cfb105b64cc9c52831748ced2"}, + {file = "psycopg2-2.9.5-cp38-cp38-win_amd64.whl", hash = "sha256:1a5c7d7d577e0eabfcf15eb87d1e19314c8c4f0e722a301f98e0e3a65e238b4e"}, + {file = "psycopg2-2.9.5-cp39-cp39-win32.whl", hash = "sha256:322fd5fca0b1113677089d4ebd5222c964b1760e361f151cbb2706c4912112c5"}, + {file = "psycopg2-2.9.5-cp39-cp39-win_amd64.whl", hash = "sha256:190d51e8c1b25a47484e52a79638a8182451d6f6dff99f26ad9bd81e5359a0fa"}, + {file = "psycopg2-2.9.5.tar.gz", hash = "sha256:a5246d2e683a972e2187a8714b5c2cf8156c064629f9a9b1a873c1730d9e245a"}, +] +pycparser = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] +pycryptodomex = [ + {file = "pycryptodomex-3.16.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b3d04c00d777c36972b539fb79958790126847d84ec0129fce1efef250bfe3ce"}, + {file = "pycryptodomex-3.16.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:e5a670919076b71522c7d567a9043f66f14b202414a63c3a078b5831ae342c03"}, + {file = "pycryptodomex-3.16.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:ce338a9703f54b2305a408fc9890eb966b727ce72b69f225898bb4e9d9ed3f1f"}, + {file = "pycryptodomex-3.16.0-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:a1c0ae7123448ecb034c75c713189cb00ebe2d415b11682865b6c54d200d9c93"}, + {file = "pycryptodomex-3.16.0-cp27-cp27m-win32.whl", hash = "sha256:8851585ff19871e5d69e1790f4ca5f6fd1699d6b8b14413b472a4c0dbc7ea780"}, + {file = "pycryptodomex-3.16.0-cp27-cp27m-win_amd64.whl", hash = "sha256:8dd2d9e3c617d0712ed781a77efd84ea579e76c5f9b2a4bc0b684ebeddf868b2"}, + {file = "pycryptodomex-3.16.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:2ad9bb86b355b6104796567dd44c215b3dc953ef2fae5e0bdfb8516731df92cf"}, + {file = "pycryptodomex-3.16.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:e25a2f5667d91795f9417cb856f6df724ccdb0cdd5cbadb212ee9bf43946e9f8"}, + {file = "pycryptodomex-3.16.0-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:b0789a8490114a2936ed77c87792cfe77582c829cb43a6d86ede0f9624ba8aa3"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:0da835af786fdd1c9930994c78b23e88d816dc3f99aa977284a21bbc26d19735"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:22aed0868622d95179217c298e37ed7410025c7b29dac236d3230617d1e4ed56"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1619087fb5b31510b0b0b058a54f001a5ffd91e6ffee220d9913064519c6a69d"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:70288d9bfe16b2fd0d20b6c365db614428f1bcde7b20d56e74cf88ade905d9eb"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7993d26dae4d83b8f4ce605bb0aecb8bee330bb3c95475ef06f3694403621e71"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:1cda60207be8c1cf0b84b9138f9e3ca29335013d2b690774a5e94678ff29659a"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:04610536921c1ec7adba158ef570348550c9f3a40bc24be9f8da2ef7ab387981"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-win32.whl", hash = "sha256:daa67f5ebb6fbf1ee9c90decaa06ca7fc88a548864e5e484d52b0920a57fe8a5"}, + {file = "pycryptodomex-3.16.0-cp35-abi3-win_amd64.whl", hash = "sha256:231dc8008cbdd1ae0e34645d4523da2dbc7a88c325f0d4a59635a86ee25b41dd"}, + {file = "pycryptodomex-3.16.0-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:4dbbe18cc232b5980c7633972ae5417d0df76fe89e7db246eefd17ef4d8e6d7a"}, + {file = "pycryptodomex-3.16.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:893f8a97d533c66cc3a56e60dd3ed40a3494ddb4aafa7e026429a08772f8a849"}, + {file = "pycryptodomex-3.16.0-pp27-pypy_73-win32.whl", hash = "sha256:6a465e4f856d2a4f2a311807030c89166529ccf7ccc65bef398de045d49144b6"}, + {file = "pycryptodomex-3.16.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ba57ac7861fd2c837cdb33daf822f2a052ff57dd769a2107807f52a36d0e8d38"}, + {file = "pycryptodomex-3.16.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f2b971a7b877348a27dcfd0e772a0343fb818df00b74078e91c008632284137d"}, + {file = "pycryptodomex-3.16.0-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e2453162f473c1eae4826eb10cd7bce19b5facac86d17fb5f29a570fde145abd"}, + {file = "pycryptodomex-3.16.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:0ba28aa97cdd3ff5ed1a4f2b7f5cd04e721166bd75bd2b929e2734433882b583"}, + {file = "pycryptodomex-3.16.0.tar.gz", hash = "sha256:e9ba9d8ed638733c9e95664470b71d624a6def149e2db6cc52c1aca5a6a2df1d"}, +] +Pygments = [ + {file = "Pygments-2.13.0-py3-none-any.whl", hash = "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42"}, + {file = "Pygments-2.13.0.tar.gz", hash = "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1"}, +] +PyJWT = [ + {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, + {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, +] +pyOpenSSL = [ + {file = "pyOpenSSL-22.1.0-py3-none-any.whl", hash = "sha256:b28437c9773bb6c6958628cf9c3bebe585de661dba6f63df17111966363dd15e"}, + {file = "pyOpenSSL-22.1.0.tar.gz", hash = "sha256:7a83b7b272dd595222d672f5ce29aa030f1fb837630ef229f62e72e395ce8968"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pytz = [ + {file = "pytz-2022.6-py2.py3-none-any.whl", hash = "sha256:222439474e9c98fced559f1709d89e6c9cbf8d79c794ff3eb9f8800064291427"}, + {file = "pytz-2022.6.tar.gz", hash = "sha256:e89512406b793ca39f5971bc999cc538ce125c0e51c27941bef4568b460095e2"}, +] +pytz-deprecation-shim = [ + {file = "pytz_deprecation_shim-0.1.0.post0-py2.py3-none-any.whl", hash = "sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6"}, + {file = "pytz_deprecation_shim-0.1.0.post0.tar.gz", hash = "sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d"}, +] +requests = [ + {file = "requests-2.28.1-py3-none-any.whl", hash = "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"}, + {file = "requests-2.28.1.tar.gz", hash = "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983"}, +] +rich = [ + {file = "rich-12.6.0-py3-none-any.whl", hash = "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e"}, + {file = "rich-12.6.0.tar.gz", hash = "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0"}, +] +runtype = [ + {file = "runtype-0.2.7-py3-none-any.whl", hash = "sha256:a2bdeb1a1dece5b753ac0cc602d373239166887fe652fd9fec41683dc38bff10"}, + {file = "runtype-0.2.7.tar.gz", hash = "sha256:5a9e1212846b3e54d4ba29fd7db602af5544a2a4253d1f8d829087214a8766ad"}, +] +setuptools = [ + {file = "setuptools-65.6.3-py3-none-any.whl", hash = "sha256:57f6f22bde4e042978bcd50176fdb381d7c21a9efa4041202288d3737a0c6a54"}, + {file = "setuptools-65.6.3.tar.gz", hash = "sha256:a7620757bf984b58deaf32fc8a4577a9bbc0850cf92c20e1ce41c38c19e5fb75"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +snowflake-connector-python = [ + {file = "snowflake-connector-python-2.9.0.tar.gz", hash = "sha256:7551b2404b26850fb12c6232d015ba5d10adb13b091e379fe8b8366492f624b8"}, + {file = "snowflake_connector_python-2.9.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:a66f90e76232db02754c34bdc1a0cda90f698658d38e1411cc6ddbe14e40f2a8"}, + {file = "snowflake_connector_python-2.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:649dc6764c1f2cf2a8aeac3cbb880c141cc38a38b72683325f87dc053539efcb"}, + {file = "snowflake_connector_python-2.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db5152f24a60544f1efee4c4022d7a07428d5bab78858bb824fd966895e4078a"}, + {file = "snowflake_connector_python-2.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ad9af11a59a8bb258f514c951f9d849747c64a860e77f512f5f113ef7067617"}, + {file = "snowflake_connector_python-2.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:e720b0e24cb34caa5ef1ff099e3d41dcc4f95c04ab70fafff390cc8045dfdb56"}, + {file = "snowflake_connector_python-2.9.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:308e7332d9f045b6f2d615a0199839ebddf6edcc614d58939d6c01926dc0fa4e"}, + {file = "snowflake_connector_python-2.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7d341672d8fbf711e8cf96a61d91ab0f065d4c212a3a2384f08dc6de75d211bd"}, + {file = "snowflake_connector_python-2.9.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2796eb05c264fd37ba2d922dcc27f04e2e4f7b938b7cae7f79b2ea7ed7d0cd63"}, + {file = "snowflake_connector_python-2.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:99d93e7ceef0aed159aee7ac36d032e8e8753be61a506f91b89b5e696d25bb62"}, + {file = "snowflake_connector_python-2.9.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:f27ef8988147688d3f9cbe798f72919b454cc295e386c21926391b4d3487d88c"}, + {file = "snowflake_connector_python-2.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:301affd1567dda268a1e57c0241721e4aa3e2bb103b4a3d2bf08e42768cf610a"}, + {file = "snowflake_connector_python-2.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e805a4e7631ef68bc373173fcd74057dfecf1003424766e6ed32948aa18130af"}, + {file = "snowflake_connector_python-2.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:014f2aa6544f1af2a190c161ad11880dc0e23cf0936b43cd90c29d6dc2bca66f"}, + {file = "snowflake_connector_python-2.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9654a7af2c407daf8525416933356eeacb7265a85a42b8e3ade2c1b912fa3295"}, + {file = "snowflake_connector_python-2.9.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:6f46c1fb9f9cc5053203b654595d4395b9a68f2cd8dccdeb805e3a823d0b5e5b"}, + {file = "snowflake_connector_python-2.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3bf763bc4f8fdc215ed2236bc709c19d2019b41b16a5df64d366fd139ca1edf0"}, + {file = "snowflake_connector_python-2.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb10e761554f3e26c586f3c10d6bdc8df502016424d210978dfa175c16ecd70f"}, + {file = "snowflake_connector_python-2.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:23401da4ac80867f1c65c7275938fadd4a796b500748b39b26f7d3e36a10405e"}, + {file = "snowflake_connector_python-2.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:6994dca106e4fb2c26135e7d3d05fdbd927cee4e63f8d3f8167b43d5b3d2c9d3"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +trino = [ + {file = "trino-0.314.0-py3-none-any.whl", hash = "sha256:eda67f29e2dc46da10dfb6315257d33b871dec5e1fa3f60d4ddc3832cd109a53"}, + {file = "trino-0.314.0.tar.gz", hash = "sha256:433174f2f1b2e079cf64c00673e8a5a5b32c012891c0a6cfd44afa293ca74467"}, +] +typing-extensions = [ + {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, + {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, +] +tzdata = [ + {file = "tzdata-2022.7-py2.py3-none-any.whl", hash = "sha256:2b88858b0e3120792a3c0635c23daf36a7d7eeeca657c323da299d2094402a0d"}, + {file = "tzdata-2022.7.tar.gz", hash = "sha256:fe5f866eddd8b96e9fcba978f8e503c909b19ea7efda11e52e39494bad3a7bfa"}, +] +tzlocal = [ + {file = "tzlocal-4.2-py3-none-any.whl", hash = "sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745"}, + {file = "tzlocal-4.2.tar.gz", hash = "sha256:ee5842fa3a795f023514ac2d801c4a81d1743bbe642e3940143326b3a00addd7"}, +] +unittest-parallel = [ + {file = "unittest-parallel-1.5.3.tar.gz", hash = "sha256:32182bb2230371d651e6fc9795ddf52c134eb36f5064dc339fdbb5984a639517"}, + {file = "unittest_parallel-1.5.3-py3-none-any.whl", hash = "sha256:5670c9eca19450dedb493e9dad2ca4dcbbe12e04477d934ff6c92071d36bace7"}, +] +urllib3 = [ + {file = "urllib3-1.26.13-py2.py3-none-any.whl", hash = "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc"}, + {file = "urllib3-1.26.13.tar.gz", hash = "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8"}, +] +vertica-python = [ + {file = "vertica-python-1.1.1.tar.gz", hash = "sha256:dedf56d76b67673b4d57a13f7f96ebdc57b39ea650b93ebf0c05eb6d1d2c0c05"}, + {file = "vertica_python-1.1.1-py2.py3-none-any.whl", hash = "sha256:63d300832d6fe471987880f06a9590eafc46a1f896860881270f6b6645f3bec6"}, +] +zipp = [ + {file = "zipp-3.11.0-py3-none-any.whl", hash = "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa"}, + {file = "zipp-3.11.0.tar.gz", hash = "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766"}, +] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..16bb40c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,72 @@ +[tool.poetry] +name = "sqeleton" +version = "0.0.1" +description = "Python library for querying SQL databases" +authors = ["Erez Shinan "] +license = "MIT" +readme = "README.md" +repository = "https://github.com/datafold/sqeleton" +documentation = "" +classifiers = [ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: System Administrators", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Development Status :: 2 - Pre-Alpha", + "Environment :: Console", + "Topic :: Database :: Database Engines/Servers", + "Typing :: Typed" +] +packages = [{ include = "sqeleton" }] + +[tool.poetry.dependencies] +python = "^3.7" +runtype = "^0.2.6" +dsnparse = "*" +click = "^8.1" +rich = "*" +toml = "^0.10.2" +mysql-connector-python = {version="8.0.29", optional=true} +psycopg2 = {version="*", optional=true} +snowflake-connector-python = {version="^2.7.2", optional=true} +cryptography = {version="*", optional=true} +trino = {version="^0.314.0", optional=true} +presto-python-client = {version="*", optional=true} +clickhouse-driver = {version="*", optional=true} +duckdb = {version="^0.6.0", optional=true} + +[tool.poetry.dev-dependencies] +parameterized = "*" +unittest-parallel = "*" + +duckdb = "^0.6.0" +mysql-connector-python = "*" +psycopg2 = "*" +snowflake-connector-python = "^2.7.2" +cryptography = "*" +trino = "^0.314.0" +presto-python-client = "*" +clickhouse-driver = "*" +vertica-python = "*" + +[tool.poetry.extras] +mysql = ["mysql-connector-python"] +postgresql = ["psycopg2"] +snowflake = ["snowflake-connector-python", "cryptography"] +presto = ["presto-python-client"] +oracle = ["cx_Oracle"] +# databricks = ["databricks-sql-connector"] +trino = ["trino"] +clickhouse = ["clickhouse-driver"] +vertica = ["vertica-python"] +duckdb = ["duckdb"] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.scripts] +sqeleton = 'sqeleton.__main__:main' diff --git a/sqeleton/__init__.py b/sqeleton/__init__.py new file mode 100644 index 0000000..8bb6cbb --- /dev/null +++ b/sqeleton/__init__.py @@ -0,0 +1 @@ +from .databases import connect \ No newline at end of file diff --git a/sqeleton/abcs/__init__.py b/sqeleton/abcs/__init__.py new file mode 100644 index 0000000..7a61893 --- /dev/null +++ b/sqeleton/abcs/__init__.py @@ -0,0 +1,13 @@ +from .database_types import ( + AbstractDatabase, + AbstractDialect, + DbKey, + DbPath, + DbTime, + IKey, + ColType_UUID, + NumericType, + PrecisionType, + StringType, +) +from .compiler import AbstractCompiler, Compilable diff --git a/sqeleton/abcs/compiler.py b/sqeleton/abcs/compiler.py new file mode 100644 index 0000000..9c734e3 --- /dev/null +++ b/sqeleton/abcs/compiler.py @@ -0,0 +1,14 @@ +from typing import Any, Dict +from abc import ABC, abstractmethod + + +class AbstractCompiler(ABC): + @abstractmethod + def compile(self, elem: Any, params: Dict[str, Any] = None) -> str: + ... + + +class Compilable(ABC): + @abstractmethod + def compile(self, c: AbstractCompiler) -> str: + ... diff --git a/sqeleton/abcs/database_types.py b/sqeleton/abcs/database_types.py new file mode 100644 index 0000000..e32c731 --- /dev/null +++ b/sqeleton/abcs/database_types.py @@ -0,0 +1,277 @@ +import decimal +from abc import ABC, abstractmethod +from typing import Sequence, Optional, Tuple, Union, Dict, List +from datetime import datetime + +from runtype import dataclass + +from ..utils import ArithAlphanumeric, ArithUUID + + +DbPath = Tuple[str, ...] +DbKey = Union[int, str, bytes, ArithUUID, ArithAlphanumeric] +DbTime = datetime + + +class ColType: + supported = True + + +@dataclass +class PrecisionType(ColType): + precision: int + rounds: bool + + +class Boolean(ColType): + supported = True + + +class TemporalType(PrecisionType): + pass + + +class Timestamp(TemporalType): + pass + + +class TimestampTZ(TemporalType): + pass + + +class Datetime(TemporalType): + pass + + +@dataclass +class NumericType(ColType): + # 'precision' signifies how many fractional digits (after the dot) we want to compare + precision: int + + +class FractionalType(NumericType): + pass + + +class Float(FractionalType): + pass + + +class IKey(ABC): + "Interface for ColType, for using a column as a key in table." + + @property + @abstractmethod + def python_type(self) -> type: + "Return the equivalent Python type of the key" + + def make_value(self, value): + return self.python_type(value) + + +class Decimal(FractionalType, IKey): # Snowflake may use Decimal as a key + @property + def python_type(self) -> type: + if self.precision == 0: + return int + return decimal.Decimal + + +@dataclass +class StringType(ColType): + pass + + +class ColType_UUID(ColType, IKey): + python_type = ArithUUID + + +class ColType_Alphanum(ColType, IKey): + python_type = ArithAlphanumeric + + +class Native_UUID(ColType_UUID): + pass + + +class String_UUID(StringType, ColType_UUID): + pass + + +class String_Alphanum(StringType, ColType_Alphanum): + @staticmethod + def test_value(value: str) -> bool: + try: + ArithAlphanumeric(value) + return True + except ValueError: + return False + + def make_value(self, value): + return self.python_type(value) + + +class String_VaryingAlphanum(String_Alphanum): + pass + + +@dataclass +class String_FixedAlphanum(String_Alphanum): + length: int + + def make_value(self, value): + if len(value) != self.length: + raise ValueError(f"Expected alphanumeric value of length {self.length}, but got '{value}'.") + return self.python_type(value, max_len=self.length) + + +@dataclass +class Text(StringType): + supported = False + + +@dataclass +class Integer(NumericType, IKey): + precision: int = 0 + python_type: type = int + + def __post_init__(self): + assert self.precision == 0 + + +@dataclass +class UnknownColType(ColType): + text: str + + supported = False + + +class AbstractDialect(ABC): + """Dialect-dependent query expressions""" + + @property + @abstractmethod + def name(self) -> str: + "Name of the dialect" + + @property + @abstractmethod + def ROUNDS_ON_PREC_LOSS(self) -> bool: + "True if db rounds real values when losing precision, False if it truncates." + + @abstractmethod + def quote(self, s: str): + "Quote SQL name" + + @abstractmethod + def concat(self, items: List[str]) -> str: + "Provide SQL for concatenating a bunch of columns into a string" + + @abstractmethod + def is_distinct_from(self, a: str, b: str) -> str: + "Provide SQL for a comparison where NULL = NULL is true" + + @abstractmethod + def to_string(self, s: str) -> str: + # TODO rewrite using cast_to(x, str) + "Provide SQL for casting a column to string" + + @abstractmethod + def random(self) -> str: + "Provide SQL for generating a random number betweein 0..1" + + @abstractmethod + def current_timestamp(self) -> str: + "Provide SQL for returning the current timestamp, aka now" + + @abstractmethod + def offset_limit(self, offset: Optional[int] = None, limit: Optional[int] = None): + "Provide SQL fragment for limit and offset inside a select" + + @abstractmethod + def explain_as_text(self, query: str) -> str: + "Provide SQL for explaining a query, returned as table(varchar)" + + @abstractmethod + def timestamp_value(self, t: datetime) -> str: + "Provide SQL for the given timestamp value" + + @abstractmethod + def set_timezone_to_utc(self) -> str: + "Provide SQL for setting the session timezone to UTC" + + @abstractmethod + def parse_type( + self, + table_path: DbPath, + col_name: str, + type_repr: str, + datetime_precision: int = None, + numeric_precision: int = None, + numeric_scale: int = None, + ) -> ColType: + "Parse type info as returned by the database" + + +class AbstractDatabase: + @property + @abstractmethod + def dialect(self) -> AbstractDialect: + "The dialect of the database. Used internally by Database, and also available publicly." + + @property + @abstractmethod + def CONNECT_URI_HELP(self) -> str: + "Example URI to show the user in help and error messages" + + @property + @abstractmethod + def CONNECT_URI_PARAMS(self) -> List[str]: + "List of parameters given in the path of the URI" + + @abstractmethod + def _query(self, sql_code: str) -> list: + "Send query to database and return result" + + @abstractmethod + def query_table_schema(self, path: DbPath) -> Dict[str, tuple]: + """Query the table for its schema for table in 'path', and return {column: tuple} + where the tuple is (table_name, col_name, type_repr, datetime_precision?, numeric_precision?, numeric_scale?) + + Note: This method exists instead of select_table_schema(), just because not all databases support + accessing the schema using a SQL query. + """ + + @abstractmethod + def select_table_unique_columns(self, path: DbPath) -> str: + "Provide SQL for selecting the names of unique columns in the table" + + @abstractmethod + def query_table_unique_columns(self, path: DbPath) -> List[str]: + """Query the table for its unique columns for table in 'path', and return {column}""" + + @abstractmethod + def _process_table_schema( + self, path: DbPath, raw_schema: Dict[str, tuple], filter_columns: Sequence[str], where: str = None + ): + """Process the result of query_table_schema(). + + Done in a separate step, to minimize the amount of processed columns. + Needed because processing each column may: + * throw errors and warnings + * query the database to sample values + + """ + + @abstractmethod + def parse_table_name(self, name: str) -> DbPath: + "Parse the given table name into a DbPath" + + @abstractmethod + def close(self): + "Close connection(s) to the database instance. Querying will stop functioning." + + @property + @abstractmethod + def is_autocommit(self) -> bool: + "Return whether the database autocommits changes. When false, COMMIT statements are skipped." diff --git a/sqeleton/abcs/mixins.py b/sqeleton/abcs/mixins.py new file mode 100644 index 0000000..5d75fe7 --- /dev/null +++ b/sqeleton/abcs/mixins.py @@ -0,0 +1,98 @@ +from abc import ABC, abstractmethod +from .database_types import TemporalType, FractionalType, ColType_UUID, Boolean, ColType, String_UUID +from .compiler import Compilable + + +class AbstractMixin_NormalizeValue(ABC): + @abstractmethod + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + """Creates an SQL expression, that converts 'value' to a normalized timestamp. + + The returned expression must accept any SQL datetime/timestamp, and return a string. + + Date format: ``YYYY-MM-DD HH:mm:SS.FFFFFF`` + + Precision of dates should be rounded up/down according to coltype.rounds + """ + + @abstractmethod + def normalize_number(self, value: str, coltype: FractionalType) -> str: + """Creates an SQL expression, that converts 'value' to a normalized number. + + The returned expression must accept any SQL int/numeric/float, and return a string. + + Floats/Decimals are expected in the format + "I.P" + + Where I is the integer part of the number (as many digits as necessary), + and must be at least one digit (0). + P is the fractional digits, the amount of which is specified with + coltype.precision. Trailing zeroes may be necessary. + If P is 0, the dot is omitted. + + Note: We use 'precision' differently than most databases. For decimals, + it's the same as ``numeric_scale``, and for floats, who use binary precision, + it can be calculated as ``log10(2**numeric_precision)``. + """ + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + """Creates an SQL expression, that converts 'value' to either '0' or '1'.""" + return self.to_string(value) + + def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str: + """Creates an SQL expression, that strips uuids of artifacts like whitespace.""" + if isinstance(coltype, String_UUID): + return f"TRIM({value})" + return self.to_string(value) + + def normalize_value_by_type(self, value: str, coltype: ColType) -> str: + """Creates an SQL expression, that converts 'value' to a normalized representation. + + The returned expression must accept any SQL value, and return a string. + + The default implementation dispatches to a method according to `coltype`: + + :: + + TemporalType -> normalize_timestamp() + FractionalType -> normalize_number() + *else* -> to_string() + + (`Integer` falls in the *else* category) + + """ + if isinstance(coltype, TemporalType): + return self.normalize_timestamp(value, coltype) + elif isinstance(coltype, FractionalType): + return self.normalize_number(value, coltype) + elif isinstance(coltype, ColType_UUID): + return self.normalize_uuid(value, coltype) + elif isinstance(coltype, Boolean): + return self.normalize_boolean(value, coltype) + return self.to_string(value) + + +class AbstractMixin_MD5(ABC): + """Methods for calculating an MD6 hash as an integer.""" + + @abstractmethod + def md5_as_int(self, s: str) -> str: + "Provide SQL for computing md5 and returning an int" + + +class AbstractMixin_Schema(ABC): + """Methods for querying the database schema + + TODO: Move AbstractDatabase.query_table_schema() and friends over here + """ + + def table_information(self) -> Compilable: + "Query to return a table of schema information about existing tables" + raise NotImplementedError() + + @abstractmethod + def list_tables(self, table_schema: str, like: Compilable = None) -> Compilable: + """Query to select the list of tables in the schema. (query return type: table[str]) + + If 'like' is specified, the value is applied to the table name, using the 'like' operator. + """ diff --git a/sqeleton/databases/__init__.py b/sqeleton/databases/__init__.py new file mode 100644 index 0000000..35ec046 --- /dev/null +++ b/sqeleton/databases/__init__.py @@ -0,0 +1,18 @@ +from .base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, QueryError, ConnectError, BaseDialect, Database +from ..abcs import DbPath, DbKey, DbTime +from ._connect import Connect + +from .postgresql import PostgreSQL +from .mysql import MySQL +from .oracle import Oracle +from .snowflake import Snowflake +from .bigquery import BigQuery +from .redshift import Redshift +from .presto import Presto +from .databricks import Databricks +from .trino import Trino +from .clickhouse import Clickhouse +from .vertica import Vertica +from .duckdb import DuckDB + +connect = Connect() \ No newline at end of file diff --git a/sqeleton/databases/_connect.py b/sqeleton/databases/_connect.py new file mode 100644 index 0000000..cb6c906 --- /dev/null +++ b/sqeleton/databases/_connect.py @@ -0,0 +1,247 @@ +from typing import Type, Optional, Union, Dict +from itertools import zip_longest +from contextlib import suppress +import dsnparse + +from runtype import dataclass + +from ..utils import WeakCache +from .base import Database, ThreadedDatabase +from .postgresql import PostgreSQL +from .mysql import MySQL +from .oracle import Oracle +from .snowflake import Snowflake +from .bigquery import BigQuery +from .redshift import Redshift +from .presto import Presto +from .databricks import Databricks +from .trino import Trino +from .clickhouse import Clickhouse +from .vertica import Vertica +from .duckdb import DuckDB + + +@dataclass +class MatchUriPath: + database_cls: Type[Database] + + def match_path(self, dsn): + help_str = self.database_cls.CONNECT_URI_HELP + params = self.database_cls.CONNECT_URI_PARAMS + kwparams = self.database_cls.CONNECT_URI_KWPARAMS + + dsn_dict = dict(dsn.query) + matches = {} + for param, arg in zip_longest(params, dsn.paths): + if param is None: + raise ValueError(f"Too many parts to path. Expected format: {help_str}") + + optional = param.endswith("?") + param = param.rstrip("?") + + if arg is None: + try: + arg = dsn_dict.pop(param) + except KeyError: + if not optional: + raise ValueError(f"URI must specify '{param}'. Expected format: {help_str}") + + arg = None + + assert param and param not in matches + matches[param] = arg + + for param in kwparams: + try: + arg = dsn_dict.pop(param) + except KeyError: + raise ValueError(f"URI must specify '{param}'. Expected format: {help_str}") + + assert param and arg and param not in matches, (param, arg, matches.keys()) + matches[param] = arg + + for param, value in dsn_dict.items(): + if param in matches: + raise ValueError( + f"Parameter '{param}' already provided as positional argument. Expected format: {help_str}" + ) + + matches[param] = value + + return matches + + +DATABASE_BY_SCHEME = { + "postgresql": PostgreSQL, + "mysql": MySQL, + "oracle": Oracle, + "redshift": Redshift, + "snowflake": Snowflake, + "presto": Presto, + "bigquery": BigQuery, + "databricks": Databricks, + "duckdb": DuckDB, + "trino": Trino, + "clickhouse": Clickhouse, + "vertica": Vertica, +} + + +class Connect: + """Provides methods for connecting to a supported database using a URL or connection dict.""" + + def __init__(self, database_by_scheme: Dict[str, Database] = DATABASE_BY_SCHEME): + self.database_by_scheme = database_by_scheme + self.match_uri_path = {name: MatchUriPath(cls) for name, cls in database_by_scheme.items()} + self.conn_cache = WeakCache() + + def connect_to_uri(self, db_uri: str, thread_count: Optional[int] = 1) -> Database: + """Connect to the given database uri + + thread_count determines the max number of worker threads per database, + if relevant. None means no limit. + + Parameters: + db_uri (str): The URI for the database to connect + thread_count (int, optional): Size of the threadpool. Ignored by cloud databases. (default: 1) + + Note: For non-cloud databases, a low thread-pool size may be a performance bottleneck. + + Supported schemes: + - postgresql + - mysql + - oracle + - snowflake + - bigquery + - redshift + - presto + - databricks + - trino + - clickhouse + - vertica + - duckdb + """ + + dsn = dsnparse.parse(db_uri) + if len(dsn.schemes) > 1: + raise NotImplementedError("No support for multiple schemes") + (scheme,) = dsn.schemes + + try: + matcher = self.match_uri_path[scheme] + except KeyError: + raise NotImplementedError(f"Scheme {scheme} currently not supported") + + cls = matcher.database_cls + + if scheme == "databricks": + assert not dsn.user + kw = {} + kw["access_token"] = dsn.password + kw["http_path"] = dsn.path + kw["server_hostname"] = dsn.host + kw.update(dsn.query) + elif scheme == "duckdb": + kw = {} + kw["filepath"] = dsn.dbname + kw["dbname"] = dsn.user + else: + kw = matcher.match_path(dsn) + + if scheme == "bigquery": + kw["project"] = dsn.host + return cls(**kw) + + if scheme == "snowflake": + kw["account"] = dsn.host + assert not dsn.port + kw["user"] = dsn.user + kw["password"] = dsn.password + else: + kw["host"] = dsn.host + kw["port"] = dsn.port + kw["user"] = dsn.user + if dsn.password: + kw["password"] = dsn.password + + kw = {k: v for k, v in kw.items() if v is not None} + + if issubclass(cls, ThreadedDatabase): + db = cls(thread_count=thread_count, **kw) + else: + db = cls(**kw) + + return self._connection_created(db) + + def connect_with_dict(self, d, thread_count): + d = dict(d) + driver = d.pop("driver") + try: + matcher = self.match_uri_path[driver] + except KeyError: + raise NotImplementedError(f"Driver {driver} currently not supported") + + cls = matcher.database_cls + if issubclass(cls, ThreadedDatabase): + db = cls(thread_count=thread_count, **d) + else: + db = cls(**d) + + return self._connection_created(db) + + def _connection_created(self, db): + "Nop function to be overridden by subclasses." + return db + + def __call__(self, db_conf: Union[str, dict], thread_count: Optional[int] = 1, shared: bool = True) -> Database: + """Connect to a database using the given database configuration. + + Configuration can be given either as a URI string, or as a dict of {option: value}. + + The dictionary configuration uses the same keys as the TOML 'database' definition given with --conf. + + thread_count determines the max number of worker threads per database, + if relevant. None means no limit. + + Parameters: + db_conf (str | dict): The configuration for the database to connect. URI or dict. + thread_count (int, optional): Size of the threadpool. Ignored by cloud databases. (default: 1) + shared (bool): Whether to cache and return the same connection for the same db_conf. (default: True) + + Note: For non-cloud databases, a low thread-pool size may be a performance bottleneck. + + Supported drivers: + - postgresql + - mysql + - oracle + - snowflake + - bigquery + - redshift + - presto + - databricks + - trino + - clickhouse + - vertica + + Example: + >>> connect("mysql://localhost/db") + # TODO + >>> connect({"driver": "mysql", "host": "localhost", "database": "db"}) + # TODO + """ + if shared: + with suppress(KeyError): + conn = self.conn_cache.get(db_conf) + if not conn.is_closed: + return conn + + if isinstance(db_conf, str): + conn = self.connect_to_uri(db_conf, thread_count) + elif isinstance(db_conf, dict): + conn = self.connect_with_dict(db_conf, thread_count) + else: + raise TypeError(f"db configuration must be a URI string or a dictionary. Instead got '{db_conf}'.") + + if shared: + self.conn_cache.add(db_conf, conn) + return conn diff --git a/sqeleton/databases/base.py b/sqeleton/databases/base.py new file mode 100644 index 0000000..ed23ba7 --- /dev/null +++ b/sqeleton/databases/base.py @@ -0,0 +1,508 @@ +from datetime import datetime +import math +import sys +import logging +from typing import Any, Callable, Dict, Generator, Tuple, Optional, Sequence, Type, List, Union +from functools import partial, wraps +from concurrent.futures import ThreadPoolExecutor +import threading +from abc import abstractmethod +from uuid import UUID +import decimal + +from ..utils import is_uuid, safezip +from ..queries import Expr, Compiler, table, Select, SKIP, Explain, Code, this +from ..abcs.database_types import ( + AbstractDatabase, + AbstractDialect, + ColType, + Integer, + Decimal, + Float, + Native_UUID, + String_UUID, + String_Alphanum, + String_VaryingAlphanum, + TemporalType, + UnknownColType, + Text, + DbTime, + DbPath, + Boolean, +) +from ..abcs.mixins import Compilable +from ..abcs.mixins import AbstractMixin_Schema + +logger = logging.getLogger("database") + + +def parse_table_name(t): + return tuple(t.split(".")) + + +def import_helper(package: str = None, text=""): + def dec(f): + @wraps(f) + def _inner(): + try: + return f() + except ModuleNotFoundError as e: + s = text + if package: + s += f"You can install it using 'pip install sqeleton[{package}]'." + raise ModuleNotFoundError(f"{e}\n\n{s}\n") + + return _inner + + return dec + + +class ConnectError(Exception): + pass + + +class QueryError(Exception): + pass + + +def _one(seq): + (x,) = seq + return x + + +class ThreadLocalInterpreter: + """An interpeter used to execute a sequence of queries within the same thread and cursor. + + Useful for cursor-sensitive operations, such as creating a temporary table. + """ + + def __init__(self, compiler: Compiler, gen: Generator): + self.gen = gen + self.compiler = compiler + + def apply_queries(self, callback: Callable[[str], Any]): + q: Expr = next(self.gen) + while True: + sql = self.compiler.compile(q) + logger.debug("Running SQL (%s-TL): %s", self.compiler.database.name, sql) + try: + try: + res = callback(sql) if sql is not SKIP else SKIP + except Exception as e: + q = self.gen.throw(type(e), e) + else: + q = self.gen.send(res) + except StopIteration: + break + + +def apply_query(callback: Callable[[str], Any], sql_code: Union[str, ThreadLocalInterpreter]) -> list: + if isinstance(sql_code, ThreadLocalInterpreter): + return sql_code.apply_queries(callback) + else: + return callback(sql_code) + + +class Mixin_Schema(AbstractMixin_Schema): + def table_information(self) -> Compilable: + return table("information_schema", "tables") + + def list_tables(self, table_schema: str, like: Compilable = None) -> Compilable: + return ( + self.table_information() + .where( + this.table_schema == table_schema, + this.table_name.like(like) if like is not None else SKIP, + this.table_type == "BASE TABLE", + ) + .select(this.table_name) + ) + + +class BaseDialect(AbstractDialect): + SUPPORTS_PRIMARY_KEY = False + SUPPORTS_INDEXES = False + TYPE_CLASSES: Dict[str, type] = {} + + PLACEHOLDER_TABLE = None # Used for Oracle + + def offset_limit(self, offset: Optional[int] = None, limit: Optional[int] = None): + if offset: + raise NotImplementedError("No support for OFFSET in query") + + return f"LIMIT {limit}" + + def concat(self, items: List[str]) -> str: + assert len(items) > 1 + joined_exprs = ", ".join(items) + return f"concat({joined_exprs})" + + def is_distinct_from(self, a: str, b: str) -> str: + return f"{a} is distinct from {b}" + + def timestamp_value(self, t: DbTime) -> str: + return f"'{t.isoformat()}'" + + def random(self) -> str: + return "random()" + + def current_timestamp(self) -> str: + return "current_timestamp()" + + def explain_as_text(self, query: str) -> str: + return f"EXPLAIN {query}" + + def _constant_value(self, v): + if v is None: + return "NULL" + elif isinstance(v, str): + return f"'{v}'" + elif isinstance(v, datetime): + return self.timestamp_value(v) + elif isinstance(v, UUID): + return f"'{v}'" + elif isinstance(v, decimal.Decimal): + return str(v) + elif isinstance(v, bytearray): + return f"'{v.decode()}'" + elif isinstance(v, Code): + return v.code + return repr(v) + + def constant_values(self, rows) -> str: + values = ", ".join("(%s)" % ", ".join(self._constant_value(v) for v in row) for row in rows) + return f"VALUES {values}" + + def type_repr(self, t) -> str: + if isinstance(t, str): + return t + return { + int: "INT", + str: "VARCHAR", + bool: "BOOLEAN", + float: "FLOAT", + datetime: "TIMESTAMP", + }[t] + + def _parse_type_repr(self, type_repr: str) -> Optional[Type[ColType]]: + return self.TYPE_CLASSES.get(type_repr) + + def parse_type( + self, + table_path: DbPath, + col_name: str, + type_repr: str, + datetime_precision: int = None, + numeric_precision: int = None, + numeric_scale: int = None, + ) -> ColType: + """ """ + + cls = self._parse_type_repr(type_repr) + if not cls: + return UnknownColType(type_repr) + + if issubclass(cls, TemporalType): + return cls( + precision=datetime_precision if datetime_precision is not None else DEFAULT_DATETIME_PRECISION, + rounds=self.ROUNDS_ON_PREC_LOSS, + ) + + elif issubclass(cls, Integer): + return cls() + + elif issubclass(cls, Boolean): + return cls() + + elif issubclass(cls, Decimal): + if numeric_scale is None: + numeric_scale = 0 # Needed for Oracle. + return cls(precision=numeric_scale) + + elif issubclass(cls, Float): + # assert numeric_scale is None + return cls( + precision=self._convert_db_precision_to_digits( + numeric_precision if numeric_precision is not None else DEFAULT_NUMERIC_PRECISION + ) + ) + + elif issubclass(cls, (Text, Native_UUID)): + return cls() + + raise TypeError(f"Parsing {type_repr} returned an unknown type '{cls}'.") + + def _convert_db_precision_to_digits(self, p: int) -> int: + """Convert from binary precision, used by floats, to decimal precision.""" + # See: https://en.wikipedia.org/wiki/Single-precision_floating-point_format + return math.floor(math.log(2**p, 10)) + + +class Database(AbstractDatabase): + """Base abstract class for databases. + + Used for providing connection code and implementation specific SQL utilities. + + Instanciated using :meth:`~sqeleton.connect` + """ + + default_schema: str = None + SUPPORTS_ALPHANUMS = True + SUPPORTS_UNIQUE_CONSTAINT = False + + CONNECT_URI_KWPARAMS = [] + + _interactive = False + is_closed = False + + @property + def name(self): + return type(self).__name__ + + def query(self, sql_ast: Union[Expr, Generator], res_type: type = list): + """Query the given SQL code/AST, and attempt to convert the result to type 'res_type' + + If given a generator, it will execute all the yielded sql queries with the same thread and cursor. + The results of the queries a returned by the `yield` stmt (using the .send() mechanism). + It's a cleaner approach than exposing cursors, but may not be enough in all cases. + """ + + compiler = Compiler(self) + if isinstance(sql_ast, Generator): + sql_code = ThreadLocalInterpreter(compiler, sql_ast) + elif isinstance(sql_ast, list): + for i in sql_ast[:-1]: + self.query(i) + return self.query(sql_ast[-1], res_type) + else: + if isinstance(sql_ast, str): + sql_code = sql_ast + else: + sql_code = compiler.compile(sql_ast) + if sql_code is SKIP: + return SKIP + + logger.debug("Running SQL (%s): %s", self.name, sql_code) + + if self._interactive and isinstance(sql_ast, Select): + explained_sql = compiler.compile(Explain(sql_ast)) + explain = self._query(explained_sql) + for row in explain: + # Most returned a 1-tuple. Presto returns a string + if isinstance(row, tuple): + (row,) = row + logger.debug("EXPLAIN: %s", row) + answer = input("Continue? [y/n] ") + if answer.lower() not in ["y", "yes"]: + sys.exit(1) + + res = self._query(sql_code) + if res_type is int: + if not res: + raise ValueError("Query returned 0 rows, expected 1") + row = _one(res) + if not row: + raise ValueError("Row is empty, expected 1 column") + res = _one(row) + if res is None: # May happen due to sum() of 0 items + return None + return int(res) + elif res_type is datetime: + res = _one(_one(res)) + if isinstance(res, str): + res = datetime.fromisoformat(res[:23]) # TODO use a better parsing method + return res + elif res_type is tuple: + assert len(res) == 1, (sql_code, res) + return res[0] + elif getattr(res_type, "__origin__", None) is list and len(res_type.__args__) == 1: + if res_type.__args__ in ((int,), (str,)): + return [_one(row) for row in res] + elif res_type.__args__ in [(Tuple,), (tuple,)]: + return [tuple(row) for row in res] + else: + raise ValueError(res_type) + return res + + def enable_interactive(self): + self._interactive = True + + def select_table_schema(self, path: DbPath) -> str: + """Provide SQL for selecting the table schema as (name, type, date_prec, num_prec)""" + schema, name = self._normalize_table_path(path) + + return ( + "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale " + "FROM information_schema.columns " + f"WHERE table_name = '{name}' AND table_schema = '{schema}'" + ) + + def query_table_schema(self, path: DbPath) -> Dict[str, tuple]: + rows = self.query(self.select_table_schema(path), list) + if not rows: + raise RuntimeError(f"{self.name}: Table '{'.'.join(path)}' does not exist, or has no columns") + + d = {r[0]: r for r in rows} + assert len(d) == len(rows) + return d + + def select_table_unique_columns(self, path: DbPath) -> str: + schema, name = self._normalize_table_path(path) + + return ( + "SELECT column_name " + "FROM information_schema.key_column_usage " + f"WHERE table_name = '{name}' AND table_schema = '{schema}'" + ) + + def query_table_unique_columns(self, path: DbPath) -> List[str]: + if not self.SUPPORTS_UNIQUE_CONSTAINT: + raise NotImplementedError("This database doesn't support 'unique' constraints") + res = self.query(self.select_table_unique_columns(path), List[str]) + return list(res) + + def _process_table_schema( + self, path: DbPath, raw_schema: Dict[str, tuple], filter_columns: Sequence[str], where: str = None + ): + accept = {i.lower() for i in filter_columns} + + col_dict = { + row[0]: self.dialect.parse_type(path, *row) for name, row in raw_schema.items() if name.lower() in accept + } + + self._refine_coltypes(path, col_dict, where) + + # Return a dict of form {name: type} after normalization + return col_dict + + def _refine_coltypes(self, table_path: DbPath, col_dict: Dict[str, ColType], where: str = None, sample_size=64): + """Refine the types in the column dict, by querying the database for a sample of their values + + 'where' restricts the rows to be sampled. + """ + + text_columns = [k for k, v in col_dict.items() if isinstance(v, Text)] + if not text_columns: + return + + fields = [Code(self.dialect.normalize_uuid(self.dialect.quote(c), String_UUID())) for c in text_columns] + samples_by_row = self.query( + table(*table_path).select(*fields).where(Code(where) if where else SKIP).limit(sample_size), list + ) + if not samples_by_row: + raise ValueError(f"Table {table_path} is empty.") + + samples_by_col = list(zip(*samples_by_row)) + + for col_name, samples in safezip(text_columns, samples_by_col): + uuid_samples = [s for s in samples if s and is_uuid(s)] + + if uuid_samples: + if len(uuid_samples) != len(samples): + logger.warning( + f"Mixed UUID/Non-UUID values detected in column {'.'.join(table_path)}.{col_name}, disabling UUID support." + ) + else: + assert col_name in col_dict + col_dict[col_name] = String_UUID() + continue + + if self.SUPPORTS_ALPHANUMS: # Anything but MySQL (so far) + alphanum_samples = [s for s in samples if String_Alphanum.test_value(s)] + if alphanum_samples: + if len(alphanum_samples) != len(samples): + logger.warning( + f"Mixed Alphanum/Non-Alphanum values detected in column {'.'.join(table_path)}.{col_name}. It cannot be used as a key." + ) + else: + assert col_name in col_dict + col_dict[col_name] = String_VaryingAlphanum() + + # @lru_cache() + # def get_table_schema(self, path: DbPath) -> Dict[str, ColType]: + # return self.query_table_schema(path) + + def _normalize_table_path(self, path: DbPath) -> DbPath: + if len(path) == 1: + return self.default_schema, path[0] + elif len(path) == 2: + return path + + raise ValueError(f"{self.name}: Bad table path for {self}: '{'.'.join(path)}'. Expected form: schema.table") + + def parse_table_name(self, name: str) -> DbPath: + return parse_table_name(name) + + def _query_cursor(self, c, sql_code: str): + assert isinstance(sql_code, str), sql_code + try: + c.execute(sql_code) + if sql_code.lower().startswith(("select", "explain", "show")): + return c.fetchall() + except Exception as _e: + # logger.exception(e) + # logger.error(f'Caused by SQL: {sql_code}') + raise + + def _query_conn(self, conn, sql_code: Union[str, ThreadLocalInterpreter]) -> list: + c = conn.cursor() + callback = partial(self._query_cursor, c) + return apply_query(callback, sql_code) + + def close(self): + self.is_closed = True + return super().close() + + +class ThreadedDatabase(Database): + """Access the database through singleton threads. + + Used for database connectors that do not support sharing their connection between different threads. + """ + + def __init__(self, thread_count=1): + self._init_error = None + self._queue = ThreadPoolExecutor(thread_count, initializer=self.set_conn) + self.thread_local = threading.local() + logger.info(f"[{self.name}] Starting a threadpool, size={thread_count}.") + + def set_conn(self): + assert not hasattr(self.thread_local, "conn") + try: + self.thread_local.conn = self.create_connection() + except ModuleNotFoundError as e: + self._init_error = e + + def _query(self, sql_code: Union[str, ThreadLocalInterpreter]): + r = self._queue.submit(self._query_in_worker, sql_code) + return r.result() + + def _query_in_worker(self, sql_code: Union[str, ThreadLocalInterpreter]): + "This method runs in a worker thread" + if self._init_error: + raise self._init_error + return self._query_conn(self.thread_local.conn, sql_code) + + @abstractmethod + def create_connection(self): + "Return a connection instance, that supports the .cursor() method." + + def close(self): + super().close() + self._queue.shutdown() + + @property + def is_autocommit(self) -> bool: + return False + + +CHECKSUM_HEXDIGITS = 15 # Must be 15 or lower, otherwise SUM() overflows +MD5_HEXDIGITS = 32 + +_CHECKSUM_BITSIZE = CHECKSUM_HEXDIGITS << 2 +CHECKSUM_MASK = (2**_CHECKSUM_BITSIZE) - 1 + +DEFAULT_DATETIME_PRECISION = 6 +DEFAULT_NUMERIC_PRECISION = 24 + +TIMESTAMP_PRECISION_POS = 20 # len("2022-06-03 12:24:35.") == 20 diff --git a/sqeleton/databases/bigquery.py b/sqeleton/databases/bigquery.py new file mode 100644 index 0000000..62e51a9 --- /dev/null +++ b/sqeleton/databases/bigquery.py @@ -0,0 +1,178 @@ +from typing import List, Union +from ..abcs.database_types import ( + Timestamp, + Datetime, + Integer, + Decimal, + Float, + Text, + DbPath, + FractionalType, + TemporalType, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue, AbstractMixin_Schema +from ..abcs import Compilable +from ..queries import this, table, SKIP +from .base import BaseDialect, Database, import_helper, parse_table_name, ConnectError, apply_query +from .base import TIMESTAMP_PRECISION_POS, ThreadLocalInterpreter + + +@import_helper(text="Please install BigQuery and configure your google-cloud access.") +def import_bigquery(): + from google.cloud import bigquery + + return bigquery + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"cast(cast( ('0x' || substr(TO_HEX(md5({s})), 18)) as int64) as numeric)" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + timestamp = f"timestamp_micros(cast(round(unix_micros(cast({value} as timestamp))/1000000, {coltype.precision})*1000000 as int))" + return f"FORMAT_TIMESTAMP('%F %H:%M:%E6S', {timestamp})" + + if coltype.precision == 0: + return f"FORMAT_TIMESTAMP('%F %H:%M:%S.000000', {value})" + elif coltype.precision == 6: + return f"FORMAT_TIMESTAMP('%F %H:%M:%E6S', {value})" + + timestamp6 = f"FORMAT_TIMESTAMP('%F %H:%M:%E6S', {value})" + return ( + f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')" + ) + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return f"format('%.{coltype.precision}f', {value})" + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"cast({value} as int)") + + +class Mixin_Schema(AbstractMixin_Schema): + def list_tables(self, table_schema: str, like: Compilable = None) -> Compilable: + return ( + table(table_schema, "INFORMATION_SCHEMA", "TABLES") + .where( + this.table_schema == table_schema, + this.table_name.like(like) if like is not None else SKIP, + this.table_type == "BASE TABLE", + ) + .select(this.table_name) + ) + + +class Dialect(BaseDialect, Mixin_Schema): + name = "BigQuery" + ROUNDS_ON_PREC_LOSS = False # Technically BigQuery doesn't allow implicit rounding or truncation + TYPE_CLASSES = { + # Dates + "TIMESTAMP": Timestamp, + "DATETIME": Datetime, + # Numbers + "INT64": Integer, + "INT32": Integer, + "NUMERIC": Decimal, + "BIGNUMERIC": Decimal, + "FLOAT64": Float, + "FLOAT32": Float, + # Text + "STRING": Text, + # Boolean + "BOOL": Boolean, + } + + def random(self) -> str: + return "RAND()" + + def quote(self, s: str): + return f"`{s}`" + + def to_string(self, s: str): + return f"cast({s} as string)" + + def type_repr(self, t) -> str: + try: + return {str: "STRING", float: "FLOAT64"}[t] + except KeyError: + return super().type_repr(t) + + def set_timezone_to_utc(self) -> str: + raise NotImplementedError() + + +class BigQuery(Database): + CONNECT_URI_HELP = "bigquery:///" + CONNECT_URI_PARAMS = ["dataset"] + dialect = Dialect() + + def __init__(self, project, *, dataset, **kw): + bigquery = import_bigquery() + + self._client = bigquery.Client(project, **kw) + self.project = project + self.dataset = dataset + + self.default_schema = dataset + + def _normalize_returned_value(self, value): + if isinstance(value, bytes): + return value.decode() + return value + + def _query_atom(self, sql_code: str): + from google.cloud import bigquery + + try: + res = list(self._client.query(sql_code)) + except Exception as e: + msg = "Exception when trying to execute SQL code:\n %s\n\nGot error: %s" + raise ConnectError(msg % (sql_code, e)) + + if res and isinstance(res[0], bigquery.table.Row): + res = [tuple(self._normalize_returned_value(v) for v in row.values()) for row in res] + return res + + def _query(self, sql_code: Union[str, ThreadLocalInterpreter]): + return apply_query(self._query_atom, sql_code) + + def close(self): + super().close() + self._client.close() + + def select_table_schema(self, path: DbPath) -> str: + project, schema, name = self._normalize_table_path(path) + return ( + "SELECT column_name, data_type, 6 as datetime_precision, 38 as numeric_precision, 9 as numeric_scale " + f"FROM `{project}`.`{schema}`.INFORMATION_SCHEMA.COLUMNS " + f"WHERE table_name = '{name}' AND table_schema = '{schema}'" + ) + + def query_table_unique_columns(self, path: DbPath) -> List[str]: + return [] + + def _normalize_table_path(self, path: DbPath) -> DbPath: + if len(path) == 0: + raise ValueError(f"{self.name}: Bad table path for {self}: ()") + elif len(path) == 1: + return (self.project, self.default_schema, path[0]) + elif len(path) == 2: + return (self.project,) + path + elif len(path) == 3: + return path + else: + raise ValueError( + f"{self.name}: Bad table path for {self}: '{'.'.join(path)}'. Expected form: [project.]schema.table" + ) + + def parse_table_name(self, name: str) -> DbPath: + path = parse_table_name(name) + return tuple(i for i in self._normalize_table_path(path) if i is not None) + + @property + def is_autocommit(self) -> bool: + return True diff --git a/sqeleton/databases/clickhouse.py b/sqeleton/databases/clickhouse.py new file mode 100644 index 0000000..eaa86b7 --- /dev/null +++ b/sqeleton/databases/clickhouse.py @@ -0,0 +1,192 @@ +from typing import Optional, Type + +from .base import ( + MD5_HEXDIGITS, + CHECKSUM_HEXDIGITS, + TIMESTAMP_PRECISION_POS, + BaseDialect, + ThreadedDatabase, + import_helper, + ConnectError, +) +from ..abcs.database_types import ( + ColType, + Decimal, + Float, + Integer, + FractionalType, + Native_UUID, + TemporalType, + Text, + Timestamp, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue + +# https://clickhouse.com/docs/en/operations/server-configuration-parameters/settings/#default-database +DEFAULT_DATABASE = "default" + + +@import_helper("clickhouse") +def import_clickhouse(): + import clickhouse_driver + + return clickhouse_driver + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + substr_idx = 1 + MD5_HEXDIGITS - CHECKSUM_HEXDIGITS + return f"reinterpretAsUInt128(reverse(unhex(lowerUTF8(substr(hex(MD5({s})), {substr_idx})))))" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_number(self, value: str, coltype: FractionalType) -> str: + # If a decimal value has trailing zeros in a fractional part, when casting to string they are dropped. + # For example: + # select toString(toDecimal128(1.10, 2)); -- the result is 1.1 + # select toString(toDecimal128(1.00, 2)); -- the result is 1 + # So, we should use some custom approach to save these trailing zeros. + # To avoid it, we can add a small value like 0.000001 to prevent dropping of zeros from the end when casting. + # For examples above it looks like: + # select toString(toDecimal128(1.10, 2 + 1) + toDecimal128(0.001, 3)); -- the result is 1.101 + # After that, cut an extra symbol from the string, i.e. 1.101 -> 1.10 + # So, the algorithm is: + # 1. Cast to decimal with precision + 1 + # 2. Add a small value 10^(-precision-1) + # 3. Cast the result to string + # 4. Drop the extra digit from the string. To do that, we need to slice the string + # with length = digits in an integer part + 1 (symbol of ".") + precision + + if coltype.precision == 0: + return self.to_string(f"round({value})") + + precision = coltype.precision + # TODO: too complex, is there better performance way? + value = f""" + if({value} >= 0, '', '-') || left( + toString( + toDecimal128( + round(abs({value}), {precision}), + {precision} + 1 + ) + + + toDecimal128( + exp10(-{precision + 1}), + {precision} + 1 + ) + ), + toUInt8( + greatest( + floor(log10(abs({value}))) + 1, + 1 + ) + ) + 1 + {precision} + ) + """ + return value + + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + prec = coltype.precision + if coltype.rounds: + timestamp = f"toDateTime64(round(toUnixTimestamp64Micro(toDateTime64({value}, 6)) / 1000000, {prec}), 6)" + return self.to_string(timestamp) + + fractional = f"toUnixTimestamp64Micro(toDateTime64({value}, {prec})) % 1000000" + fractional = f"lpad({self.to_string(fractional)}, 6, '0')" + value = f"formatDateTime({value}, '%Y-%m-%d %H:%M:%S') || '.' || {self.to_string(fractional)}" + return f"rpad({value}, {TIMESTAMP_PRECISION_POS + 6}, '0')" + + +class Dialect(BaseDialect): + name = "Clickhouse" + ROUNDS_ON_PREC_LOSS = False + TYPE_CLASSES = { + "Int8": Integer, + "Int16": Integer, + "Int32": Integer, + "Int64": Integer, + "Int128": Integer, + "Int256": Integer, + "UInt8": Integer, + "UInt16": Integer, + "UInt32": Integer, + "UInt64": Integer, + "UInt128": Integer, + "UInt256": Integer, + "Float32": Float, + "Float64": Float, + "Decimal": Decimal, + "UUID": Native_UUID, + "String": Text, + "FixedString": Text, + "DateTime": Timestamp, + "DateTime64": Timestamp, + } + + def quote(self, s: str) -> str: + return f'"{s}"' + + def to_string(self, s: str) -> str: + return f"toString({s})" + + def _convert_db_precision_to_digits(self, p: int) -> int: + # Done the same as for PostgreSQL but need to rewrite in another way + # because it does not help for float with a big integer part. + return super()._convert_db_precision_to_digits(p) - 2 + + def _parse_type_repr(self, type_repr: str) -> Optional[Type[ColType]]: + nullable_prefix = "Nullable(" + if type_repr.startswith(nullable_prefix): + type_repr = type_repr[len(nullable_prefix) :].rstrip(")") + + if type_repr.startswith("Decimal"): + type_repr = "Decimal" + elif type_repr.startswith("FixedString"): + type_repr = "FixedString" + elif type_repr.startswith("DateTime64"): + type_repr = "DateTime64" + + return self.TYPE_CLASSES.get(type_repr) + + # def timestamp_value(self, t: DbTime) -> str: + # # return f"'{t}'" + # return f"'{str(t)[:19]}'" + + def set_timezone_to_utc(self) -> str: + raise NotImplementedError() + + def current_timestamp(self) -> str: + return "now()" + + +class Clickhouse(ThreadedDatabase): + dialect = Dialect() + CONNECT_URI_HELP = "clickhouse://:@/" + CONNECT_URI_PARAMS = ["database?"] + + def __init__(self, *, thread_count: int, **kw): + super().__init__(thread_count=thread_count) + + self._args = kw + # In Clickhouse database and schema are the same + self.default_schema = kw.get("database", DEFAULT_DATABASE) + + def create_connection(self): + clickhouse = import_clickhouse() + + class SingleConnection(clickhouse.dbapi.connection.Connection): + """Not thread-safe connection to Clickhouse""" + + def cursor(self, cursor_factory=None): + if not len(self.cursors): + _ = super().cursor() + return self.cursors[0] + + try: + return SingleConnection(**self._args) + except clickhouse.OperationError as e: + raise ConnectError(*e.args) from e + + @property + def is_autocommit(self) -> bool: + return True diff --git a/sqeleton/databases/databricks.py b/sqeleton/databases/databricks.py new file mode 100644 index 0000000..833e4b8 --- /dev/null +++ b/sqeleton/databases/databricks.py @@ -0,0 +1,178 @@ +import math +from typing import Dict, Sequence +import logging + +from ..abcs.database_types import ( + Integer, + Float, + Decimal, + Timestamp, + Text, + TemporalType, + NumericType, + DbPath, + ColType, + UnknownColType, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue +from .base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, BaseDialect, ThreadedDatabase, import_helper, parse_table_name + + +@import_helper(text="You can install it using 'pip install databricks-sql-connector'") +def import_databricks(): + import databricks.sql + + return databricks + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"cast(conv(substr(md5({s}), {1+MD5_HEXDIGITS-CHECKSUM_HEXDIGITS}), 16, 10) as decimal(38, 0))" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + """Databricks timestamp contains no more than 6 digits in precision""" + + if coltype.rounds: + timestamp = f"cast(round(unix_micros({value}) / 1000000, {coltype.precision}) * 1000000 as bigint)" + return f"date_format(timestamp_micros({timestamp}), 'yyyy-MM-dd HH:mm:ss.SSSSSS')" + + precision_format = "S" * coltype.precision + "0" * (6 - coltype.precision) + return f"date_format({value}, 'yyyy-MM-dd HH:mm:ss.{precision_format}')" + + def normalize_number(self, value: str, coltype: NumericType) -> str: + value = f"cast({value} as decimal(38, {coltype.precision}))" + if coltype.precision > 0: + value = f"format_number({value}, {coltype.precision})" + return f"replace({self.to_string(value)}, ',', '')" + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"cast ({value} as int)") + + +class Dialect(BaseDialect): + name = "Databricks" + ROUNDS_ON_PREC_LOSS = True + TYPE_CLASSES = { + # Numbers + "INT": Integer, + "SMALLINT": Integer, + "TINYINT": Integer, + "BIGINT": Integer, + "FLOAT": Float, + "DOUBLE": Float, + "DECIMAL": Decimal, + # Timestamps + "TIMESTAMP": Timestamp, + # Text + "STRING": Text, + # Boolean + "BOOLEAN": Boolean, + } + + def quote(self, s: str): + return f"`{s}`" + + def to_string(self, s: str) -> str: + return f"cast({s} as string)" + + def _convert_db_precision_to_digits(self, p: int) -> int: + # Subtracting 2 due to wierd precision issues + return max(super()._convert_db_precision_to_digits(p) - 2, 0) + + def set_timezone_to_utc(self) -> str: + return "SET TIME ZONE 'UTC'" + + +class Databricks(ThreadedDatabase): + dialect = Dialect() + CONNECT_URI_HELP = "databricks://:@/" + CONNECT_URI_PARAMS = ["catalog", "schema"] + + def __init__(self, *, thread_count, **kw): + logging.getLogger("databricks.sql").setLevel(logging.WARNING) + + self._args = kw + self.default_schema = kw.get("schema", "default") + self.catalog = self._args.get("catalog", "hive_metastore") + super().__init__(thread_count=thread_count) + + def create_connection(self): + databricks = import_databricks() + + try: + return databricks.sql.connect( + server_hostname=self._args["server_hostname"], + http_path=self._args["http_path"], + access_token=self._args["access_token"], + catalog=self.catalog, + ) + except databricks.sql.exc.Error as e: + raise ConnectionError(*e.args) from e + + def query_table_schema(self, path: DbPath) -> Dict[str, tuple]: + # Databricks has INFORMATION_SCHEMA only for Databricks Runtime, not for Databricks SQL. + # https://docs.databricks.com/spark/latest/spark-sql/language-manual/information-schema/columns.html + # So, to obtain information about schema, we should use another approach. + + conn = self.create_connection() + + schema, table = self._normalize_table_path(path) + with conn.cursor() as cursor: + cursor.columns(catalog_name=self.catalog, schema_name=schema, table_name=table) + try: + rows = cursor.fetchall() + finally: + conn.close() + if not rows: + raise RuntimeError(f"{self.name}: Table '{'.'.join(path)}' does not exist, or has no columns") + + d = {r.COLUMN_NAME: (r.COLUMN_NAME, r.TYPE_NAME, r.DECIMAL_DIGITS, None, None) for r in rows} + assert len(d) == len(rows) + return d + + def _process_table_schema( + self, path: DbPath, raw_schema: Dict[str, tuple], filter_columns: Sequence[str], where: str = None + ): + accept = {i.lower() for i in filter_columns} + rows = [row for name, row in raw_schema.items() if name.lower() in accept] + + resulted_rows = [] + for row in rows: + row_type = "DECIMAL" if row[1].startswith("DECIMAL") else row[1] + type_cls = self.dialect.TYPE_CLASSES.get(row_type, UnknownColType) + + if issubclass(type_cls, Integer): + row = (row[0], row_type, None, None, 0) + + elif issubclass(type_cls, Float): + numeric_precision = math.ceil(row[2] / math.log(2, 10)) + row = (row[0], row_type, None, numeric_precision, None) + + elif issubclass(type_cls, Decimal): + items = row[1][8:].rstrip(")").split(",") + numeric_precision, numeric_scale = int(items[0]), int(items[1]) + row = (row[0], row_type, None, numeric_precision, numeric_scale) + + elif issubclass(type_cls, Timestamp): + row = (row[0], row_type, row[2], None, None) + + else: + row = (row[0], row_type, None, None, None) + + resulted_rows.append(row) + + col_dict: Dict[str, ColType] = {row[0]: self.dialect.parse_type(path, *row) for row in resulted_rows} + + self._refine_coltypes(path, col_dict, where) + return col_dict + + def parse_table_name(self, name: str) -> DbPath: + path = parse_table_name(name) + return tuple(i for i in self._normalize_table_path(path) if i is not None) + + @property + def is_autocommit(self) -> bool: + return True diff --git a/sqeleton/databases/duckdb.py b/sqeleton/databases/duckdb.py new file mode 100644 index 0000000..1ba537c --- /dev/null +++ b/sqeleton/databases/duckdb.py @@ -0,0 +1,146 @@ +from typing import Union + +from ..utils import match_regexps +from ..abcs.database_types import ( + Timestamp, + TimestampTZ, + DbPath, + ColType, + Float, + Decimal, + Integer, + TemporalType, + Native_UUID, + Text, + FractionalType, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue +from .base import ( + Database, + BaseDialect, + import_helper, + ConnectError, + ThreadLocalInterpreter, + TIMESTAMP_PRECISION_POS, +) +from .base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, Mixin_Schema + + +@import_helper("duckdb") +def import_duckdb(): + import duckdb + + return duckdb + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"('0x' || SUBSTRING(md5({s}), {1+MD5_HEXDIGITS-CHECKSUM_HEXDIGITS},{CHECKSUM_HEXDIGITS}))::BIGINT" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + # It's precision 6 by default. If precision is less than 6 -> we remove the trailing numbers. + if coltype.rounds and coltype.precision > 0: + return f"CONCAT(SUBSTRING(STRFTIME({value}::TIMESTAMP, '%Y-%m-%d %H:%M:%S.'),1,23), LPAD(((ROUND(strftime({value}::timestamp, '%f')::DECIMAL(15,7)/100000,{coltype.precision-1})*100000)::INT)::VARCHAR,6,'0'))" + + return f"rpad(substring(strftime({value}::timestamp, '%Y-%m-%d %H:%M:%S.%f'),1,{TIMESTAMP_PRECISION_POS+coltype.precision}),26,'0')" + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"{value}::DECIMAL(38, {coltype.precision})") + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"{value}::INTEGER") + + +class Dialect(BaseDialect, Mixin_Schema): + name = "DuckDB" + ROUNDS_ON_PREC_LOSS = False + SUPPORTS_PRIMARY_KEY = True + SUPPORTS_INDEXES = True + + TYPE_CLASSES = { + # Timestamps + "TIMESTAMP WITH TIME ZONE": TimestampTZ, + "TIMESTAMP": Timestamp, + # Numbers + "DOUBLE": Float, + "FLOAT": Float, + "DECIMAL": Decimal, + "INTEGER": Integer, + "BIGINT": Integer, + # Text + "VARCHAR": Text, + "TEXT": Text, + # UUID + "UUID": Native_UUID, + # Bool + "BOOLEAN": Boolean, + } + + def quote(self, s: str): + return f'"{s}"' + + def to_string(self, s: str): + return f"{s}::VARCHAR" + + def _convert_db_precision_to_digits(self, p: int) -> int: + # Subtracting 2 due to wierd precision issues in PostgreSQL + return super()._convert_db_precision_to_digits(p) - 2 + + def parse_type( + self, + table_path: DbPath, + col_name: str, + type_repr: str, + datetime_precision: int = None, + numeric_precision: int = None, + numeric_scale: int = None, + ) -> ColType: + regexps = { + r"DECIMAL\((\d+),(\d+)\)": Decimal, + } + + for m, t_cls in match_regexps(regexps, type_repr): + precision = int(m.group(2)) + return t_cls(precision=precision) + + return super().parse_type(table_path, col_name, type_repr, datetime_precision, numeric_precision, numeric_scale) + + def set_timezone_to_utc(self) -> str: + return "SET GLOBAL TimeZone='UTC'" + + def current_timestamp(self) -> str: + return "current_timestamp" + + +class DuckDB(Database): + dialect = Dialect() + SUPPORTS_UNIQUE_CONSTAINT = False # Temporary, until we implement it + default_schema = "main" + CONNECT_URI_HELP = "duckdb://@" + CONNECT_URI_PARAMS = ["database", "dbpath"] + + def __init__(self, **kw): + self._args = kw + self._conn = self.create_connection() + + @property + def is_autocommit(self) -> bool: + return True + + def _query(self, sql_code: Union[str, ThreadLocalInterpreter]): + "Uses the standard SQL cursor interface" + return self._query_conn(self._conn, sql_code) + + def close(self): + super().close() + self._conn.close() + + def create_connection(self): + ddb = import_duckdb() + try: + return ddb.connect(self._args["filepath"]) + except ddb.OperationalError as e: + raise ConnectError(*e.args) from e diff --git a/sqeleton/databases/mssql.py b/sqeleton/databases/mssql.py new file mode 100644 index 0000000..8d394e3 --- /dev/null +++ b/sqeleton/databases/mssql.py @@ -0,0 +1,25 @@ +# class MsSQL(ThreadedDatabase): +# "AKA sql-server" + +# def __init__(self, host, port, user, password, *, database, thread_count, **kw): +# args = dict(server=host, port=port, database=database, user=user, password=password, **kw) +# self._args = {k: v for k, v in args.items() if v is not None} + +# super().__init__(thread_count=thread_count) + +# def create_connection(self): +# mssql = import_mssql() +# try: +# return mssql.connect(**self._args) +# except mssql.Error as e: +# raise ConnectError(*e.args) from e + +# def quote(self, s: str): +# return f"[{s}]" + +# def md5_as_int(self, s: str) -> str: +# return f"CONVERT(decimal(38,0), CONVERT(bigint, HashBytes('MD5', {s}), 2))" +# # return f"CONVERT(bigint, (CHECKSUM({s})))" + +# def to_string(self, s: str): +# return f"CONVERT(varchar, {s})" diff --git a/sqeleton/databases/mysql.py b/sqeleton/databases/mysql.py new file mode 100644 index 0000000..87dcc3e --- /dev/null +++ b/sqeleton/databases/mysql.py @@ -0,0 +1,128 @@ +from ..abcs.database_types import ( + Datetime, + Timestamp, + Float, + Decimal, + Integer, + Text, + TemporalType, + FractionalType, + ColType_UUID, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue +from .base import ( + ThreadedDatabase, + import_helper, + ConnectError, + BaseDialect, +) +from .base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, TIMESTAMP_PRECISION_POS, Mixin_Schema + + +@import_helper("mysql") +def import_mysql(): + import mysql.connector + + return mysql.connector + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"cast(conv(substring(md5({s}), {1+MD5_HEXDIGITS-CHECKSUM_HEXDIGITS}), 16, 10) as unsigned)" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + return self.to_string(f"cast( cast({value} as datetime({coltype.precision})) as datetime(6))") + + s = self.to_string(f"cast({value} as datetime(6))") + return f"RPAD(RPAD({s}, {TIMESTAMP_PRECISION_POS+coltype.precision}, '.'), {TIMESTAMP_PRECISION_POS+6}, '0')" + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"cast({value} as decimal(38, {coltype.precision}))") + + def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str: + return f"TRIM(CAST({value} AS char))" + + +class Dialect(BaseDialect, Mixin_Schema): + name = "MySQL" + ROUNDS_ON_PREC_LOSS = True + SUPPORTS_PRIMARY_KEY = True + SUPPORTS_INDEXES = True + TYPE_CLASSES = { + # Dates + "datetime": Datetime, + "timestamp": Timestamp, + # Numbers + "double": Float, + "float": Float, + "decimal": Decimal, + "int": Integer, + "bigint": Integer, + # Text + "varchar": Text, + "char": Text, + "varbinary": Text, + "binary": Text, + # Boolean + "boolean": Boolean, + } + + def quote(self, s: str): + return f"`{s}`" + + def to_string(self, s: str): + return f"cast({s} as char)" + + def is_distinct_from(self, a: str, b: str) -> str: + return f"not ({a} <=> {b})" + + def random(self) -> str: + return "RAND()" + + def type_repr(self, t) -> str: + try: + return { + str: "VARCHAR(1024)", + }[t] + except KeyError: + return super().type_repr(t) + + def explain_as_text(self, query: str) -> str: + return f"EXPLAIN FORMAT=TREE {query}" + + def set_timezone_to_utc(self) -> str: + return "SET @@session.time_zone='+00:00'" + + +class MySQL(ThreadedDatabase): + dialect = Dialect() + SUPPORTS_ALPHANUMS = False + SUPPORTS_UNIQUE_CONSTAINT = True + CONNECT_URI_HELP = "mysql://:@/" + CONNECT_URI_PARAMS = ["database?"] + + def __init__(self, *, thread_count, **kw): + self._args = kw + + super().__init__(thread_count=thread_count) + + # In MySQL schema and database are synonymous + try: + self.default_schema = kw["database"] + except KeyError: + raise ValueError("MySQL URL must specify a database") + + def create_connection(self): + mysql = import_mysql() + try: + return mysql.connect(charset="utf8", use_unicode=True, **self._args) + except mysql.Error as e: + if e.errno == mysql.errorcode.ER_ACCESS_DENIED_ERROR: + raise ConnectError("Bad user name or password") from e + elif e.errno == mysql.errorcode.ER_BAD_DB_ERROR: + raise ConnectError("Database does not exist") from e + raise ConnectError(*e.args) from e diff --git a/sqeleton/databases/oracle.py b/sqeleton/databases/oracle.py new file mode 100644 index 0000000..729511b --- /dev/null +++ b/sqeleton/databases/oracle.py @@ -0,0 +1,194 @@ +from typing import Dict, List, Optional + +from ..utils import match_regexps +from ..abcs.database_types import ( + Decimal, + Float, + Text, + DbPath, + TemporalType, + ColType, + DbTime, + ColType_UUID, + Timestamp, + TimestampTZ, + FractionalType, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue, AbstractMixin_Schema +from ..abcs import Compilable +from ..queries import this, table, SKIP +from .base import BaseDialect, ThreadedDatabase, import_helper, ConnectError, QueryError +from .base import TIMESTAMP_PRECISION_POS + +SESSION_TIME_ZONE = None # Changed by the tests + + +@import_helper("oracle") +def import_oracle(): + import cx_Oracle + + return cx_Oracle + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + # standard_hash is faster than DBMS_CRYPTO.Hash + # TODO: Find a way to use UTL_RAW.CAST_TO_BINARY_INTEGER ? + return f"to_number(substr(standard_hash({s}, 'MD5'), 18), 'xxxxxxxxxxxxxxx')" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str: + # Cast is necessary for correct MD5 (trimming not enough) + return f"CAST(TRIM({value}) AS VARCHAR(36))" + + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + return f"to_char(cast({value} as timestamp({coltype.precision})), 'YYYY-MM-DD HH24:MI:SS.FF6')" + + if coltype.precision > 0: + truncated = f"to_char({value}, 'YYYY-MM-DD HH24:MI:SS.FF{coltype.precision}')" + else: + truncated = f"to_char({value}, 'YYYY-MM-DD HH24:MI:SS.')" + return f"RPAD({truncated}, {TIMESTAMP_PRECISION_POS+6}, '0')" + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + # FM999.9990 + format_str = "FM" + "9" * (38 - coltype.precision) + if coltype.precision: + format_str += "0." + "9" * (coltype.precision - 1) + "0" + return f"to_char({value}, '{format_str}')" + + +class Mixin_Schema(AbstractMixin_Schema): + def list_tables(self, table_schema: str, like: Compilable = None) -> Compilable: + return ( + table("ALL_TABLES") + .where( + this.OWNER == table_schema, + this.TABLE_NAME.like(like) if like is not None else SKIP, + ) + .select(table_name=this.TABLE_NAME) + ) + + +class Dialect(BaseDialect, Mixin_Schema): + name = "Oracle" + SUPPORTS_PRIMARY_KEY = True + SUPPORTS_INDEXES = True + TYPE_CLASSES: Dict[str, type] = { + "NUMBER": Decimal, + "FLOAT": Float, + # Text + "CHAR": Text, + "NCHAR": Text, + "NVARCHAR2": Text, + "VARCHAR2": Text, + } + ROUNDS_ON_PREC_LOSS = True + PLACEHOLDER_TABLE = "DUAL" + + def quote(self, s: str): + return f'"{s}"' + + def to_string(self, s: str): + return f"cast({s} as varchar(1024))" + + def offset_limit(self, offset: Optional[int] = None, limit: Optional[int] = None): + if offset: + raise NotImplementedError("No support for OFFSET in query") + + return f"FETCH NEXT {limit} ROWS ONLY" + + def concat(self, items: List[str]) -> str: + joined_exprs = " || ".join(items) + return f"({joined_exprs})" + + def timestamp_value(self, t: DbTime) -> str: + return "timestamp '%s'" % t.isoformat(" ") + + def random(self) -> str: + return "dbms_random.value" + + def is_distinct_from(self, a: str, b: str) -> str: + return f"DECODE({a}, {b}, 1, 0) = 0" + + def type_repr(self, t) -> str: + try: + return { + str: "VARCHAR(1024)", + }[t] + except KeyError: + return super().type_repr(t) + + def constant_values(self, rows) -> str: + return " UNION ALL ".join( + "SELECT %s FROM DUAL" % ", ".join(self._constant_value(v) for v in row) for row in rows + ) + + def explain_as_text(self, query: str) -> str: + raise NotImplementedError("Explain not yet implemented in Oracle") + + def parse_type( + self, + table_path: DbPath, + col_name: str, + type_repr: str, + datetime_precision: int = None, + numeric_precision: int = None, + numeric_scale: int = None, + ) -> ColType: + regexps = { + r"TIMESTAMP\((\d)\) WITH LOCAL TIME ZONE": Timestamp, + r"TIMESTAMP\((\d)\) WITH TIME ZONE": TimestampTZ, + r"TIMESTAMP\((\d)\)": Timestamp, + } + + for m, t_cls in match_regexps(regexps, type_repr): + precision = int(m.group(1)) + return t_cls(precision=precision, rounds=self.ROUNDS_ON_PREC_LOSS) + + return super().parse_type(table_path, col_name, type_repr, datetime_precision, numeric_precision, numeric_scale) + + def set_timezone_to_utc(self) -> str: + return "ALTER SESSION SET TIME_ZONE = 'UTC'" + + def current_timestamp(self) -> str: + return "LOCALTIMESTAMP" + + +class Oracle(ThreadedDatabase): + dialect = Dialect() + CONNECT_URI_HELP = "oracle://:@/" + CONNECT_URI_PARAMS = ["database?"] + + def __init__(self, *, host, database, thread_count, **kw): + self.kwargs = dict(dsn=f"{host}/{database}" if database else host, **kw) + + self.default_schema = kw.get("user").upper() + + super().__init__(thread_count=thread_count) + + def create_connection(self): + self._oracle = import_oracle() + try: + c = self._oracle.connect(**self.kwargs) + if SESSION_TIME_ZONE: + c.cursor().execute(f"ALTER SESSION SET TIME_ZONE = '{SESSION_TIME_ZONE}'") + return c + except Exception as e: + raise ConnectError(*e.args) from e + + def _query_cursor(self, c, sql_code: str): + try: + return super()._query_cursor(c, sql_code) + except self._oracle.DatabaseError as e: + raise QueryError(e) + + def select_table_schema(self, path: DbPath) -> str: + schema, name = self._normalize_table_path(path) + + return ( + f"SELECT column_name, data_type, 6 as datetime_precision, data_precision as numeric_precision, data_scale as numeric_scale" + f" FROM ALL_TAB_COLUMNS WHERE table_name = '{name}' AND owner = '{schema}'" + ) diff --git a/sqeleton/databases/postgresql.py b/sqeleton/databases/postgresql.py new file mode 100644 index 0000000..7983ea0 --- /dev/null +++ b/sqeleton/databases/postgresql.py @@ -0,0 +1,121 @@ +from ..abcs.database_types import ( + Timestamp, + TimestampTZ, + Float, + Decimal, + Integer, + TemporalType, + Native_UUID, + Text, + FractionalType, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue +from .base import BaseDialect, ThreadedDatabase, import_helper, ConnectError, Mixin_Schema +from .base import MD5_HEXDIGITS, CHECKSUM_HEXDIGITS, _CHECKSUM_BITSIZE, TIMESTAMP_PRECISION_POS + +SESSION_TIME_ZONE = None # Changed by the tests + + +@import_helper("postgresql") +def import_postgresql(): + import psycopg2 + import psycopg2.extras + + psycopg2.extensions.set_wait_callback(psycopg2.extras.wait_select) + return psycopg2 + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"('x' || substring(md5({s}), {1+MD5_HEXDIGITS-CHECKSUM_HEXDIGITS}))::bit({_CHECKSUM_BITSIZE})::bigint" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + return f"to_char({value}::timestamp({coltype.precision}), 'YYYY-mm-dd HH24:MI:SS.US')" + + timestamp6 = f"to_char({value}::timestamp(6), 'YYYY-mm-dd HH24:MI:SS.US')" + return ( + f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')" + ) + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"{value}::decimal(38, {coltype.precision})") + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"{value}::int") + + +class PostgresqlDialect(BaseDialect, Mixin_Schema): + name = "PostgreSQL" + ROUNDS_ON_PREC_LOSS = True + SUPPORTS_PRIMARY_KEY = True + SUPPORTS_INDEXES = True + + TYPE_CLASSES = { + # Timestamps + "timestamp with time zone": TimestampTZ, + "timestamp without time zone": Timestamp, + "timestamp": Timestamp, + # Numbers + "double precision": Float, + "real": Float, + "decimal": Decimal, + "integer": Integer, + "numeric": Decimal, + "bigint": Integer, + # Text + "character": Text, + "character varying": Text, + "varchar": Text, + "text": Text, + # UUID + "uuid": Native_UUID, + # Boolean + "boolean": Boolean, + } + + def quote(self, s: str): + return f'"{s}"' + + def to_string(self, s: str): + return f"{s}::varchar" + + def _convert_db_precision_to_digits(self, p: int) -> int: + # Subtracting 2 due to wierd precision issues in PostgreSQL + return super()._convert_db_precision_to_digits(p) - 2 + + def set_timezone_to_utc(self) -> str: + return "SET TIME ZONE 'UTC'" + + def current_timestamp(self) -> str: + return "current_timestamp" + + +class PostgreSQL(ThreadedDatabase): + dialect = PostgresqlDialect() + SUPPORTS_UNIQUE_CONSTAINT = True + CONNECT_URI_HELP = "postgresql://:@/" + CONNECT_URI_PARAMS = ["database?"] + + default_schema = "public" + + def __init__(self, *, thread_count, **kw): + self._args = kw + + super().__init__(thread_count=thread_count) + + def create_connection(self): + if not self._args: + self._args["host"] = None # psycopg2 requires 1+ arguments + + pg = import_postgresql() + try: + c = pg.connect(**self._args) + if SESSION_TIME_ZONE: + c.cursor().execute(f"SET TIME ZONE '{SESSION_TIME_ZONE}'") + return c + except pg.OperationalError as e: + raise ConnectError(*e.args) from e diff --git a/sqeleton/databases/presto.py b/sqeleton/databases/presto.py new file mode 100644 index 0000000..fed4861 --- /dev/null +++ b/sqeleton/databases/presto.py @@ -0,0 +1,191 @@ +from functools import partial +import re + +from ..utils import match_regexps + +from ..abcs.database_types import ( + Timestamp, + TimestampTZ, + Integer, + Float, + Text, + FractionalType, + DbPath, + DbTime, + Decimal, + ColType, + ColType_UUID, + TemporalType, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue +from .base import BaseDialect, Database, import_helper, ThreadLocalInterpreter, Mixin_Schema +from .base import ( + MD5_HEXDIGITS, + CHECKSUM_HEXDIGITS, + TIMESTAMP_PRECISION_POS, +) + + +def query_cursor(c, sql_code): + c.execute(sql_code) + if sql_code.lower().startswith("select"): + return c.fetchall() + # Required for the query to actually run 🤯 + if re.match(r"(insert|create|truncate|drop|explain)", sql_code, re.IGNORECASE): + return c.fetchone() + + +@import_helper("presto") +def import_presto(): + import prestodb + + return prestodb + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"cast(from_base(substr(to_hex(md5(to_utf8({s}))), {1+MD5_HEXDIGITS-CHECKSUM_HEXDIGITS}), 16) as decimal(38, 0))" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str: + # Trim doesn't work on CHAR type + return f"TRIM(CAST({value} AS VARCHAR))" + + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + # TODO rounds + if coltype.rounds: + s = f"date_format(cast({value} as timestamp(6)), '%Y-%m-%d %H:%i:%S.%f')" + else: + s = f"date_format(cast({value} as timestamp(6)), '%Y-%m-%d %H:%i:%S.%f')" + + return f"RPAD(RPAD({s}, {TIMESTAMP_PRECISION_POS+coltype.precision}, '.'), {TIMESTAMP_PRECISION_POS+6}, '0')" + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"cast({value} as decimal(38,{coltype.precision}))") + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"cast ({value} as int)") + + +class Dialect(BaseDialect, Mixin_Schema): + name = "Presto" + ROUNDS_ON_PREC_LOSS = True + TYPE_CLASSES = { + # Timestamps + "timestamp with time zone": TimestampTZ, + "timestamp without time zone": Timestamp, + "timestamp": Timestamp, + # Numbers + "integer": Integer, + "bigint": Integer, + "real": Float, + "double": Float, + # Text + "varchar": Text, + # Boolean + "boolean": Boolean, + } + + def explain_as_text(self, query: str) -> str: + return f"EXPLAIN (FORMAT TEXT) {query}" + + def type_repr(self, t) -> str: + try: + return {float: "REAL"}[t] + except KeyError: + return super().type_repr(t) + + def timestamp_value(self, t: DbTime) -> str: + return f"timestamp '{t.isoformat(' ')}'" + + def quote(self, s: str): + return f'"{s}"' + + def to_string(self, s: str): + return f"cast({s} as varchar)" + + def parse_type( + self, + table_path: DbPath, + col_name: str, + type_repr: str, + datetime_precision: int = None, + numeric_precision: int = None, + _numeric_scale: int = None, + ) -> ColType: + timestamp_regexps = { + r"timestamp\((\d)\)": Timestamp, + r"timestamp\((\d)\) with time zone": TimestampTZ, + } + for m, t_cls in match_regexps(timestamp_regexps, type_repr): + precision = int(m.group(1)) + return t_cls(precision=precision, rounds=self.ROUNDS_ON_PREC_LOSS) + + number_regexps = {r"decimal\((\d+),(\d+)\)": Decimal} + for m, n_cls in match_regexps(number_regexps, type_repr): + _prec, scale = map(int, m.groups()) + return n_cls(scale) + + string_regexps = {r"varchar\((\d+)\)": Text, r"char\((\d+)\)": Text} + for m, n_cls in match_regexps(string_regexps, type_repr): + return n_cls() + + return super().parse_type(table_path, col_name, type_repr, datetime_precision, numeric_precision) + + def set_timezone_to_utc(self) -> str: + return "SET TIME ZONE '+00:00'" + + def current_timestamp(self) -> str: + return "current_timestamp" + + +class Presto(Database): + dialect = Dialect() + CONNECT_URI_HELP = "presto://@//" + CONNECT_URI_PARAMS = ["catalog", "schema"] + + default_schema = "public" + + def __init__(self, **kw): + prestodb = import_presto() + + if kw.get("schema"): + self.default_schema = kw.get("schema") + + if kw.get("auth") == "basic": # if auth=basic, add basic authenticator for Presto + kw["auth"] = prestodb.auth.BasicAuthentication(kw.pop("user"), kw.pop("password")) + + if "cert" in kw: # if a certificate was specified in URI, verify session with cert + cert = kw.pop("cert") + self._conn = prestodb.dbapi.connect(**kw) + self._conn._http_session.verify = cert + else: + self._conn = prestodb.dbapi.connect(**kw) + + def _query(self, sql_code: str) -> list: + "Uses the standard SQL cursor interface" + c = self._conn.cursor() + + if isinstance(sql_code, ThreadLocalInterpreter): + return sql_code.apply_queries(partial(query_cursor, c)) + + return query_cursor(c, sql_code) + + def close(self): + super().close() + self._conn.close() + + def select_table_schema(self, path: DbPath) -> str: + schema, table = self._normalize_table_path(path) + + return ( + "SELECT column_name, data_type, 3 as datetime_precision, 3 as numeric_precision, NULL as numeric_scale " + "FROM INFORMATION_SCHEMA.COLUMNS " + f"WHERE table_name = '{table}' AND table_schema = '{schema}'" + ) + + @property + def is_autocommit(self) -> bool: + return False diff --git a/sqeleton/databases/redshift.py b/sqeleton/databases/redshift.py new file mode 100644 index 0000000..e44847c --- /dev/null +++ b/sqeleton/databases/redshift.py @@ -0,0 +1,100 @@ +from typing import List, Dict +from ..abcs.database_types import Float, TemporalType, FractionalType, DbPath +from ..abcs.mixins import AbstractMixin_MD5 +from .postgresql import ( + PostgreSQL, + MD5_HEXDIGITS, + CHECKSUM_HEXDIGITS, + TIMESTAMP_PRECISION_POS, + PostgresqlDialect, + Mixin_NormalizeValue, +) + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"strtol(substring(md5({s}), {1+MD5_HEXDIGITS-CHECKSUM_HEXDIGITS}), 16)::decimal(38)" + + +class Mixin_NormalizeValue(Mixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + timestamp = f"{value}::timestamp(6)" + # Get seconds since epoch. Redshift doesn't support milli- or micro-seconds. + secs = f"timestamp 'epoch' + round(extract(epoch from {timestamp})::decimal(38)" + # Get the milliseconds from timestamp. + ms = f"extract(ms from {timestamp})" + # Get the microseconds from timestamp, without the milliseconds! + us = f"extract(us from {timestamp})" + # epoch = Total time since epoch in microseconds. + epoch = f"{secs}*1000000 + {ms}*1000 + {us}" + timestamp6 = ( + f"to_char({epoch}, -6+{coltype.precision}) * interval '0.000001 seconds', 'YYYY-mm-dd HH24:MI:SS.US')" + ) + else: + timestamp6 = f"to_char({value}::timestamp(6), 'YYYY-mm-dd HH24:MI:SS.US')" + return ( + f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')" + ) + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"{value}::decimal(38,{coltype.precision})") + + +class Dialect(PostgresqlDialect): + name = "Redshift" + TYPE_CLASSES = { + **PostgresqlDialect.TYPE_CLASSES, + "double": Float, + "real": Float, + } + SUPPORTS_INDEXES = False + + def concat(self, items: List[str]) -> str: + joined_exprs = " || ".join(items) + return f"({joined_exprs})" + + def is_distinct_from(self, a: str, b: str) -> str: + return f"{a} IS NULL AND NOT {b} IS NULL OR {b} IS NULL OR {a}!={b}" + + +class Redshift(PostgreSQL): + dialect = Dialect() + CONNECT_URI_HELP = "redshift://:@/" + CONNECT_URI_PARAMS = ["database?"] + + def select_table_schema(self, path: DbPath) -> str: + schema, table = self._normalize_table_path(path) + + return ( + "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM information_schema.columns " + f"WHERE table_name = '{table.lower()}' AND table_schema = '{schema.lower()}'" + ) + + def select_external_table_schema(self, path: DbPath) -> str: + schema, table = self._normalize_table_path(path) + + return f"""SELECT + columnname AS column_name + , CASE WHEN external_type = 'string' THEN 'varchar' ELSE external_type END AS data_type + , NULL AS datetime_precision + , NULL AS numeric_precision + , NULL AS numeric_scale + FROM svv_external_columns + WHERE tablename = '{table.lower()}' AND schemaname = '{schema.lower()}' + """ + + def query_external_table_schema(self, path: DbPath) -> Dict[str, tuple]: + rows = self.query(self.select_external_table_schema(path), list) + if not rows: + raise RuntimeError(f"{self.name}: Table '{'.'.join(path)}' does not exist, or has no columns") + + d = {r[0]: r for r in rows} + assert len(d) == len(rows) + return d + + def query_table_schema(self, path: DbPath) -> Dict[str, tuple]: + try: + return super().query_table_schema(path) + except RuntimeError: + return self.query_external_table_schema(path) diff --git a/sqeleton/databases/snowflake.py b/sqeleton/databases/snowflake.py new file mode 100644 index 0000000..e866f00 --- /dev/null +++ b/sqeleton/databases/snowflake.py @@ -0,0 +1,174 @@ +from typing import Union, List +import logging + +from ..abcs.database_types import ( + Timestamp, + TimestampTZ, + Decimal, + Float, + Text, + FractionalType, + TemporalType, + DbPath, + Boolean, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue, AbstractMixin_Schema +from ..abcs import Compilable +from sqeleton.queries import table, this, SKIP +from .base import BaseDialect, ConnectError, Database, import_helper, CHECKSUM_MASK, ThreadLocalInterpreter + + +@import_helper("snowflake") +def import_snowflake(): + import snowflake.connector + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.backends import default_backend + + return snowflake, serialization, default_backend + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"BITAND(md5_number_lower64({s}), {CHECKSUM_MASK})" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + timestamp = f"to_timestamp(round(date_part(epoch_nanosecond, {value}::timestamp(9))/1000000000, {coltype.precision}))" + else: + timestamp = f"cast({value} as timestamp({coltype.precision}))" + + return f"to_char({timestamp}, 'YYYY-MM-DD HH24:MI:SS.FF6')" + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"cast({value} as decimal(38, {coltype.precision}))") + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"{value}::int") + + +class Mixin_Schema(AbstractMixin_Schema): + def table_information(self) -> Compilable: + return table("INFORMATION_SCHEMA", "TABLES") + + def list_tables(self, table_schema: str, like: Compilable = None) -> Compilable: + return ( + self.table_information() + .where( + this.TABLE_SCHEMA == table_schema, + this.TABLE_NAME.like(like) if like is not None else SKIP, + this.TABLE_TYPE == "BASE TABLE", + ) + .select(table_name=this.TABLE_NAME) + ) + + +class Dialect(BaseDialect, Mixin_Schema): + name = "Snowflake" + ROUNDS_ON_PREC_LOSS = False + TYPE_CLASSES = { + # Timestamps + "TIMESTAMP_NTZ": Timestamp, + "TIMESTAMP_LTZ": Timestamp, + "TIMESTAMP_TZ": TimestampTZ, + # Numbers + "NUMBER": Decimal, + "FLOAT": Float, + # Text + "TEXT": Text, + # Boolean + "BOOLEAN": Boolean, + } + + def explain_as_text(self, query: str) -> str: + return f"EXPLAIN USING TEXT {query}" + + def quote(self, s: str): + return f'"{s}"' + + def to_string(self, s: str): + return f"cast({s} as string)" + + def table_information(self) -> Compilable: + return table("INFORMATION_SCHEMA", "TABLES") + + def set_timezone_to_utc(self) -> str: + return "ALTER SESSION SET TIMEZONE = 'UTC'" + + +class Snowflake(Database): + dialect = Dialect() + CONNECT_URI_HELP = "snowflake://:@//?warehouse=" + CONNECT_URI_PARAMS = ["database", "schema"] + CONNECT_URI_KWPARAMS = ["warehouse"] + + def __init__(self, *, schema: str, **kw): + snowflake, serialization, default_backend = import_snowflake() + logging.getLogger("snowflake.connector").setLevel(logging.WARNING) + + # Ignore the error: snowflake.connector.network.RetryRequest: could not find io module state + # It's a known issue: https://github.com/snowflakedb/snowflake-connector-python/issues/145 + logging.getLogger("snowflake.connector.network").disabled = True + + assert '"' not in schema, "Schema name should not contain quotes!" + # If a private key is used, read it from the specified path and pass it as "private_key" to the connector. + if "key" in kw: + with open(kw.get("key"), "rb") as key: + if "password" in kw: + raise ConnectError("Cannot use password and key at the same time") + p_key = serialization.load_pem_private_key( + key.read(), + password=None, + backend=default_backend(), + ) + + kw["private_key"] = p_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + self._conn = snowflake.connector.connect(schema=f'"{schema}"', **kw) + + self.default_schema = schema + + def close(self): + super().close() + self._conn.close() + + def _query(self, sql_code: Union[str, ThreadLocalInterpreter]): + "Uses the standard SQL cursor interface" + return self._query_conn(self._conn, sql_code) + + def select_table_schema(self, path: DbPath) -> str: + """Provide SQL for selecting the table schema as (name, type, date_prec, num_prec)""" + database, schema, name = self._normalize_table_path(path) + info_schema_path = ['information_schema','columns'] + if database: + info_schema_path.insert(0, database) + + return ( + "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale " + f"FROM {'.'.join(info_schema_path)} " + f"WHERE table_name = '{name}' AND table_schema = '{schema}'" + ) + + def _normalize_table_path(self, path: DbPath) -> DbPath: + if len(path) == 1: + return None, self.default_schema, path[0] + elif len(path) == 2: + return None, path[0], path[1] + elif len(path) == 3: + return path + + raise ValueError( + f"{self.name}: Bad table path for {self}: '{'.'.join(path)}'. Expected format: table, schema.table, or database.schema.table" + ) + + @property + def is_autocommit(self) -> bool: + return True + + def query_table_unique_columns(self, path: DbPath) -> List[str]: + return [] diff --git a/sqeleton/databases/trino.py b/sqeleton/databases/trino.py new file mode 100644 index 0000000..f997447 --- /dev/null +++ b/sqeleton/databases/trino.py @@ -0,0 +1,47 @@ +from ..abcs.database_types import TemporalType, ColType_UUID +from . import presto +from .base import import_helper +from .base import TIMESTAMP_PRECISION_POS + + +@import_helper("trino") +def import_trino(): + import trino + + return trino + + +Mixin_MD5 = presto.Mixin_MD5 + + +class Mixin_NormalizeValue(presto.Mixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + s = f"date_format(cast({value} as timestamp({coltype.precision})), '%Y-%m-%d %H:%i:%S.%f')" + else: + s = f"date_format(cast({value} as timestamp(6)), '%Y-%m-%d %H:%i:%S.%f')" + + return ( + f"RPAD(RPAD({s}, {TIMESTAMP_PRECISION_POS + coltype.precision}, '.'), {TIMESTAMP_PRECISION_POS + 6}, '0')" + ) + + def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str: + return f"TRIM({value})" + + +class Dialect(presto.Dialect): + name = "Trino" + + +class Trino(presto.Presto): + dialect = Dialect() + CONNECT_URI_HELP = "trino://@//" + CONNECT_URI_PARAMS = ["catalog", "schema"] + + def __init__(self, **kw): + trino = import_trino() + + if kw.get("schema"): + self.default_schema = kw.get("schema") + + self._conn = trino.dbapi.connect(**kw) diff --git a/sqeleton/databases/vertica.py b/sqeleton/databases/vertica.py new file mode 100644 index 0000000..6f60ef3 --- /dev/null +++ b/sqeleton/databases/vertica.py @@ -0,0 +1,179 @@ +from typing import List + +from ..utils import match_regexps +from .base import ( + CHECKSUM_HEXDIGITS, + MD5_HEXDIGITS, + TIMESTAMP_PRECISION_POS, + BaseDialect, + ConnectError, + DbPath, + ColType, + ThreadedDatabase, + import_helper, +) +from ..abcs.database_types import ( + Decimal, + Float, + FractionalType, + Integer, + TemporalType, + Text, + Timestamp, + TimestampTZ, + Boolean, + ColType_UUID, +) +from ..abcs.mixins import AbstractMixin_MD5, AbstractMixin_NormalizeValue, AbstractMixin_Schema +from ..abcs import Compilable +from ..queries import table, this, SKIP + + +@import_helper("vertica") +def import_vertica(): + import vertica_python + + return vertica_python + + +class Mixin_MD5(AbstractMixin_MD5): + def md5_as_int(self, s: str) -> str: + return f"CAST(HEX_TO_INTEGER(SUBSTRING(MD5({s}), {1 + MD5_HEXDIGITS - CHECKSUM_HEXDIGITS})) AS NUMERIC(38, 0))" + + +class Mixin_NormalizeValue(AbstractMixin_NormalizeValue): + def normalize_timestamp(self, value: str, coltype: TemporalType) -> str: + if coltype.rounds: + return f"TO_CHAR({value}::TIMESTAMP({coltype.precision}), 'YYYY-MM-DD HH24:MI:SS.US')" + + timestamp6 = f"TO_CHAR({value}::TIMESTAMP(6), 'YYYY-MM-DD HH24:MI:SS.US')" + return ( + f"RPAD(LEFT({timestamp6}, {TIMESTAMP_PRECISION_POS+coltype.precision}), {TIMESTAMP_PRECISION_POS+6}, '0')" + ) + + def normalize_number(self, value: str, coltype: FractionalType) -> str: + return self.to_string(f"CAST({value} AS NUMERIC(38, {coltype.precision}))") + + def normalize_uuid(self, value: str, _coltype: ColType_UUID) -> str: + # Trim doesn't work on CHAR type + return f"TRIM(CAST({value} AS VARCHAR))" + + def normalize_boolean(self, value: str, _coltype: Boolean) -> str: + return self.to_string(f"cast ({value} as int)") + + +class Mixin_Schema(AbstractMixin_Schema): + def table_information(self) -> Compilable: + return table("v_catalog", "tables") + + def list_tables(self, table_schema: str, like: Compilable = None) -> Compilable: + return ( + self.table_information() + .where( + this.table_schema == table_schema, + this.table_name.like(like) if like is not None else SKIP, + ) + .select(this.table_name) + ) + + +class Dialect(BaseDialect, Mixin_Schema): + name = "Vertica" + ROUNDS_ON_PREC_LOSS = True + + TYPE_CLASSES = { + # Timestamps + "timestamp": Timestamp, + "timestamptz": TimestampTZ, + # Numbers + "numeric": Decimal, + "int": Integer, + "float": Float, + # Text + "char": Text, + "varchar": Text, + # Boolean + "boolean": Boolean, + } + + def quote(self, s: str): + return f'"{s}"' + + def concat(self, items: List[str]) -> str: + return " || ".join(items) + + def to_string(self, s: str) -> str: + return f"CAST({s} AS VARCHAR)" + + def is_distinct_from(self, a: str, b: str) -> str: + return f"not ({a} <=> {b})" + + def parse_type( + self, + table_path: DbPath, + col_name: str, + type_repr: str, + datetime_precision: int = None, + numeric_precision: int = None, + numeric_scale: int = None, + ) -> ColType: + timestamp_regexps = { + r"timestamp\(?(\d?)\)?": Timestamp, + r"timestamptz\(?(\d?)\)?": TimestampTZ, + } + for m, t_cls in match_regexps(timestamp_regexps, type_repr): + precision = int(m.group(1)) if m.group(1) else 6 + return t_cls(precision=precision, rounds=self.ROUNDS_ON_PREC_LOSS) + + number_regexps = { + r"numeric\((\d+),(\d+)\)": Decimal, + } + for m, n_cls in match_regexps(number_regexps, type_repr): + _prec, scale = map(int, m.groups()) + return n_cls(scale) + + string_regexps = { + r"varchar\((\d+)\)": Text, + r"char\((\d+)\)": Text, + } + for m, n_cls in match_regexps(string_regexps, type_repr): + return n_cls() + + return super().parse_type(table_path, col_name, type_repr, datetime_precision, numeric_precision) + + def set_timezone_to_utc(self) -> str: + return "SET TIME ZONE TO 'UTC'" + + def current_timestamp(self) -> str: + return "current_timestamp(6)" + + +class Vertica(ThreadedDatabase): + dialect = Dialect() + CONNECT_URI_HELP = "vertica://:@/" + CONNECT_URI_PARAMS = ["database?"] + + default_schema = "public" + + def __init__(self, *, thread_count, **kw): + self._args = kw + self._args["AUTOCOMMIT"] = False + + super().__init__(thread_count=thread_count) + + def create_connection(self): + vertica = import_vertica() + try: + c = vertica.connect(**self._args) + return c + except vertica.errors.ConnectionError as e: + raise ConnectError(*e.args) from e + + def select_table_schema(self, path: DbPath) -> str: + schema, name = self._normalize_table_path(path) + + return ( + "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale " + "FROM V_CATALOG.COLUMNS " + f"WHERE table_name = '{name}' AND table_schema = '{schema}'" + ) diff --git a/sqeleton/queries/__init__.py b/sqeleton/queries/__init__.py new file mode 100644 index 0000000..1f3d7ed --- /dev/null +++ b/sqeleton/queries/__init__.py @@ -0,0 +1,24 @@ +from .compiler import Compiler +from .api import ( + this, + join, + outerjoin, + table, + SKIP, + sum_, + avg, + min_, + max_, + cte, + commit, + when, + coalesce, + and_, + if_, + or_, + leftjoin, + rightjoin, + current_timestamp, +) +from .ast_classes import Expr, ExprNode, Select, Count, BinOp, Explain, In, Code, Column +from .extras import Checksum, NormalizeAsString, ApplyFuncAndNormalizeAsString diff --git a/sqeleton/queries/api.py b/sqeleton/queries/api.py new file mode 100644 index 0000000..cfdd09e --- /dev/null +++ b/sqeleton/queries/api.py @@ -0,0 +1,102 @@ +from typing import Optional + +from ..utils import CaseAwareMapping, CaseSensitiveDict +from .ast_classes import * +from .base import args_as_tuple + + +this = This() + + +def join(*tables: ITable): + "Joins each table into a 'struct'" + return Join(tables) + + +def leftjoin(*tables: ITable): + "Left-joins each table into a 'struct'" + return Join(tables, "LEFT") + + +def rightjoin(*tables: ITable): + "Right-joins each table into a 'struct'" + return Join(tables, "RIGHT") + + +def outerjoin(*tables: ITable): + "Outer-joins each table into a 'struct'" + return Join(tables, "FULL OUTER") + + +def cte(expr: Expr, *, name: Optional[str] = None, params: Sequence[str] = None): + return Cte(expr, name, params) + + +def table(*path: str, schema: Union[dict, CaseAwareMapping] = None) -> TablePath: + if len(path) == 1 and isinstance(path[0], tuple): + (path,) = path + if not all(isinstance(i, str) for i in path): + raise TypeError(f"All elements of table path must be of type 'str'. Got: {path}") + if schema and not isinstance(schema, CaseAwareMapping): + assert isinstance(schema, dict) + schema = CaseSensitiveDict(schema) + return TablePath(path, schema) + + +def or_(*exprs: Expr): + exprs = args_as_tuple(exprs) + if len(exprs) == 1: + return exprs[0] + return BinBoolOp("OR", exprs) + + +def and_(*exprs: Expr): + exprs = args_as_tuple(exprs) + if len(exprs) == 1: + return exprs[0] + return BinBoolOp("AND", exprs) + + +def sum_(expr: Expr): + return Func("sum", [expr]) + + +def avg(expr: Expr): + return Func("avg", [expr]) + + +def min_(expr: Expr): + return Func("min", [expr]) + + +def max_(expr: Expr): + return Func("max", [expr]) + + +def if_(cond: Expr, then: Expr, else_: Optional[Expr] = None): + return when(cond).then(then).else_(else_) + + +def when(*when_exprs: Expr): + return CaseWhen([]).when(*when_exprs) + + +def coalesce(*exprs): + exprs = args_as_tuple(exprs) + return Func("COALESCE", exprs) + + +def insert_rows_in_batches(db, tbl: TablePath, rows, *, columns=None, batch_size=1024 * 8): + assert batch_size > 0 + rows = list(rows) + + while rows: + batch, rows = rows[:batch_size], rows[batch_size:] + db.query(tbl.insert_rows(batch, columns=columns)) + + +def current_timestamp(): + return CurrentTimestamp() + + +commit = Commit() diff --git a/sqeleton/queries/ast_classes.py b/sqeleton/queries/ast_classes.py new file mode 100644 index 0000000..51f9369 --- /dev/null +++ b/sqeleton/queries/ast_classes.py @@ -0,0 +1,874 @@ +from dataclasses import field +from datetime import datetime +from typing import Any, Generator, List, Optional, Sequence, Union + +from runtype import dataclass + +from ..utils import join_iter, ArithString +from ..abcs import Compilable +from ..schema import Schema + +from .compiler import Compiler, cv_params, Root +from .base import SKIP, CompileError, DbPath, args_as_tuple + + +class SqeletonError(Exception): + pass + + +class QueryBuilderError(SqeletonError): + pass + + +class QB_TypeError(QueryBuilderError): + pass + + +class ExprNode(Compilable): + type: Any = None + + def _dfs_values(self): + yield self + for k, vs in dict(self).items(): # __dict__ provided by runtype.dataclass + if k == "source_table": + # Skip data-sources, we're only interested in data-parameters + continue + if not isinstance(vs, (list, tuple)): + vs = [vs] + for v in vs: + if isinstance(v, ExprNode): + yield from v._dfs_values() + + def cast_to(self, to): + return Cast(self, to) + + +Expr = Union[ExprNode, str, bool, int, datetime, ArithString, None] + + +@dataclass +class Code(ExprNode, Root): + code: str + + def compile(self, c: Compiler) -> str: + return self.code + + +def _expr_type(e: Expr) -> type: + if isinstance(e, ExprNode): + return e.type + return type(e) + + +@dataclass +class Alias(ExprNode): + expr: Expr + name: str + + def compile(self, c: Compiler) -> str: + return f"{c.compile(self.expr)} AS {c.quote(self.name)}" + + @property + def type(self): + return _expr_type(self.expr) + + +def _drop_skips(exprs): + return [e for e in exprs if e is not SKIP] + + +def _drop_skips_dict(exprs_dict): + return {k: v for k, v in exprs_dict.items() if v is not SKIP} + + +class ITable: + source_table: Any + schema: Schema = None + + def select(self, *exprs, distinct=SKIP, **named_exprs): + exprs = args_as_tuple(exprs) + exprs = _drop_skips(exprs) + named_exprs = _drop_skips_dict(named_exprs) + exprs += _named_exprs_as_aliases(named_exprs) + resolve_names(self.source_table, exprs) + return Select.make(self, columns=exprs, distinct=distinct) + + def where(self, *exprs): + exprs = args_as_tuple(exprs) + exprs = _drop_skips(exprs) + if not exprs: + return self + + resolve_names(self.source_table, exprs) + return Select.make(self, where_exprs=exprs) + + def order_by(self, *exprs): + exprs = _drop_skips(exprs) + if not exprs: + return self + + resolve_names(self.source_table, exprs) + return Select.make(self, order_by_exprs=exprs) + + def limit(self, limit: int): + if limit is SKIP: + return self + + return Select.make(self, limit_expr=limit) + + def at(self, *exprs): + # TODO + exprs = _drop_skips(exprs) + if not exprs: + return self + + raise NotImplementedError() + + def join(self, target): + return Join(self, target) + + def group_by(self, *, keys=None, values=None): + keys = _drop_skips(keys) + resolve_names(self.source_table, keys) + + values = _drop_skips(values) + resolve_names(self.source_table, values) + + return GroupBy(self, keys, values) + + def with_schema(self): + # TODO + raise NotImplementedError() + + def _get_column(self, name: str): + if self.schema: + name = self.schema.get_key(name) # Get the actual name. Might be case-insensitive. + return Column(self, name) + + # def __getattr__(self, column): + # return self._get_column(column) + + def __getitem__(self, column): + if not isinstance(column, str): + raise TypeError() + return self._get_column(column) + + def count(self): + return Select(self, [Count()]) + + def union(self, other: "ITable"): + return TableOp("UNION", self, other) + + def union_all(self, other: "ITable"): + return TableOp("UNION ALL", self, other) + + def minus(self, other: "ITable"): + # aka + return TableOp("EXCEPT", self, other) + + def intersect(self, other: "ITable"): + return TableOp("INTERSECT", self, other) + + +@dataclass +class Concat(ExprNode): + exprs: list + sep: str = None + + def compile(self, c: Compiler) -> str: + # We coalesce because on some DBs (e.g. MySQL) concat('a', NULL) is NULL + items = [f"coalesce({c.compile(Code(c.dialect.to_string(c.compile(expr))))}, '')" for expr in self.exprs] + assert items + if len(items) == 1: + return items[0] + + if self.sep: + items = list(join_iter(f"'{self.sep}'", items)) + return c.dialect.concat(items) + + +@dataclass +class Count(ExprNode): + expr: Expr = None + distinct: bool = False + + type = int + + def compile(self, c: Compiler) -> str: + expr = c.compile(self.expr) if self.expr else "*" + if self.distinct: + return f"count(distinct {expr})" + + return f"count({expr})" + + +class LazyOps: + def __add__(self, other): + return BinOp("+", [self, other]) + + def __sub__(self, other): + return BinOp("-", [self, other]) + + def __neg__(self): + return UnaryOp("-", self) + + def __gt__(self, other): + return BinBoolOp(">", [self, other]) + + def __ge__(self, other): + return BinBoolOp(">=", [self, other]) + + def __eq__(self, other): + if other is None: + return BinBoolOp("IS", [self, None]) + return BinBoolOp("=", [self, other]) + + def __lt__(self, other): + return BinBoolOp("<", [self, other]) + + def __le__(self, other): + return BinBoolOp("<=", [self, other]) + + def __or__(self, other): + return BinBoolOp("OR", [self, other]) + + def __and__(self, other): + return BinBoolOp("AND", [self, other]) + + def is_distinct_from(self, other): + return IsDistinctFrom(self, other) + + def like(self, other): + return BinBoolOp("LIKE", [self, other]) + + def sum(self): + return Func("SUM", [self]) + + +@dataclass +class Func(ExprNode, LazyOps): + name: str + args: Sequence[Expr] + + def compile(self, c: Compiler) -> str: + args = ", ".join(c.compile(e) for e in self.args) + return f"{self.name}({args})" + + +@dataclass +class WhenThen(ExprNode): + when: Expr + then: Expr + + def compile(self, c: Compiler) -> str: + return f"WHEN {c.compile(self.when)} THEN {c.compile(self.then)}" + + +@dataclass +class CaseWhen(ExprNode): + cases: Sequence[WhenThen] + else_expr: Expr = None + + def compile(self, c: Compiler) -> str: + assert self.cases + when_thens = " ".join(c.compile(case) for case in self.cases) + else_expr = (" ELSE " + c.compile(self.else_expr)) if self.else_expr is not None else "" + return f"CASE {when_thens}{else_expr} END" + + @property + def type(self): + then_types = {_expr_type(case.then) for case in self.cases} + if self.else_expr: + then_types |= _expr_type(self.else_expr) + if len(then_types) > 1: + raise QB_TypeError(f"Non-matching types in when: {then_types}") + (t,) = then_types + return t + + def when(self, *whens: Expr) -> "QB_When": + whens = args_as_tuple(whens) + whens = _drop_skips(whens) + if not whens: + raise QueryBuilderError("Expected valid whens") + + # XXX reimplementing api.and_() + if len(whens) == 1: + return QB_When(self, whens[0]) + return QB_When(self, BinBoolOp("AND", whens)) + + def else_(self, then: Expr): + if self.else_expr is not None: + raise QueryBuilderError(f"Else clause already specified in {self}") + + return self.replace(else_expr=then) + + +@dataclass +class QB_When: + "Partial case-when, used for query-building" + casewhen: CaseWhen + when: Expr + + def then(self, then: Expr) -> CaseWhen: + case = WhenThen(self.when, then) + return self.casewhen.replace(cases=self.casewhen.cases + [case]) + + +@dataclass(eq=False, order=False) +class IsDistinctFrom(ExprNode, LazyOps): + a: Expr + b: Expr + type = bool + + def compile(self, c: Compiler) -> str: + return c.dialect.is_distinct_from(c.compile(self.a), c.compile(self.b)) + + +@dataclass(eq=False, order=False) +class BinOp(ExprNode, LazyOps): + op: str + args: Sequence[Expr] + + def compile(self, c: Compiler) -> str: + expr = f" {self.op} ".join(c.compile(a) for a in self.args) + return f"({expr})" + + @property + def type(self): + types = {_expr_type(i) for i in self.args} + if len(types) > 1: + raise TypeError(f"Expected all args to have the same type, got {types}") + (t,) = types + return t + + +@dataclass +class UnaryOp(ExprNode, LazyOps): + op: str + expr: Expr + + def compile(self, c: Compiler) -> str: + return f"({self.op}{c.compile(self.expr)})" + + +class BinBoolOp(BinOp): + type = bool + + +@dataclass(eq=False, order=False) +class Column(ExprNode, LazyOps): + source_table: ITable + name: str + + @property + def type(self): + if self.source_table.schema is None: + raise QueryBuilderError(f"Schema required for table {self.source_table}") + return self.source_table.schema[self.name] + + def compile(self, c: Compiler) -> str: + if c._table_context: + if len(c._table_context) > 1: + aliases = [ + t for t in c._table_context if isinstance(t, TableAlias) and t.source_table is self.source_table + ] + if not aliases: + return c.quote(self.name) + elif len(aliases) > 1: + raise CompileError(f"Too many aliases for column {self.name}") + (alias,) = aliases + + return f"{c.quote(alias.name)}.{c.quote(self.name)}" + + return c.quote(self.name) + + +@dataclass +class TablePath(ExprNode, ITable): + path: DbPath + schema: Optional[Schema] = field(default=None, repr=False) + + @property + def source_table(self): + return self + + def compile(self, c: Compiler) -> str: + path = self.path # c.database._normalize_table_path(self.name) + return ".".join(map(c.quote, path)) + + # Statement shorthands + def create(self, source_table: ITable = None, *, if_not_exists=False, primary_keys=None): + + if source_table is None and not self.schema: + raise ValueError("Either schema or source table needed to create table") + if isinstance(source_table, TablePath): + source_table = source_table.select() + return CreateTable(self, source_table, if_not_exists=if_not_exists, primary_keys=primary_keys) + + def drop(self, if_exists=False): + return DropTable(self, if_exists=if_exists) + + def truncate(self): + return TruncateTable(self) + + def insert_rows(self, rows, *, columns=None): + rows = list(rows) + return InsertToTable(self, ConstantTable(rows), columns=columns) + + def insert_row(self, *values, columns=None): + return InsertToTable(self, ConstantTable([values]), columns=columns) + + def insert_expr(self, expr: Expr): + if isinstance(expr, TablePath): + expr = expr.select() + return InsertToTable(self, expr) + + +@dataclass +class TableAlias(ExprNode, ITable): + source_table: ITable + name: str + + def compile(self, c: Compiler) -> str: + return f"{c.compile(self.source_table)} {c.quote(self.name)}" + + +@dataclass +class Join(ExprNode, ITable, Root): + source_tables: Sequence[ITable] + op: str = None + on_exprs: Sequence[Expr] = None + columns: Sequence[Expr] = None + + @property + def source_table(self): + return self # TODO is this right? + + @property + def schema(self): + assert self.columns # TODO Implement SELECT * + s = self.source_tables[0].schema # XXX + return type(s)({c.name: c.type for c in self.columns}) + + def on(self, *exprs): + if len(exprs) == 1: + (e,) = exprs + if isinstance(e, Generator): + exprs = tuple(e) + + exprs = _drop_skips(exprs) + if not exprs: + return self + + return self.replace(on_exprs=(self.on_exprs or []) + exprs) + + def select(self, *exprs, **named_exprs): + if self.columns is not None: + # join-select already applied + return super().select(*exprs, **named_exprs) + + exprs = _drop_skips(exprs) + named_exprs = _drop_skips_dict(named_exprs) + exprs += _named_exprs_as_aliases(named_exprs) + resolve_names(self.source_table, exprs) + # TODO Ensure exprs <= self.columns ? + return self.replace(columns=exprs) + + def compile(self, parent_c: Compiler) -> str: + tables = [ + t if isinstance(t, TableAlias) else TableAlias(t, parent_c.new_unique_name()) for t in self.source_tables + ] + c = parent_c.add_table_context(*tables, in_join=True, in_select=False) + op = " JOIN " if self.op is None else f" {self.op} JOIN " + joined = op.join(c.compile(t) for t in tables) + + if self.on_exprs: + on = " AND ".join(c.compile(e) for e in self.on_exprs) + res = f"{joined} ON {on}" + else: + res = joined + + columns = "*" if self.columns is None else ", ".join(map(c.compile, self.columns)) + select = f"SELECT {columns} FROM {res}" + + if parent_c.in_select: + select = f"({select}) {c.new_unique_name()}" + elif parent_c.in_join: + select = f"({select})" + return select + + +@dataclass +class GroupBy(ExprNode, ITable, Root): + table: ITable + keys: Sequence[Expr] = None # IKey? + values: Sequence[Expr] = None + having_exprs: Sequence[Expr] = None + + def __post_init__(self): + assert self.keys or self.values + + def having(self, *exprs): + exprs = args_as_tuple(exprs) + exprs = _drop_skips(exprs) + if not exprs: + return self + + resolve_names(self.table, exprs) + return self.replace(having_exprs=(self.having_exprs or []) + exprs) + + def compile(self, c: Compiler) -> str: + keys = [str(i + 1) for i in range(len(self.keys))] + columns = (self.keys or []) + (self.values or []) + if isinstance(self.table, Select) and self.table.columns is None and self.table.group_by_exprs is None: + return c.compile( + self.table.replace( + columns=columns, + group_by_exprs=[Code(k) for k in keys], + having_exprs=self.having_exprs, + ) + ) + + keys_str = ", ".join(keys) + columns_str = ", ".join(c.compile(x) for x in columns) + having_str = ( + " HAVING " + " AND ".join(map(c.compile, self.having_exprs)) if self.having_exprs is not None else "" + ) + return ( + f"SELECT {columns_str} FROM {c.replace(in_select=True).compile(self.table)} GROUP BY {keys_str}{having_str}" + ) + + +@dataclass +class TableOp(ExprNode, ITable, Root): + op: str + table1: ITable + table2: ITable + + @property + def source_table(self): + return self # TODO is this right? + + @property + def type(self): + return self.table1.type + + @property + def schema(self): + s1 = self.table1.schema + s2 = self.table2.schema + assert len(s1) == len(s2) + return s1 + + def compile(self, parent_c: Compiler) -> str: + c = parent_c.replace(in_select=False) + table_expr = f"{c.compile(self.table1)} {self.op} {c.compile(self.table2)}" + if parent_c.in_select: + table_expr = f"({table_expr}) {c.new_unique_name()}" + elif parent_c.in_join: + table_expr = f"({table_expr})" + return table_expr + + +@dataclass +class Select(ExprNode, ITable, Root): + table: Expr = None + columns: Sequence[Expr] = None + where_exprs: Sequence[Expr] = None + order_by_exprs: Sequence[Expr] = None + group_by_exprs: Sequence[Expr] = None + having_exprs: Sequence[Expr] = None + limit_expr: int = None + distinct: bool = False + + @property + def schema(self): + s = self.table.schema + if s is None or self.columns is None: + return s + return type(s)({c.name: c.type for c in self.columns}) + + @property + def source_table(self): + return self + + def compile(self, parent_c: Compiler) -> str: + c = parent_c.replace(in_select=True) # .add_table_context(self.table) + + columns = ", ".join(map(c.compile, self.columns)) if self.columns else "*" + distinct = "DISTINCT " if self.distinct else "" + select = f"SELECT {distinct}{columns}" + + if self.table: + select += " FROM " + c.compile(self.table) + elif c.dialect.PLACEHOLDER_TABLE: + select += f" FROM {c.dialect.PLACEHOLDER_TABLE}" + + if self.where_exprs: + select += " WHERE " + " AND ".join(map(c.compile, self.where_exprs)) + + if self.group_by_exprs: + select += " GROUP BY " + ", ".join(map(c.compile, self.group_by_exprs)) + + if self.having_exprs: + assert self.group_by_exprs + select += " HAVING " + " AND ".join(map(c.compile, self.having_exprs)) + + if self.order_by_exprs: + select += " ORDER BY " + ", ".join(map(c.compile, self.order_by_exprs)) + + if self.limit_expr is not None: + select += " " + c.dialect.offset_limit(0, self.limit_expr) + + if parent_c.in_select: + select = f"({select}) {c.new_unique_name()}" + elif parent_c.in_join: + select = f"({select})" + return select + + @classmethod + def make(cls, table: ITable, distinct: bool = SKIP, **kwargs): + assert "table" not in kwargs + + if not isinstance(table, cls): # If not Select + if distinct is not SKIP: + kwargs["distinct"] = distinct + return cls(table, **kwargs) + + # We can safely assume isinstance(table, Select) + + if distinct is not SKIP: + if distinct == False and table.distinct: + return cls(table, **kwargs) + kwargs["distinct"] = distinct + + if table.limit_expr or table.group_by_exprs: + return cls(table, **kwargs) + + # Fill in missing attributes, instead of creating a new instance. + for k, v in kwargs.items(): + if getattr(table, k) is not None: + if k == "where_exprs": # Additive attribute + kwargs[k] = getattr(table, k) + v + elif k == "distinct": + pass + else: + raise ValueError(k) + + return table.replace(**kwargs) + + +@dataclass +class Cte(ExprNode, ITable): + source_table: Expr + name: str = None + params: Sequence[str] = None + + def compile(self, parent_c: Compiler) -> str: + c = parent_c.replace(_table_context=[], in_select=False) + compiled = c.compile(self.source_table) + + name = self.name or parent_c.new_unique_name() + name_params = f"{name}({', '.join(self.params)})" if self.params else name + parent_c._subqueries[name_params] = compiled + + return name + + @property + def schema(self): + # TODO add cte to schema + return self.source_table.schema + + +def _named_exprs_as_aliases(named_exprs): + return [Alias(expr, name) for name, expr in named_exprs.items()] + + +def resolve_names(source_table, exprs): + i = 0 + for expr in exprs: + # Iterate recursively and update _ResolveColumn instances with the right expression + if isinstance(expr, ExprNode): + for v in expr._dfs_values(): + if isinstance(v, _ResolveColumn): + v.resolve(source_table._get_column(v.resolve_name)) + i += 1 + + +@dataclass(frozen=False, eq=False, order=False) +class _ResolveColumn(ExprNode, LazyOps): + resolve_name: str + resolved: Expr = None + + def resolve(self, expr: Expr): + if self.resolved is not None: + raise QueryBuilderError("Already resolved!") + self.resolved = expr + + def _get_resolved(self) -> Expr: + if self.resolved is None: + raise QueryBuilderError(f"Column not resolved: {self.resolve_name}") + return self.resolved + + def compile(self, c: Compiler) -> str: + return self._get_resolved().compile(c) + + @property + def type(self): + return self._get_resolved().type + + @property + def name(self): + return self._get_resolved().name + + +class This: + def __getattr__(self, name): + return _ResolveColumn(name) + + def __getitem__(self, name): + if isinstance(name, (list, tuple)): + return [_ResolveColumn(n) for n in name] + return _ResolveColumn(name) + + +@dataclass +class In(ExprNode): + expr: Expr + list: Sequence[Expr] + + type = bool + + def compile(self, c: Compiler): + elems = ", ".join(map(c.compile, self.list)) + return f"({c.compile(self.expr)} IN ({elems}))" + + +@dataclass +class Cast(ExprNode): + expr: Expr + target_type: Expr + + def compile(self, c: Compiler) -> str: + return f"cast({c.compile(self.expr)} as {c.compile(self.target_type)})" + + +@dataclass +class Random(ExprNode): + type = float + + def compile(self, c: Compiler) -> str: + return c.dialect.random() + + +@dataclass +class ConstantTable(ExprNode): + rows: Sequence[Sequence] + + def compile(self, c: Compiler) -> str: + raise NotImplementedError() + + def compile_for_insert(self, c: Compiler): + return c.dialect.constant_values(self.rows) + + +@dataclass +class Explain(ExprNode, Root): + select: Select + + type = str + + def compile(self, c: Compiler) -> str: + return c.dialect.explain_as_text(c.compile(self.select)) + + +class CurrentTimestamp(ExprNode): + type = datetime + + def compile(self, c: Compiler) -> str: + return c.dialect.current_timestamp() + + +# DDL + + +class Statement(Compilable, Root): + type = None + + +@dataclass +class CreateTable(Statement): + path: TablePath + source_table: Expr = None + if_not_exists: bool = False + primary_keys: List[str] = None + + def compile(self, c: Compiler) -> str: + ne = "IF NOT EXISTS " if self.if_not_exists else "" + if self.source_table: + return f"CREATE TABLE {ne}{c.compile(self.path)} AS {c.compile(self.source_table)}" + + schema = ", ".join(f"{c.dialect.quote(k)} {c.dialect.type_repr(v)}" for k, v in self.path.schema.items()) + pks = ( + ", PRIMARY KEY (%s)" % ", ".join(self.primary_keys) + if self.primary_keys and c.dialect.SUPPORTS_PRIMARY_KEY + else "" + ) + return f"CREATE TABLE {ne}{c.compile(self.path)}({schema}{pks})" + + +@dataclass +class DropTable(Statement): + path: TablePath + if_exists: bool = False + + def compile(self, c: Compiler) -> str: + ie = "IF EXISTS " if self.if_exists else "" + return f"DROP TABLE {ie}{c.compile(self.path)}" + + +@dataclass +class TruncateTable(Statement): + path: TablePath + + def compile(self, c: Compiler) -> str: + return f"TRUNCATE TABLE {c.compile(self.path)}" + + +@dataclass +class InsertToTable(Statement): + # TODO Support insert for only some columns + path: TablePath + expr: Expr + columns: List[str] = None + + def compile(self, c: Compiler) -> str: + if isinstance(self.expr, ConstantTable): + expr = self.expr.compile_for_insert(c) + else: + expr = c.compile(self.expr) + + columns = "(%s)" % ", ".join(map(c.quote, self.columns)) if self.columns is not None else "" + + return f"INSERT INTO {c.compile(self.path)}{columns} {expr}" + + +@dataclass +class Commit(Statement): + def compile(self, c: Compiler) -> str: + return "COMMIT" if not c.database.is_autocommit else SKIP + + +@dataclass +class Param(ExprNode, ITable): + """A value placeholder, to be specified at compilation time using the `cv_params` context variable.""" + + name: str + + @property + def source_table(self): + return self + + def compile(self, c: Compiler) -> str: + params = cv_params.get() + return c._compile(params[self.name]) diff --git a/sqeleton/queries/base.py b/sqeleton/queries/base.py new file mode 100644 index 0000000..006b4c7 --- /dev/null +++ b/sqeleton/queries/base.py @@ -0,0 +1,24 @@ +from typing import Generator + +from ..abcs import DbPath, DbKey +from ..schema import Schema + + +class _SKIP: + def __repr__(self): + return "SKIP" + + +SKIP = _SKIP() + + +class CompileError(Exception): + pass + + +def args_as_tuple(exprs): + if len(exprs) == 1: + (e,) = exprs + if isinstance(e, Generator): + return tuple(e) + return exprs diff --git a/sqeleton/queries/compiler.py b/sqeleton/queries/compiler.py new file mode 100644 index 0000000..977e871 --- /dev/null +++ b/sqeleton/queries/compiler.py @@ -0,0 +1,81 @@ +import random +from datetime import datetime +from typing import Any, Dict, Sequence, List + +from runtype import dataclass + +from ..utils import ArithString +from ..abcs import AbstractDatabase, AbstractDialect, DbPath, AbstractCompiler, Compilable + +import contextvars + +cv_params = contextvars.ContextVar("params") + + +class Root: + "Nodes inheriting from Root can be used as root statements in SQL (e.g. SELECT yes, RANDOM() no)" + + +@dataclass +class Compiler(AbstractCompiler): + database: AbstractDatabase + params: dict = {} + in_select: bool = False # Compilation runtime flag + in_join: bool = False # Compilation runtime flag + + _table_context: List = [] # List[ITable] + _subqueries: Dict[str, Any] = {} # XXX not thread-safe + root: bool = True + + _counter: List = [0] + + @property + def dialect(self) -> AbstractDialect: + return self.database.dialect + + def compile(self, elem, params=None) -> str: + if params: + cv_params.set(params) + + if self.root and isinstance(elem, Compilable) and not isinstance(elem, Root): + from .ast_classes import Select + + elem = Select(columns=[elem]) + + res = self._compile(elem) + if self.root and self._subqueries: + subq = ", ".join(f"\n {k} AS ({v})" for k, v in self._subqueries.items()) + self._subqueries.clear() + return f"WITH {subq}\n{res}" + return res + + def _compile(self, elem) -> str: + if elem is None: + return "NULL" + elif isinstance(elem, Compilable): + return elem.compile(self.replace(root=False)) + elif isinstance(elem, str): + return f"'{elem}'" + elif isinstance(elem, int): + return str(elem) + elif isinstance(elem, datetime): + return self.dialect.timestamp_value(elem) + elif isinstance(elem, bytes): + return f"b'{elem.decode()}'" + elif isinstance(elem, ArithString): + return f"'{elem}'" + assert False, elem + + def new_unique_name(self, prefix="tmp"): + self._counter[0] += 1 + return f"{prefix}{self._counter[0]}" + + def new_unique_table_name(self, prefix="tmp") -> DbPath: + self._counter[0] += 1 + return self.database.parse_table_name(f"{prefix}{self._counter[0]}_{'%x'%random.randrange(2**32)}") + + def add_table_context(self, *tables: Sequence, **kw): + return self.replace(_table_context=self._table_context + list(tables), **kw) + + def quote(self, s: str): + return self.dialect.quote(s) diff --git a/sqeleton/queries/extras.py b/sqeleton/queries/extras.py new file mode 100644 index 0000000..1014c37 --- /dev/null +++ b/sqeleton/queries/extras.py @@ -0,0 +1,62 @@ +"Useful AST classes that don't quite fall within the scope of regular SQL" + +from typing import Callable, Sequence +from runtype import dataclass + +from ..abcs.database_types import ColType, Native_UUID + +from .compiler import Compiler +from .ast_classes import Expr, ExprNode, Concat, Code + + +@dataclass +class NormalizeAsString(ExprNode): + expr: ExprNode + expr_type: ColType = None + type = str + + def compile(self, c: Compiler) -> str: + expr = c.compile(self.expr) + return c.dialect.normalize_value_by_type(expr, self.expr_type or self.expr.type) + + +@dataclass +class ApplyFuncAndNormalizeAsString(ExprNode): + expr: ExprNode + apply_func: Callable = None + + def compile(self, c: Compiler) -> str: + expr = self.expr + expr_type = expr.type + + if isinstance(expr_type, Native_UUID): + # Normalize first, apply template after (for uuids) + # Needed because min/max(uuid) fails in postgresql + expr = NormalizeAsString(expr, expr_type) + if self.apply_func is not None: + expr = self.apply_func(expr) # Apply template using Python's string formatting + + else: + # Apply template before normalizing (for ints) + if self.apply_func is not None: + expr = self.apply_func(expr) # Apply template using Python's string formatting + expr = NormalizeAsString(expr, expr_type) + + return c.compile(expr) + + +@dataclass +class Checksum(ExprNode): + exprs: Sequence[Expr] + + def compile(self, c: Compiler): + if len(self.exprs) > 1: + exprs = [Code(f"coalesce({c.compile(expr)}, '')") for expr in self.exprs] + # exprs = [c.compile(e) for e in exprs] + expr = Concat(exprs, "|") + else: + # No need to coalesce - safe to assume that key cannot be null + (expr,) = self.exprs + expr = c.compile(expr) + md5 = c.dialect.md5_as_int(expr) + return f"sum({md5})" diff --git a/sqeleton/query_utils.py b/sqeleton/query_utils.py new file mode 100644 index 0000000..4eb0744 --- /dev/null +++ b/sqeleton/query_utils.py @@ -0,0 +1,54 @@ +"Module for query utilities that didn't make it into the query-builder (yet)" + +from contextlib import suppress + +from sqeleton.databases import DbPath, QueryError, Oracle +from sqeleton.queries import table, commit, Expr + + +def _drop_table_oracle(name: DbPath): + t = table(name) + # Experience shows double drop is necessary + with suppress(QueryError): + yield t.drop() + yield t.drop() + yield commit + + +def _drop_table(name: DbPath): + t = table(name) + yield t.drop(if_exists=True) + yield commit + + +def drop_table(db, tbl): + if isinstance(db, Oracle): + db.query(_drop_table_oracle(tbl)) + else: + db.query(_drop_table(tbl)) + + +def _append_to_table_oracle(path: DbPath, expr: Expr): + """See append_to_table""" + assert expr.schema, expr + t = table(path, schema=expr.schema) + with suppress(QueryError): + yield t.create() # uses expr.schema + yield commit + yield t.insert_expr(expr) + yield commit + + +def _append_to_table(path: DbPath, expr: Expr): + """Append to table""" + assert expr.schema, expr + t = table(path, schema=expr.schema) + yield t.create(if_not_exists=True) # uses expr.schema + yield commit + yield t.insert_expr(expr) + yield commit + + +def append_to_table(db, path, expr): + f = _append_to_table_oracle if isinstance(db, Oracle) else _append_to_table + db.query(f(path, expr)) diff --git a/sqeleton/repl.py b/sqeleton/repl.py new file mode 100644 index 0000000..9d51e59 --- /dev/null +++ b/sqeleton/repl.py @@ -0,0 +1,65 @@ +import rich.table +import logging + +# logging.basicConfig(level=logging.DEBUG) + +from . import connect +from .queries import table + +import sys + + +def print_table(rows, schema, table_name=""): + # Print rows in a rich table + t = rich.table.Table(title=table_name, caption=f"{len(rows)} rows") + for col in schema: + t.add_column(col) + for r in rows: + t.add_row(*map(str, r)) + rich.print(t) + + +def help(): + rich.print("Commands:") + rich.print(" ?mytable - shows schema of table 'mytable'") + rich.print(" * - shows list of all tables") + rich.print(" *pattern - shows list of all tables with name like pattern") + rich.print("Otherwise, runs regular SQL query") + + +def main(): + uri = sys.argv[1] + db = connect(uri) + db_name = db.name + + while True: + q = input(f"{db_name}> ").strip() + if not q: + continue + if q.startswith("*"): + pattern = q[1:] + names = db.query(db.dialect.list_tables(db.default_schema, like=f"%{pattern}%" if pattern else None)) + print_table(names, ["name"], "List of tables") + elif q.startswith("?"): + table_name = q[1:] + if not table_name: + help() + continue + try: + schema = db.query_table_schema((table_name,)) + except Exception as e: + logging.error(e) + else: + print_table([(k, v[1]) for k, v in schema.items()], ["name", "type"], f"Table '{table_name}'") + else: + try: + res = db.query(q) + except Exception as e: + logging.error(e) + else: + if res: + print_table(res, [str(i) for i in range(len(res[0]))], q) + + +if __name__ == "__main__": + main() diff --git a/sqeleton/schema.py b/sqeleton/schema.py new file mode 100644 index 0000000..ddf7e78 --- /dev/null +++ b/sqeleton/schema.py @@ -0,0 +1,20 @@ +import logging + +from .utils import CaseAwareMapping, CaseInsensitiveDict, CaseSensitiveDict +from .abcs import AbstractDatabase, DbPath + +logger = logging.getLogger("schema") + +Schema = CaseAwareMapping + + +def create_schema(db: AbstractDatabase, table_path: DbPath, schema: dict, case_sensitive: bool) -> CaseAwareMapping: + logger.debug(f"[{db.name}] Schema = {schema}") + + if case_sensitive: + return CaseSensitiveDict(schema) + + if len({k.lower() for k in schema}) < len(schema): + logger.warning(f'Ambiguous schema for {db}:{".".join(table_path)} | Columns = {", ".join(list(schema))}') + logger.warning("We recommend to disable case-insensitivity (set --case-sensitive).") + return CaseInsensitiveDict(schema) diff --git a/sqeleton/utils.py b/sqeleton/utils.py new file mode 100644 index 0000000..0502fd5 --- /dev/null +++ b/sqeleton/utils.py @@ -0,0 +1,263 @@ +from typing import Iterable, Iterator, MutableMapping, Union, Any, Sequence, Dict, Hashable, TypeVar +from abc import abstractmethod +from weakref import ref +import math +import string +import re +from uuid import UUID + +# -- Common -- + + +class WeakCache: + def __init__(self): + self._cache = {} + + def _hashable_key(self, k: Union[dict, Hashable]) -> Hashable: + if isinstance(k, dict): + return tuple(k.items()) + return k + + def add(self, key: Union[dict, Hashable], value: Any): + key = self._hashable_key(key) + self._cache[key] = ref(value) + + def get(self, key: Union[dict, Hashable]) -> Any: + key = self._hashable_key(key) + + value = self._cache[key]() + if value is None: + del self._cache[key] + raise KeyError(f"Key {key} not found, or no longer a valid reference") + + return value + + +def join_iter(joiner: Any, iterable: Iterable) -> Iterable: + it = iter(iterable) + try: + yield next(it) + except StopIteration: + return + for i in it: + yield joiner + yield i + + +def safezip(*args): + "zip but makes sure all sequences are the same length" + lens = list(map(len, args)) + if len(set(lens)) != 1: + raise ValueError(f"Mismatching lengths in arguments to safezip: {lens}") + return zip(*args) + + +def is_uuid(u): + try: + UUID(u) + except ValueError: + return False + return True + + +def match_regexps(regexps: Dict[str, Any], s: str) -> Sequence[tuple]: + for regexp, v in regexps.items(): + m = re.match(regexp + "$", s) + if m: + yield m, v + + +# -- Schema -- + +V = TypeVar("V") + + +class CaseAwareMapping(MutableMapping[str, V]): + @abstractmethod + def get_key(self, key: str) -> str: + ... + + +class CaseInsensitiveDict(CaseAwareMapping): + def __init__(self, initial): + self._dict = {k.lower(): (k, v) for k, v in dict(initial).items()} + + def __getitem__(self, key: str) -> V: + return self._dict[key.lower()][1] + + def __iter__(self) -> Iterator[V]: + return iter(self._dict) + + def __len__(self) -> int: + return len(self._dict) + + def __setitem__(self, key: str, value): + k = key.lower() + if k in self._dict: + key = self._dict[k][0] + self._dict[k] = key, value + + def __delitem__(self, key: str): + del self._dict[key.lower()] + + def get_key(self, key: str) -> str: + return self._dict[key.lower()][0] + + def __repr__(self) -> str: + return repr(dict(self.items())) + + +class CaseSensitiveDict(dict, CaseAwareMapping): + def get_key(self, key): + self[key] # Throw KeyError is key doesn't exist + return key + + def as_insensitive(self): + return CaseInsensitiveDict(self) + + +# -- Alphanumerics -- + +alphanums = " -" + string.digits + string.ascii_uppercase + "_" + string.ascii_lowercase + + +class ArithString: + @classmethod + def new(cls, *args, **kw): + return cls(*args, **kw) + + def range(self, other: "ArithString", count: int): + assert isinstance(other, ArithString) + checkpoints = split_space(self.int, other.int, count) + return [self.new(int=i) for i in checkpoints] + + +class ArithUUID(UUID, ArithString): + "A UUID that supports basic arithmetic (add, sub)" + + def __int__(self): + return self.int + + def __add__(self, other: int): + if isinstance(other, int): + return self.new(int=self.int + other) + return NotImplemented + + def __sub__(self, other: Union[UUID, int]): + if isinstance(other, int): + return self.new(int=self.int - other) + elif isinstance(other, UUID): + return self.int - other.int + return NotImplemented + + +def numberToAlphanum(num: int, base: str = alphanums) -> str: + digits = [] + while num > 0: + num, remainder = divmod(num, len(base)) + digits.append(remainder) + return "".join(base[i] for i in digits[::-1]) + + +def alphanumToNumber(alphanum: str, base: str = alphanums) -> int: + num = 0 + for c in alphanum: + num = num * len(base) + base.index(c) + return num + + +def justify_alphanums(s1: str, s2: str): + max_len = max(len(s1), len(s2)) + s1 = s1.ljust(max_len) + s2 = s2.ljust(max_len) + return s1, s2 + + +def alphanums_to_numbers(s1: str, s2: str): + s1, s2 = justify_alphanums(s1, s2) + n1 = alphanumToNumber(s1) + n2 = alphanumToNumber(s2) + return n1, n2 + + +class ArithAlphanumeric(ArithString): + def __init__(self, s: str, max_len=None): + if s is None: + raise ValueError("Alphanum string cannot be None") + if max_len and len(s) > max_len: + raise ValueError(f"Length of alphanum value '{str}' is longer than the expected {max_len}") + + for ch in s: + if ch not in alphanums: + raise ValueError(f"Unexpected character {ch} in alphanum string") + + self._str = s + self._max_len = max_len + + # @property + # def int(self): + # return alphanumToNumber(self._str, alphanums) + + def __str__(self): + s = self._str + if self._max_len: + s = s.rjust(self._max_len, alphanums[0]) + return s + + def __len__(self): + return len(self._str) + + def __repr__(self): + return f'alphanum"{self._str}"' + + def __add__(self, other: "Union[ArithAlphanumeric, int]") -> "ArithAlphanumeric": + if isinstance(other, int): + if other != 1: + raise NotImplementedError("not implemented for arbitrary numbers") + num = alphanumToNumber(self._str) + return self.new(numberToAlphanum(num + 1)) + + return NotImplemented + + def range(self, other: "ArithAlphanumeric", count: int): + assert isinstance(other, ArithAlphanumeric) + n1, n2 = alphanums_to_numbers(self._str, other._str) + split = split_space(n1, n2, count) + return [self.new(numberToAlphanum(s)) for s in split] + + def __sub__(self, other: "Union[ArithAlphanumeric, int]") -> float: + if isinstance(other, ArithAlphanumeric): + n1, n2 = alphanums_to_numbers(self._str, other._str) + return n1 - n2 + + return NotImplemented + + def __ge__(self, other): + if not isinstance(other, type(self)): + return NotImplemented + return self._str >= other._str + + def __lt__(self, other): + if not isinstance(other, type(self)): + return NotImplemented + return self._str < other._str + + def new(self, *args, **kw): + return type(self)(*args, **kw, max_len=self._max_len) + + +def number_to_human(n): + millnames = ["", "k", "m", "b"] + n = float(n) + millidx = max( + 0, + min(len(millnames) - 1, int(math.floor(0 if n == 0 else math.log10(abs(n)) / 3))), + ) + + return "{:.0f}{}".format(n / 10 ** (3 * millidx), millnames[millidx]) + + +def split_space(start, end, count): + size = end - start + assert count <= size, (count, size) + return list(range(start, end, (size + 1) // (count + 1)))[1 : count + 1] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/common.py b/tests/common.py new file mode 100644 index 0000000..294281a --- /dev/null +++ b/tests/common.py @@ -0,0 +1,165 @@ +import hashlib +import os +import string +import random +from typing import Callable +import unittest +import logging +import subprocess + +from parameterized import parameterized_class + +from sqeleton import databases as db +from sqeleton import connect +from sqeleton.queries import table +from sqeleton.databases import Database +from sqeleton.query_utils import drop_table + + +# We write 'or None' because Github sometimes creates empty env vars for secrets +TEST_MYSQL_CONN_STRING: str = "mysql://mysql:Password1@localhost/mysql" +TEST_POSTGRESQL_CONN_STRING: str = "postgresql://postgres:Password1@localhost/postgres" +TEST_SNOWFLAKE_CONN_STRING: str = os.environ.get("SNOWFLAKE_URI") or None +TEST_PRESTO_CONN_STRING: str = os.environ.get("PRESTO_URI") or None +TEST_BIGQUERY_CONN_STRING: str = os.environ.get("BIGQUERY_URI") or None +TEST_REDSHIFT_CONN_STRING: str = os.environ.get("REDSHIFT_URI") or None +TEST_ORACLE_CONN_STRING: str = None +TEST_DATABRICKS_CONN_STRING: str = os.environ.get("DATABRICKS_URI") +TEST_TRINO_CONN_STRING: str = os.environ.get("TRINO_URI") or None +# clickhouse uri for provided docker - "clickhouse://clickhouse:Password1@localhost:9000/clickhouse" +TEST_CLICKHOUSE_CONN_STRING: str = os.environ.get("CLICKHOUSE_URI") +# vertica uri provided for docker - "vertica://vertica:Password1@localhost:5433/vertica" +TEST_VERTICA_CONN_STRING: str = os.environ.get("VERTICA_URI") +TEST_DUCKDB_CONN_STRING: str = "duckdb://main:@:memory:" + + +DEFAULT_N_SAMPLES = 50 +N_SAMPLES = int(os.environ.get("N_SAMPLES", DEFAULT_N_SAMPLES)) +BENCHMARK = os.environ.get("BENCHMARK", False) +N_THREADS = int(os.environ.get("N_THREADS", 1)) +TEST_ACROSS_ALL_DBS = os.environ.get("TEST_ACROSS_ALL_DBS", True) # Should we run the full db<->db test suite? + + +def get_git_revision_short_hash() -> str: + return subprocess.check_output(["git", "rev-parse", "--short", "HEAD"]).decode("ascii").strip() + + +GIT_REVISION = get_git_revision_short_hash() + +level = logging.ERROR +if os.environ.get("LOG_LEVEL", False): + level = getattr(logging, os.environ["LOG_LEVEL"].upper()) + +logging.basicConfig(level=level) +logging.getLogger("database").setLevel(level) + +try: + from .local_settings import * +except ImportError: + pass # No local settings + + +CONN_STRINGS = { + db.BigQuery: TEST_BIGQUERY_CONN_STRING, + db.MySQL: TEST_MYSQL_CONN_STRING, + db.PostgreSQL: TEST_POSTGRESQL_CONN_STRING, + db.Snowflake: TEST_SNOWFLAKE_CONN_STRING, + db.Redshift: TEST_REDSHIFT_CONN_STRING, + db.Oracle: TEST_ORACLE_CONN_STRING, + db.Presto: TEST_PRESTO_CONN_STRING, + db.Databricks: TEST_DATABRICKS_CONN_STRING, + db.Trino: TEST_TRINO_CONN_STRING, + db.Clickhouse: TEST_CLICKHOUSE_CONN_STRING, + db.Vertica: TEST_VERTICA_CONN_STRING, + db.DuckDB: TEST_DUCKDB_CONN_STRING, +} + +_database_instances = {} + + +def get_conn(cls: type, shared: bool = True) -> Database: + if shared: + if cls not in _database_instances: + _database_instances[cls] = get_conn(cls, shared=False) + return _database_instances[cls] + + return connect(CONN_STRINGS[cls], N_THREADS) + + +def _print_used_dbs(): + used = {k.__name__ for k, v in CONN_STRINGS.items() if v is not None} + unused = {k.__name__ for k, v in CONN_STRINGS.items() if v is None} + + print(f"Testing databases: {', '.join(used)}") + if unused: + logging.info(f"Connection not configured; skipping tests for: {', '.join(unused)}") + if TEST_ACROSS_ALL_DBS: + logging.info( + f"Full tests enabled (every db<->db). May take very long when many dbs are involved. ={TEST_ACROSS_ALL_DBS}" + ) + + +_print_used_dbs() +CONN_STRINGS = {k: v for k, v in CONN_STRINGS.items() if v is not None} + + +def random_table_suffix() -> str: + char_set = string.ascii_lowercase + string.digits + suffix = "_" + suffix += "".join(random.choice(char_set) for _ in range(5)) + return suffix + + +def str_to_checksum(str: str): + # hello world + # => 5eb63bbbe01eeed093cb22bb8f5acdc3 + # => cb22bb8f5acdc3 + # => 273350391345368515 + m = hashlib.md5() + m.update(str.encode("utf-8")) # encode to binary + md5 = m.hexdigest() + # 0-indexed, unlike DBs which are 1-indexed here, so +1 in dbs + half_pos = db.MD5_HEXDIGITS - db.CHECKSUM_HEXDIGITS + return int(md5[half_pos:], 16) + + +class DbTestCase(unittest.TestCase): + "Sets up a table for testing" + db_cls = None + table1_schema = None + shared_connection = True + + def setUp(self): + assert self.db_cls, self.db_cls + + self.connection = get_conn(self.db_cls, self.shared_connection) + + table_suffix = random_table_suffix() + self.table1_name = f"src{table_suffix}" + + self.table1_path = self.connection.parse_table_name(self.table1_name) + + drop_table(self.connection, self.table1_path) + + self.src_table = table(self.table1_path, schema=self.table1_schema) + if self.table1_schema: + self.connection.query(self.src_table.create()) + + return super().setUp() + + def tearDown(self): + drop_table(self.connection, self.table1_path) + + +def _parameterized_class_per_conn(test_databases): + test_databases = set(test_databases) + names = [(cls.__name__, cls) for cls in CONN_STRINGS if cls in test_databases] + return parameterized_class(("name", "db_cls"), names) + + +def test_each_database_in_list(databases) -> Callable: + def _test_per_database(cls): + return _parameterized_class_per_conn(databases)(cls) + + return _test_per_database + diff --git a/tests/test_database.py b/tests/test_database.py new file mode 100644 index 0000000..63c729b --- /dev/null +++ b/tests/test_database.py @@ -0,0 +1,81 @@ +from typing import Callable, List +from datetime import datetime +import unittest + +from .common import str_to_checksum, TEST_MYSQL_CONN_STRING +from .common import str_to_checksum, test_each_database_in_list, get_conn, random_table_suffix + +from sqeleton.queries import table, current_timestamp + +from sqeleton import databases as dbs +from sqeleton import connect + + +TEST_DATABASES = { + dbs.MySQL, + dbs.PostgreSQL, + dbs.Oracle, + dbs.Redshift, + dbs.Snowflake, + dbs.DuckDB, + dbs.BigQuery, + dbs.Presto, + dbs.Trino, + dbs.Vertica, +} + +test_each_database: Callable = test_each_database_in_list(TEST_DATABASES) + + +class TestDatabase(unittest.TestCase): + def setUp(self): + self.mysql = connect(TEST_MYSQL_CONN_STRING) + + def test_connect_to_db(self): + self.assertEqual(1, self.mysql.query("SELECT 1", int)) + +class TestMD5(unittest.TestCase): + def test_md5_as_int(self): + class MD5Dialect(dbs.mysql.Dialect, dbs.mysql.Mixin_MD5): + pass + + self.mysql = connect(TEST_MYSQL_CONN_STRING) + self.mysql.dialect = MD5Dialect() + + str = "hello world" + query_fragment = self.mysql.dialect.md5_as_int("'{0}'".format(str)) + query = f"SELECT {query_fragment}" + + self.assertEqual(str_to_checksum(str), self.mysql.query(query, int)) + + +class TestConnect(unittest.TestCase): + def test_bad_uris(self): + self.assertRaises(ValueError, connect, "p") + self.assertRaises(ValueError, connect, "postgresql:///bla/foo") + self.assertRaises(ValueError, connect, "snowflake://user:pass@foo/bar/TEST1") + self.assertRaises(ValueError, connect, "snowflake://user:pass@foo/bar/TEST1?warehouse=ha&schema=dup") + + +@test_each_database +class TestSchema(unittest.TestCase): + def test_table_list(self): + name = "tbl_" + random_table_suffix() + db = get_conn(self.db_cls) + tbl = table(db.parse_table_name(name), schema={"id": int}) + q = db.dialect.list_tables(db.default_schema, name) + assert not db.query(q) + + db.query(tbl.create()) + self.assertEqual(db.query(q, List[str]), [name]) + + db.query(tbl.drop()) + assert not db.query(q) + + +@test_each_database +class TestQueries(unittest.TestCase): + def test_current_timestamp(self): + db = get_conn(self.db_cls) + res = db.query(current_timestamp(), datetime) + assert isinstance(res, datetime), (res, type(res)) diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..1d9ce5a --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,254 @@ +from datetime import datetime +from typing import List, Optional +import unittest +from sqeleton.abcs import AbstractDatabase, AbstractDialect +from sqeleton.utils import CaseInsensitiveDict, CaseSensitiveDict + +from sqeleton.queries import this, table, Compiler, outerjoin, cte, when, coalesce +from sqeleton.queries.ast_classes import Random + + +def normalize_spaces(s: str): + return " ".join(s.split()) + + +class MockDialect(AbstractDialect): + name = "MockDialect" + + ROUNDS_ON_PREC_LOSS = False + + def quote(self, s: str) -> str: + return s + + def concat(self, l: List[str]) -> str: + s = ", ".join(l) + return f"concat({s})" + + def to_string(self, s: str) -> str: + return f"cast({s} as varchar)" + + def is_distinct_from(self, a: str, b: str) -> str: + return f"{a} is distinct from {b}" + + def random(self) -> str: + return "random()" + + def current_timestamp(self) -> str: + return "now()" + + def offset_limit(self, offset: Optional[int] = None, limit: Optional[int] = None): + x = offset and f"OFFSET {offset}", limit and f"LIMIT {limit}" + return " ".join(filter(None, x)) + + def explain_as_text(self, query: str) -> str: + return f"explain {query}" + + def timestamp_value(self, t: datetime) -> str: + return f"timestamp '{t}'" + + def set_timezone_to_utc(self) -> str: + return "set timezone 'UTC'" + + parse_type = NotImplemented + + +class MockDatabase(AbstractDatabase): + dialect = MockDialect() + + _query = NotImplemented + query_table_schema = NotImplemented + select_table_schema = NotImplemented + _process_table_schema = NotImplemented + parse_table_name = NotImplemented + close = NotImplemented + _normalize_table_path = NotImplemented + is_autocommit = NotImplemented + + +class TestQuery(unittest.TestCase): + def setUp(self): + pass + + def test_basic(self): + c = Compiler(MockDatabase()) + + t = table("point") + t2 = t.select(x=this.x + 1, y=t["y"] + this.x) + assert c.compile(t2) == "SELECT (x + 1) AS x, (y + x) AS y FROM point" + + t = table("point").where(this.x == 1, this.y == 2) + assert c.compile(t) == "SELECT * FROM point WHERE (x = 1) AND (y = 2)" + + t = table("person").where(this.name == "Albert") + self.assertEqual(c.compile(t), "SELECT * FROM person WHERE (name = 'Albert')") + + def test_outerjoin(self): + c = Compiler(MockDatabase()) + + a = table("a") + b = table("b") + keys = ["x", "y"] + cols = ["u", "v"] + + j = outerjoin(a, b).on(a[k] == b[k] for k in keys) + + self.assertEqual( + c.compile(j), "SELECT * FROM a tmp1 FULL OUTER JOIN b tmp2 ON (tmp1.x = tmp2.x) AND (tmp1.y = tmp2.y)" + ) + + + def test_schema(self): + c = Compiler(MockDatabase()) + schema = dict(id="int", comment="varchar") + + # test table + t = table("a", schema=CaseInsensitiveDict(schema)) + q = t.select(this.Id, t["COMMENT"]) + assert c.compile(q) == "SELECT id, comment FROM a" + + t = table("a", schema=CaseSensitiveDict(schema)) + self.assertRaises(KeyError, t.__getitem__, "Id") + self.assertRaises(KeyError, t.select, this.Id) + + # test select + q = t.select(this.id) + self.assertRaises(KeyError, q.__getitem__, "comment") + + # test join + s = CaseInsensitiveDict({"x": int, "y": int}) + a = table("a", schema=s) + b = table("b", schema=s) + keys = ["x", "y"] + j = outerjoin(a, b).on(a[k] == b[k] for k in keys).select(a["x"], b["y"], xsum=a["x"] + b["x"]) + j["x"], j["y"], j["xsum"] + self.assertRaises(KeyError, j.__getitem__, "ysum") + + def test_commutable_select(self): + # c = Compiler(MockDatabase()) + + t = table("a") + q1 = t.select("a").where("b") + q2 = t.where("b").select("a") + assert q1 == q2, (q1, q2) + + def test_cte(self): + c = Compiler(MockDatabase()) + + t = table("a") + + # single cte + t2 = cte(t.select(this.x)) + t3 = t2.select(this.x) + + expected = "WITH tmp1 AS (SELECT x FROM a) SELECT x FROM tmp1" + assert normalize_spaces(c.compile(t3)) == expected + + # nested cte + c = Compiler(MockDatabase()) + t4 = cte(t3).select(this.x) + + expected = "WITH tmp1 AS (SELECT x FROM a), tmp2 AS (SELECT x FROM tmp1) SELECT x FROM tmp2" + assert normalize_spaces(c.compile(t4)) == expected + + # parameterized cte + c = Compiler(MockDatabase()) + t2 = cte(t.select(this.x), params=["y"]) + t3 = t2.select(this.y) + + expected = "WITH tmp1(y) AS (SELECT x FROM a) SELECT y FROM tmp1" + assert normalize_spaces(c.compile(t3)) == expected + + def test_funcs(self): + c = Compiler(MockDatabase()) + t = table("a") + + q = c.compile(t.order_by(Random()).limit(10)) + self.assertEqual(q, "SELECT * FROM a ORDER BY random() LIMIT 10") + + q = c.compile(t.select(coalesce(this.a, this.b))) + self.assertEqual(q, "SELECT COALESCE(a, b) FROM a") + + def test_select_distinct(self): + c = Compiler(MockDatabase()) + t = table("a") + + q = c.compile(t.select(this.b, distinct=True)) + assert q == "SELECT DISTINCT b FROM a" + + # selects merge + q = c.compile(t.where(this.b > 10).select(this.b, distinct=True)) + self.assertEqual(q, "SELECT DISTINCT b FROM a WHERE (b > 10)") + + # selects stay apart + q = c.compile(t.limit(10).select(this.b, distinct=True)) + self.assertEqual(q, "SELECT DISTINCT b FROM (SELECT * FROM a LIMIT 10) tmp1") + + q = c.compile(t.select(this.b, distinct=True).select(distinct=False)) + self.assertEqual(q, "SELECT * FROM (SELECT DISTINCT b FROM a) tmp2") + + def test_table_ops(self): + c = Compiler(MockDatabase()) + a = table("a").select(this.x) + b = table("b").select(this.y) + + q = c.compile(a.union(b)) + assert q == "SELECT x FROM a UNION SELECT y FROM b" + + q = c.compile(a.union_all(b)) + assert q == "SELECT x FROM a UNION ALL SELECT y FROM b" + + q = c.compile(a.minus(b)) + assert q == "SELECT x FROM a EXCEPT SELECT y FROM b" + + q = c.compile(a.intersect(b)) + assert q == "SELECT x FROM a INTERSECT SELECT y FROM b" + + def test_ops(self): + c = Compiler(MockDatabase()) + t = table("a") + + q = c.compile(t.select(this.b + this.c)) + self.assertEqual(q, "SELECT (b + c) FROM a") + + q = c.compile(t.select(this.b.like(this.c))) + self.assertEqual(q, "SELECT (b LIKE c) FROM a") + + q = c.compile(t.select(-this.b.sum())) + self.assertEqual(q, "SELECT (-SUM(b)) FROM a") + + def test_group_by(self): + c = Compiler(MockDatabase()) + t = table("a") + + q = c.compile(t.group_by(keys=[this.b], values=[this.c])) + self.assertEqual(q, "SELECT b, c FROM a GROUP BY 1") + + q = c.compile(t.where(this.b > 1).group_by(keys=[this.b], values=[this.c])) + self.assertEqual(q, "SELECT b, c FROM a WHERE (b > 1) GROUP BY 1") + + q = c.compile(t.select(this.b).group_by(keys=[this.b], values=[])) + self.assertEqual(q, "SELECT b FROM (SELECT b FROM a) tmp1 GROUP BY 1") + + # Having + q = c.compile(t.group_by(keys=[this.b], values=[this.c]).having(this.b > 1)) + self.assertEqual(q, "SELECT b, c FROM a GROUP BY 1 HAVING (b > 1)") + + q = c.compile(t.select(this.b).group_by(keys=[this.b], values=[]).having(this.b > 1)) + self.assertEqual(q, "SELECT b FROM (SELECT b FROM a) tmp2 GROUP BY 1 HAVING (b > 1)") + + # Having sum + q = c.compile(t.group_by(keys=[this.b], values=[this.c]).having(this.b.sum() > 1)) + self.assertEqual(q, "SELECT b, c FROM a GROUP BY 1 HAVING (SUM(b) > 1)") + + def test_case_when(self): + c = Compiler(MockDatabase()) + t = table("a") + + z = when(this.b).then(this.c) + y = t.select(z) + + q = c.compile(t.select(when(this.b).then(this.c))) + self.assertEqual(q, "SELECT CASE WHEN b THEN c END FROM a") + + q = c.compile(t.select(when(this.b).then(this.c).else_(this.d))) + self.assertEqual(q, "SELECT CASE WHEN b THEN c ELSE d END FROM a") diff --git a/tests/test_sql.py b/tests/test_sql.py new file mode 100644 index 0000000..ac5ddef --- /dev/null +++ b/tests/test_sql.py @@ -0,0 +1,106 @@ +import unittest + +from .common import TEST_MYSQL_CONN_STRING + +from sqeleton import connect +from sqeleton.queries import Compiler, Count, Explain, Select, table, In, BinOp, Code + + +class TestSQL(unittest.TestCase): + def setUp(self): + self.mysql = connect(TEST_MYSQL_CONN_STRING) + self.compiler = Compiler(self.mysql) + + def test_compile_string(self): + self.assertEqual("SELECT 1", self.compiler.compile(Code("SELECT 1"))) + + def test_compile_int(self): + self.assertEqual("1", self.compiler.compile(1)) + + def test_compile_table_name(self): + self.assertEqual( + "`marine_mammals`.`walrus`", self.compiler.replace(root=False).compile(table("marine_mammals", "walrus")) + ) + + def test_compile_select(self): + expected_sql = "SELECT name FROM `marine_mammals`.`walrus`" + self.assertEqual( + expected_sql, + self.compiler.compile( + Select( + table("marine_mammals", "walrus"), + [Code("name")], + ) + ), + ) + + # def test_enum(self): + # expected_sql = "(SELECT *, (row_number() over (ORDER BY id)) as idx FROM `walrus` ORDER BY id) tmp" + # self.assertEqual( + # expected_sql, + # self.compiler.compile( + # Enum( + # ("walrus",), + # "id", + # ) + # ), + # ) + + # def test_checksum(self): + # expected_sql = "SELECT name, sum(cast(conv(substring(md5(concat(cast(id as char), cast(timestamp as char))), 18), 16, 10) as unsigned)) FROM `marine_mammals`.`walrus`" + # self.assertEqual( + # expected_sql, + # self.compiler.compile( + # Select( + # ["name", Checksum(["id", "timestamp"])], + # TableName(("marine_mammals", "walrus")), + # ) + # ), + # ) + + def test_compare(self): + expected_sql = "SELECT name FROM `marine_mammals`.`walrus` WHERE (id <= 1000) AND (id > 1)" + self.assertEqual( + expected_sql, + self.compiler.compile( + Select( + table("marine_mammals", "walrus"), + [Code("name")], + [BinOp("<=", [Code("id"), Code("1000")]), BinOp(">", [Code("id"), Code("1")])], + ) + ), + ) + + def test_in(self): + expected_sql = "SELECT name FROM `marine_mammals`.`walrus` WHERE (id IN (1, 2, 3))" + self.assertEqual( + expected_sql, + self.compiler.compile( + Select(table("marine_mammals", "walrus"), [Code("name")], [In(Code("id"), [1, 2, 3])]) + ), + ) + + def test_count(self): + expected_sql = "SELECT count(*) FROM `marine_mammals`.`walrus` WHERE (id IN (1, 2, 3))" + self.assertEqual( + expected_sql, + self.compiler.compile(Select(table("marine_mammals", "walrus"), [Count()], [In(Code("id"), [1, 2, 3])])), + ) + + def test_count_with_column(self): + expected_sql = "SELECT count(id) FROM `marine_mammals`.`walrus` WHERE (id IN (1, 2, 3))" + self.assertEqual( + expected_sql, + self.compiler.compile( + Select(table("marine_mammals", "walrus"), [Count(Code("id"))], [In(Code("id"), [1, 2, 3])]) + ), + ) + + def test_explain(self): + expected_sql = "EXPLAIN FORMAT=TREE SELECT count(id) FROM `marine_mammals`.`walrus` WHERE (id IN (1, 2, 3))" + self.assertEqual( + expected_sql, + self.compiler.compile( + Explain(Select(table("marine_mammals", "walrus"), [Count(Code("id"))], [In(Code("id"), [1, 2, 3])])) + ), + ) diff --git a/tests/waiting_for_stack_up.sh b/tests/waiting_for_stack_up.sh new file mode 100644 index 0000000..a085c76 --- /dev/null +++ b/tests/waiting_for_stack_up.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ -n "$VERTICA_URI" ] + then + echo "Check Vertica DB running..." + while true + do + if docker logs dd-vertica | tail -n 100 | grep -q -i "vertica is now running" + then + echo "Vertica DB is ready"; + break; + else + echo "Waiting for Vertica DB starting..."; + sleep 10; + fi + done +fi