From 30c038800cbb2397fa17d83089044bdcfb63bf37 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Thu, 6 Mar 2025 23:09:59 +0000 Subject: [PATCH 1/3] Chore: Switch from setup.py to pyproject.toml --- .circleci/continue_config.yml | 8 +- Makefile | 6 +- pyproject.toml | 155 ++++++++++++++++++++++++++++++ setup.cfg | 3 - setup.py | 175 ---------------------------------- tests/.gitignore | 1 + tests/pyproject.toml | 29 ++++++ tests/setup.py | 33 ++----- 8 files changed, 201 insertions(+), 209 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py create mode 100644 tests/.gitignore create mode 100644 tests/pyproject.toml diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index 8a1000c6b5..b5ff6eec29 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -281,10 +281,10 @@ workflows: - airflow_docker_tests: requires: - style_and_slow_tests - filters: - branches: - only: - - main + #filters: + # branches: + # only: + # - main - engine_tests_docker: name: engine_<< matrix.engine >> matrix: diff --git a/Makefile b/Makefile index 3889aea59e..0e221220b4 100644 --- a/Makefile +++ b/Makefile @@ -19,16 +19,16 @@ ui-style: SKIP=ruff,ruff-format,mypy pre-commit run --all-files doc-test: - PYTEST_PLUGINS=tests.common_fixtures pytest --doctest-modules sqlmesh/core sqlmesh/utils + PYTEST_PLUGINS=tests.common_fixtures python -m pytest --doctest-modules sqlmesh/core sqlmesh/utils package: - pip3 install wheel && python3 setup.py sdist bdist_wheel + pip3 install build && python3 -m build publish: package pip3 install twine && python3 -m twine upload dist/* package-tests: - pip3 install wheel && python3 tests/setup.py sdist bdist_wheel + pip3 install build && cp pyproject.toml tests/sqlmesh_pyproject.toml && python3 -m build tests/ publish-tests: package-tests pip3 install twine && python3 -m twine upload -r tobiko-private tests/dist/* diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000..05446314bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,155 @@ +[project] +name = "sqlmesh" +dynamic = ["version"] +description = "Next-generation data transformation framework" +readme = "README.md" +authors = [{ name = "TobikoData Inc.", email = "engineering@tobikodata.com" }] +license = { file = "LICENSE" } +requires-python = ">= 3.9" +dependencies = [ + "astor", + "click", + "croniter", + "duckdb!=0.10.3", + "dateparser", + "hyperscript>=0.1.0", + "importlib-metadata; python_version<'3.12'", + "ipywidgets", + "jinja2", + "pandas", + "pydantic>=2.0.0", + "requests", + "rich[jupyter]", + "ruamel.yaml", + "sqlglot[rs]~=26.8.0", + "tenacity", + "time-machine", +] +classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: SQL", + "Programming Language :: Python :: 3 :: Only", +] + +[project.optional-dependencies] +athena = ["PyAthena[Pandas]"] +azuresql = ["pymssql"] +bigquery = [ + "google-cloud-bigquery[pandas]", + "google-cloud-bigquery-storage" +] +bigframes = ["bigframes>=1.32.0"] +clickhouse = ["clickhouse-connect"] +databricks = ["databricks-sql-connector"] +dev = [ + "agate==1.7.1", + "apache-airflow==2.9.1", + "opentelemetry-proto==1.27.0", # pip was having trouble resolving this transitive dependency of airflow + "beautifulsoup4", + "clickhouse-connect", + "cryptography", + "databricks-sql-connector", + "dbt-bigquery", + "dbt-core", + "dbt-duckdb>=1.7.1", + "dbt-snowflake", + "dbt-athena-community", + "dbt-clickhouse", + "dbt-databricks", + "dbt-redshift", + "dbt-sqlserver>=1.7.0", + "dbt-trino", + "Faker", + "google-auth", + "google-cloud-bigquery", + "google-cloud-bigquery-storage", + "mypy~=1.13.0", + "pandas-stubs", + "pre-commit", + "psycopg2-binary", + "pydantic", + "PyAthena[Pandas]", + "PyGithub~=2.5.0", + "pyspark~=3.5.0", + "pytest", + "pytest-asyncio", + "pytest-mock", + "pytest-retry", + "pytest-xdist", + "pytz", + "redshift_connector", + "ruff~=0.7.0", + "snowflake-connector-python[pandas,secure-local-storage]>=3.0.2", + "sqlalchemy-stubs", + "trino", + "types-croniter", + "types-dateparser", + "types-PyMySQL", + "types-python-dateutil", + "types-pytz", + "types-requests==2.28.8", + "typing-extensions", +] +dbt = ["dbt-core<2"] +dlt = ["dlt"] +gcppostgres = ["cloud-sql-python-connector[pg8000]>=1.8.0"] +github = ["PyGithub~=2.5.0"] +llm = ["langchain", "openai"] +mssql = ["pymssql"] +mysql = ["pymysql"] +mwaa = ["boto3"] +postgres = ["psycopg2"] +redshift = ["redshift_connector"] +slack = ["slack_sdk"] +snowflake = [ + "cryptography", + "snowflake-connector-python[pandas,secure-local-storage]", + # as at 2024-08-05, snowflake-snowpark-python is only available up to Python 3.11 + "snowflake-snowpark-python; python_version<'3.12'", +] +trino = ["trino"] +web = [ + "fastapi==0.115.5", + "watchfiles>=0.19.0", + "uvicorn[standard]==0.22.0", + "sse-starlette>=0.2.2", + "pyarrow", +] +risingwave = ["psycopg2"] + +[project.scripts] +sqlmesh = "sqlmesh.cli.main:cli" +sqlmesh_cicd = "sqlmesh.cicd.bot:bot" + +[project.entry-points."airflow.plugins"] +sqlmesh_airflow = "sqlmesh.schedulers.airflow.plugin:SqlmeshAirflowPlugin" + +[project.urls] +Homepage = "https://sqlmesh.com/" +Documentation = "https://sqlmesh.readthedocs.io/en/stable/" +Repository = "https://github.com/TobikoData/sqlmesh" +Issues = "https://github.com/TobikoData/sqlmesh/issues" + +[build-system] +requires = ["setuptools >= 61.0", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +include-package-data = false + +[tool.setuptools_scm] +version_file = "sqlmesh/_version.py" +fallback_version = "0.0.0" +local_scheme = "no-local-version" + +[tool.setuptools.packages.find] +include=["sqlmesh", "sqlmesh.*", "web*"] + +[tool.setuptools.package-data] +web = ["client/dist/**"] +"*" = ["py.typed"] + + diff --git a/setup.cfg b/setup.cfg index bbda584160..90e478cd02 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,3 @@ -[metadata] -version = attr: sqlmesh.__version__ - [mypy] plugins = pydantic.mypy no_implicit_optional = True diff --git a/setup.py b/setup.py deleted file mode 100644 index 4f26e3bd76..0000000000 --- a/setup.py +++ /dev/null @@ -1,175 +0,0 @@ -import os -from os.path import exists - -from setuptools import find_packages, setup - -description = open("README.md").read() if exists("README.md") else "" - -setup( - name="sqlmesh", - description="", - long_description=description, - long_description_content_type="text/markdown", - url="https://github.com/TobikoData/sqlmesh", - author="TobikoData Inc.", - author_email="engineering@tobikodata.com", - license="Apache License 2.0", - packages=find_packages(include=["sqlmesh", "sqlmesh.*", "web*"]), - package_data={"web": ["client/dist/**"], "": ["py.typed"]}, - entry_points={ - "console_scripts": [ - "sqlmesh = sqlmesh.cli.main:cli", - "sqlmesh_cicd = sqlmesh.cicd.bot:bot", - ], - "airflow.plugins": [ - "sqlmesh_airflow = sqlmesh.schedulers.airflow.plugin:SqlmeshAirflowPlugin", - ], - }, - use_scm_version={ - "write_to": "sqlmesh/_version.py", - "fallback_version": "0.0.0", - "local_scheme": "no-local-version", - }, - setup_requires=["setuptools_scm"], - install_requires=[ - "astor", - "click", - "croniter", - "duckdb!=0.10.3", - "dateparser", - "hyperscript>=0.1.0", - "importlib-metadata; python_version<'3.12'", - "ipywidgets", - "jinja2", - "pandas", - "pydantic>=2.0.0", - "requests", - "rich[jupyter]", - "ruamel.yaml", - "setuptools; python_version>='3.12'", - "sqlglot[rs]~=26.8.0", - "tenacity", - "time-machine", - ], - extras_require={ - "athena": ["PyAthena[Pandas]"], - "azuresql": ["pymssql"], - "bigquery": [ - "google-cloud-bigquery[pandas]", - "google-cloud-bigquery-storage", - ], - "bigframes": ["bigframes>=1.32.0"], - "clickhouse": ["clickhouse-connect"], - "databricks": ["databricks-sql-connector"], - "dev": [ - "agate==1.7.1", - f"apache-airflow=={os.environ.get('AIRFLOW_VERSION', '2.9.1')}", - "opentelemetry-proto==1.27.0", # pip was having trouble resolving this transitive dependency of airflow - "beautifulsoup4", - "clickhouse-connect", - "cryptography", - "databricks-sql-connector", - "dbt-bigquery", - "dbt-core", - "dbt-duckdb>=1.7.1", - "dbt-snowflake", - "dbt-athena-community", - "dbt-clickhouse", - "dbt-databricks", - "dbt-redshift", - "dbt-sqlserver>=1.7.0", - "dbt-trino", - "Faker", - "google-auth", - "google-cloud-bigquery", - "google-cloud-bigquery-storage", - "mypy~=1.13.0", - "pandas-stubs", - "pre-commit", - "psycopg2-binary", - "pydantic", - "PyAthena[Pandas]", - "PyGithub~=2.5.0", - "pyspark~=3.5.0", - "pytest", - "pytest-asyncio", - "pytest-mock", - "pytest-retry", - "pytest-xdist", - "pytz", - "redshift_connector", - "ruff~=0.7.0", - "snowflake-connector-python[pandas,secure-local-storage]>=3.0.2", - "sqlalchemy-stubs", - "trino", - "types-croniter", - "types-dateparser", - "types-PyMySQL", - "types-python-dateutil", - "types-pytz", - "types-requests==2.28.8", - "typing-extensions", - ], - "dbt": [ - "dbt-core<2", - ], - "dlt": [ - "dlt", - ], - "gcppostgres": [ - "cloud-sql-python-connector[pg8000]>=1.8.0", - ], - "github": [ - "PyGithub~=2.5.0", - ], - "llm": [ - "langchain", - "openai", - ], - "mssql": [ - "pymssql", - ], - "mysql": [ - "pymysql", - ], - "mwaa": [ - "boto3", - ], - "postgres": [ - "psycopg2", - ], - "redshift": [ - "redshift_connector", - ], - "slack": [ - "slack_sdk", - ], - "snowflake": [ - "cryptography", - "snowflake-connector-python[pandas,secure-local-storage]", - # as at 2024-08-05, snowflake-snowpark-python is only available up to Python 3.11 - "snowflake-snowpark-python; python_version<'3.12'", - ], - "trino": [ - "trino", - ], - "web": [ - "fastapi==0.115.5", - "watchfiles>=0.19.0", - "uvicorn[standard]==0.22.0", - "sse-starlette>=0.2.2", - "pyarrow", - ], - "risingwave": [ - "psycopg2", - ], - }, - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Operating System :: OS Independent", - "Programming Language :: SQL", - "Programming Language :: Python :: 3 :: Only", - ], -) diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000..fff33c1c74 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +sqlmesh_pyproject.toml \ No newline at end of file diff --git a/tests/pyproject.toml b/tests/pyproject.toml new file mode 100644 index 0000000000..172710d9ec --- /dev/null +++ b/tests/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "sqlmesh-tests" +dynamic = ["version", "dependencies"] +description = "Tests for SQLMesh" +authors = [{ name = "TobikoData Inc.", email = "engineering@tobikodata.com" }] +license = { text = "Apache License 2.0" } + +[project.urls] +Homepage = "https://sqlmesh.com/" +Documentation = "https://sqlmesh.readthedocs.io/en/stable/" +Repository = "https://github.com/TobikoData/sqlmesh" +Issues = "https://github.com/TobikoData/sqlmesh/issues" + +[build-system] +requires = ["setuptools", "setuptools_scm", "toml"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.package-dir] +sqlmesh_tests = "" + +[tool.setuptools_scm] +version_file = "_version.py" +fallback_version = "0.0.0" +local_scheme = "no-local-version" + +[tool.setuptools.package-data] +"*" = ["fixtures/**", "*.toml"] + + diff --git a/tests/setup.py b/tests/setup.py index a58c3cbadd..d072cb555b 100644 --- a/tests/setup.py +++ b/tests/setup.py @@ -1,27 +1,12 @@ -import os - import setuptools +from pathlib import Path +import toml # type: ignore -os.chdir(os.path.join(os.path.dirname(__file__), "..")) -sqlmesh_dist = setuptools.distutils.core.run_setup("setup.py", stop_after="init") # type: ignore -requirements = sqlmesh_dist.install_requires + sqlmesh_dist.extras_require["dev"] # type: ignore -os.chdir(os.path.dirname(__file__)) +# This relies on `make package-tests` copying the sqlmesh pyproject.toml into tests/ so we can reference it +# Otherwise, it's not available in the build environment +sqlmesh_pyproject = Path(__file__).parent / "sqlmesh_pyproject.toml" +parsed = toml.load(sqlmesh_pyproject)["project"] +install_requires = parsed["dependencies"] + parsed["optional-dependencies"]["dev"] -setuptools.setup( - name="sqlmesh-tests", - description="Tests for SQLMesh", - url="https://github.com/TobikoData/sqlmesh", - author="TobikoData Inc.", - author_email="engineering@tobikodata.com", - license="Apache License 2.0", - package_dir={"sqlmesh_tests": ""}, - package_data={"": ["fixtures/**"]}, - use_scm_version={ - "root": "..", - "write_to": "_version.py", - "fallback_version": "0.0.0", - "local_scheme": "no-local-version", - }, - setup_requires=["setuptools_scm"], - install_requires=requirements, -) +# this is just so we can have a dynamic install_requires, everything else is defined in pyproject.toml +setuptools.setup(install_requires=install_requires) From 4b538f4c4494f9c702e04503e718b2bd045435e2 Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Fri, 7 Mar 2025 05:13:00 +0000 Subject: [PATCH 2/3] Airflow shotgun debugging --- examples/airflow/Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/airflow/Dockerfile.template b/examples/airflow/Dockerfile.template index ec9281ea3e..6e74b230a7 100644 --- a/examples/airflow/Dockerfile.template +++ b/examples/airflow/Dockerfile.template @@ -64,7 +64,7 @@ RUN pip install apache-airflow-providers-databricks==6.4.0 \ # Install Deps USER root -ADD setup.py /opt/sqlmesh/setup.py +ADD pyproject.toml /opt/sqlmesh/pyproject.toml RUN mkdir /opt/sqlmesh/sqlmesh && touch /opt/sqlmesh/sqlmesh/__init__.py && chown -R airflow /opt/sqlmesh ADD examples/custom_materializations /opt/custom_materializations From 7ea1e6b5fe53c4d46a3570ae3c4996bcaf07503c Mon Sep 17 00:00:00 2001 From: Erin Drummond Date: Mon, 10 Mar 2025 19:25:48 +0000 Subject: [PATCH 3/3] Reinstate airflow branch filter --- .circleci/continue_config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/continue_config.yml b/.circleci/continue_config.yml index b5ff6eec29..8a1000c6b5 100644 --- a/.circleci/continue_config.yml +++ b/.circleci/continue_config.yml @@ -281,10 +281,10 @@ workflows: - airflow_docker_tests: requires: - style_and_slow_tests - #filters: - # branches: - # only: - # - main + filters: + branches: + only: + - main - engine_tests_docker: name: engine_<< matrix.engine >> matrix: