Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .circleci/continue_config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,10 @@ jobs:
echo "export REDSHIFT_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
echo "export GCP_POSTGRES_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"
echo "export FABRIC_DATABASE='$TEST_DB_NAME'" >> "$BASH_ENV"

# Make snowflake private key available
echo $SNOWFLAKE_PRIVATE_KEY_RAW | base64 -d > /tmp/snowflake-keyfile.p8
echo "export SNOWFLAKE_PRIVATE_KEY_FILE='/tmp/snowflake-keyfile.p8'" >> "$BASH_ENV"
- run:
name: Create test database
command: ./.circleci/manage-test-db.sh << parameters.engine >> "$TEST_DB_NAME" up
Expand Down Expand Up @@ -309,7 +313,7 @@ workflows:
matrix:
parameters:
engine:
#- snowflake
- snowflake
- databricks
- redshift
- bigquery
Expand Down
16 changes: 1 addition & 15 deletions .circleci/manage-test-db.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ function_exists() {
# Snowflake
snowflake_init() {
echo "Installing Snowflake CLI"
pip install "snowflake-cli-labs<3.8.0"
pip install "snowflake-cli"
}

snowflake_up() {
Expand All @@ -40,20 +40,6 @@ snowflake_down() {
databricks_init() {
echo "Installing Databricks CLI"
curl -fsSL https://raw.githubusercontent.com/databricks/setup-cli/main/install.sh | sudo sh || true

echo "Writing out Databricks CLI config file"
echo -e "[DEFAULT]\nhost = $DATABRICKS_SERVER_HOSTNAME\ntoken = $DATABRICKS_ACCESS_TOKEN" > ~/.databrickscfg

# this takes a path like 'sql/protocolv1/o/2934659247569/0723-005339-foobar' and extracts '0723-005339-foobar' from it
CLUSTER_ID=${DATABRICKS_HTTP_PATH##*/}

echo "Extracted cluster id: $CLUSTER_ID from '$DATABRICKS_HTTP_PATH'"

# Note: the cluster doesnt need to be running to create / drop catalogs, but it does need to be running to run the integration tests
echo "Ensuring cluster is running"
# the || true is to prevent the following error from causing an abort:
# > Error: is in unexpected state Running.
databricks clusters start $CLUSTER_ID || true
}

databricks_up() {
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/pr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ jobs:
test-vscode-e2e:
runs-on:
labels: [ubuntu-2204-8]
# As at 2026-01-12 this job flakes 100% of the time. It needs investigation
if: false
steps:
- uses: actions/checkout@v5
- uses: actions/setup-node@v6
Expand Down
4 changes: 2 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -212,14 +212,14 @@ risingwave-test: engine-risingwave-up
# Cloud Engines #
#################

snowflake-test: guard-SNOWFLAKE_ACCOUNT guard-SNOWFLAKE_WAREHOUSE guard-SNOWFLAKE_DATABASE guard-SNOWFLAKE_USER guard-SNOWFLAKE_PASSWORD engine-snowflake-install
snowflake-test: guard-SNOWFLAKE_ACCOUNT guard-SNOWFLAKE_WAREHOUSE guard-SNOWFLAKE_DATABASE guard-SNOWFLAKE_USER engine-snowflake-install
pytest -n auto -m "snowflake" --reruns 3 --junitxml=test-results/junit-snowflake.xml

bigquery-test: guard-BIGQUERY_KEYFILE engine-bigquery-install
$(PIP) install -e ".[bigframes]"
pytest -n auto -m "bigquery" --reruns 3 --junitxml=test-results/junit-bigquery.xml

databricks-test: guard-DATABRICKS_CATALOG guard-DATABRICKS_SERVER_HOSTNAME guard-DATABRICKS_HTTP_PATH guard-DATABRICKS_ACCESS_TOKEN guard-DATABRICKS_CONNECT_VERSION engine-databricks-install
databricks-test: guard-DATABRICKS_CATALOG guard-DATABRICKS_SERVER_HOSTNAME guard-DATABRICKS_HTTP_PATH guard-DATABRICKS_CONNECT_VERSION engine-databricks-install
$(PIP) install 'databricks-connect==${DATABRICKS_CONNECT_VERSION}'
pytest -n auto -m "databricks" --reruns 3 --junitxml=test-results/junit-databricks.xml

Expand Down
17 changes: 17 additions & 0 deletions docs/integrations/engines/bigquery.md
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,23 @@ If the `impersonated_service_account` argument is set, SQLMesh will:

The user account must have [sufficient permissions to impersonate the service account](https://cloud.google.com/docs/authentication/use-service-account-impersonation).

## Query Label

BigQuery supports a `query_label` session variable which is attached to query jobs and can be used for auditing / attribution.

SQLMesh supports setting it via `session_properties.query_label` on a model, as an array (or tuple) of key/value tuples.

Example:
```sql
MODEL (
name my_project.my_dataset.my_model,
dialect 'bigquery',
session_properties (
query_label = [('team', 'data_platform'), ('env', 'prod')]
)
);
```

## Permissions Required
With any of the above connection methods, ensure these BigQuery permissions are enabled to allow SQLMesh to work correctly.

Expand Down
1 change: 1 addition & 0 deletions docs/integrations/engines/trino.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ hive.metastore.glue.default-warehouse-dir=s3://my-bucket/
| `http_scheme` | The HTTP scheme to use when connecting to your cluster. By default, it's `https` and can only be `http` for no-auth or basic auth. | string | N |
| `port` | The port to connect to your cluster. By default, it's `443` for `https` scheme and `80` for `http` | int | N |
| `roles` | Mapping of catalog name to a role | dict | N |
| `source` | Value to send as Trino's `source` field for query attribution / auditing. Default: `sqlmesh`. | string | N |
| `http_headers` | Additional HTTP headers to send with each request. | dict | N |
| `session_properties` | Trino session properties. Run `SHOW SESSION` to see all options. | dict | N |
| `retries` | Number of retries to attempt when a request fails. Default: `3` | int | N |
Expand Down
Loading