diff --git a/.github/workflows/gen.yml b/.github/workflows/gen.yml index 503af3ee37..eb83825c39 100644 --- a/.github/workflows/gen.yml +++ b/.github/workflows/gen.yml @@ -32,7 +32,7 @@ jobs: PG_PASSWORD: postgres PG_PORT: ${{ job.services.postgres.ports['5432'] }} - name: Save results - uses: actions/upload-artifact@v5 + uses: actions/upload-artifact@v6 with: name: sqlc-pg-gen-results path: gen diff --git a/CLAUDE.md b/CLAUDE.md index 9d637256a1..43abb0d491 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,9 +7,108 @@ This document provides essential information for working with the sqlc codebase, ### Prerequisites - **Go 1.25.0+** - Required for building and testing -- **Docker & Docker Compose** - Required for integration tests with databases +- **Docker & Docker Compose** - Required for integration tests with databases (local development) - **Git** - For version control +## Claude Code Remote Environment Setup + +When running in the Claude Code remote environment (or any environment without Docker), you can install PostgreSQL and MySQL natively. The test framework automatically detects and uses native database installations. + +### Step 1: Configure apt Proxy (Required in Remote Environment) + +The Claude Code remote environment requires an HTTP proxy for apt. Configure it: + +```bash +bash -c 'echo "Acquire::http::Proxy \"$http_proxy\";"' | sudo tee /etc/apt/apt.conf.d/99proxy +``` + +### Step 2: Install PostgreSQL + +```bash +sudo apt-get update +sudo apt-get install -y postgresql +sudo service postgresql start +``` + +Configure PostgreSQL for password authentication: + +```bash +# Set password for postgres user +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + +# Enable password authentication for localhost +echo 'host all all 127.0.0.1/32 md5' | sudo tee -a /etc/postgresql/16/main/pg_hba.conf +sudo service postgresql reload +``` + +Test the connection: + +```bash +PGPASSWORD=postgres psql -h 127.0.0.1 -U postgres -c "SELECT 1;" +``` + +### Step 3: Install MySQL 9 + +MySQL 9 is required for full test compatibility (includes VECTOR type support). Download and install from Oracle: + +```bash +# Download MySQL 9 bundle +curl -LO https://dev.mysql.com/get/Downloads/MySQL-9.1/mysql-server_9.1.0-1ubuntu24.04_amd64.deb-bundle.tar + +# Extract packages +mkdir -p /tmp/mysql9 +tar -xf mysql-server_9.1.0-1ubuntu24.04_amd64.deb-bundle.tar -C /tmp/mysql9 + +# Install packages (in order) +cd /tmp/mysql9 +sudo dpkg -i mysql-common_*.deb \ + mysql-community-client-plugins_*.deb \ + mysql-community-client-core_*.deb \ + mysql-community-client_*.deb \ + mysql-client_*.deb \ + mysql-community-server-core_*.deb \ + mysql-community-server_*.deb \ + mysql-server_*.deb + +# Make init script executable +sudo chmod +x /etc/init.d/mysql + +# Initialize data directory and start MySQL +sudo mysqld --initialize-insecure --user=mysql +sudo /etc/init.d/mysql start + +# Set root password +mysql -u root -e "ALTER USER 'root'@'localhost' IDENTIFIED BY 'mysecretpassword'; FLUSH PRIVILEGES;" +``` + +Test the connection: + +```bash +mysql -h 127.0.0.1 -u root -pmysecretpassword -e "SELECT VERSION();" +``` + +### Step 4: Run End-to-End Tests + +With both databases running, the test framework automatically detects them: + +```bash +# Run all end-to-end tests +go test --tags=examples -timeout 20m ./internal/endtoend/... + +# Run example tests +go test --tags=examples -timeout 20m ./examples/... + +# Run the full test suite +go test --tags=examples -timeout 20m ./... +``` + +The native database support (in `internal/sqltest/native/`) automatically: +- Detects running PostgreSQL and MySQL instances +- Starts services if installed but not running +- Uses standard connection URIs: + - PostgreSQL: `postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable` + - MySQL: `root:mysecretpassword@tcp(127.0.0.1:3306)/mysql` + ### Running Tests #### Basic Unit Tests (No Database Required) diff --git a/Dockerfile b/Dockerfile index f5d1f4f93a..05a93abf7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # STEP 1: Build sqlc -FROM golang:1.25.4 AS builder +FROM golang:1.25.5 AS builder COPY . /workspace WORKDIR /workspace diff --git a/docs/reference/environment-variables.md b/docs/reference/environment-variables.md index 185807168c..837dd13980 100644 --- a/docs/reference/environment-variables.md +++ b/docs/reference/environment-variables.md @@ -1,5 +1,22 @@ # Environment variables +## SQLCEXPERIMENT + +The `SQLCEXPERIMENT` variable controls experimental features within sqlc. It is +a comma-separated list of experiment names. This is modeled after Go's +[GOEXPERIMENT](https://pkg.go.dev/internal/goexperiment) environment variable. + +Experiment names can be prefixed with `no` to explicitly disable them. + +``` +SQLCEXPERIMENT=foo,bar # enable foo and bar experiments +SQLCEXPERIMENT=nofoo # explicitly disable foo experiment +SQLCEXPERIMENT=foo,nobar # enable foo, disable bar +``` + +Currently, no experiments are defined. Experiments will be documented here as +they are introduced. + ## SQLCCACHE The `SQLCCACHE` environment variable dictates where `sqlc` will store cached diff --git a/docs/reference/language-support.rst b/docs/reference/language-support.rst index 057a5ef65f..20de2817d6 100644 --- a/docs/reference/language-support.rst +++ b/docs/reference/language-support.rst @@ -29,6 +29,8 @@ Python `rayakame/sqlc-gen-better-python`_ N/A Beta [Any] `fdietze/sqlc-gen-from-template`_ Stable Stable Stable ======== ================================== =============== =============== =============== +Plugins developed by our Community can also be found using our `github topic`_. + Community projects ****************** @@ -51,3 +53,4 @@ Gleam `daniellionel01/parrot`_ Stable Stable S .. _tinyzimmer/sqlc-gen-zig: https://github.com/tinyzimmer/sqlc-gen-zig .. _daniellionel01/parrot: https://github.com/daniellionel01/parrot .. _rayakame/sqlc-gen-better-python: https://github.com/rayakame/sqlc-gen-better-python +.. _github topic: https://github.com/topics/sqlc-plugin diff --git a/docs/requirements.txt b/docs/requirements.txt index f6503a79bd..ae7506b14f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,7 +3,7 @@ Jinja2==3.1.6 MarkupSafe==3.0.3 Pygments==2.19.2 Sphinx==7.4.7 -certifi==2025.11.12 +certifi==2026.1.4 chardet==5.2.0 commonmark==0.9.1 docutils==0.20.1 @@ -11,7 +11,7 @@ idna==3.11 imagesize==1.4.1 myst-parser==4.0.1 packaging==25.0 -pyparsing==3.2.5 +pyparsing==3.3.1 pytz==2025.2 requests==2.32.5 snowballstemmer==3.0.1 @@ -24,4 +24,4 @@ sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 sphinxext-rediraffe==0.3.0 -urllib3==2.5.0 +urllib3==2.6.2 diff --git a/go.mod b/go.mod index 630795248e..44ecebecb4 100644 --- a/go.mod +++ b/go.mod @@ -13,21 +13,21 @@ require ( github.com/google/cel-go v0.26.1 github.com/google/go-cmp v0.7.0 github.com/jackc/pgx/v4 v4.18.3 - github.com/jackc/pgx/v5 v5.7.6 + github.com/jackc/pgx/v5 v5.8.0 github.com/jinzhu/inflection v1.0.0 github.com/lib/pq v1.10.9 - github.com/ncruces/go-sqlite3 v0.30.2 + github.com/ncruces/go-sqlite3 v0.30.4 github.com/pganalyze/pg_query_go/v6 v6.1.0 github.com/pingcap/tidb/pkg/parser v0.0.0-20250324122243-d51e00e5bbf0 github.com/riza-io/grpc-go v0.2.0 - github.com/spf13/cobra v1.10.1 + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 - github.com/tetratelabs/wazero v1.10.1 + github.com/tetratelabs/wazero v1.11.0 github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/sync v0.18.0 - google.golang.org/grpc v1.77.0 - google.golang.org/protobuf v1.36.10 + golang.org/x/sync v0.19.0 + google.golang.org/grpc v1.78.0 + google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 ) @@ -55,13 +55,13 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.45.0 // indirect + golang.org/x/crypto v0.46.0 // indirect golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b // indirect golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.32.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) diff --git a/go.sum b/go.sum index 002020f15c..0fb994c119 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= -github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= +github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= @@ -117,8 +117,8 @@ github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/ncruces/go-sqlite3 v0.30.2 h1:1GVbHAkKAOwjJd3JYl8ldrYROudfZUOah7oXPD7VZbQ= -github.com/ncruces/go-sqlite3 v0.30.2/go.mod h1:AxKu9sRxkludimFocbktlY6LiYSkxiI5gTA8r+os/Nw= +github.com/ncruces/go-sqlite3 v0.30.4 h1:j9hEoOL7f9ZoXl8uqXVniaq1VNwlWAXihZbTvhqPPjA= +github.com/ncruces/go-sqlite3 v0.30.4/go.mod h1:7WR20VSC5IZusKhUdiR9y1NsUqnZgqIYCmKKoMEYg68= github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M= github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g= github.com/pganalyze/pg_query_go/v6 v6.1.0 h1:jG5ZLhcVgL1FAw4C/0VNQaVmX1SUJx71wBGdtTtBvls= @@ -152,8 +152,8 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -169,10 +169,10 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tetratelabs/wazero v1.10.1 h1:2DugeJf6VVk58KTPszlNfeeN8AhhpwcZqkJj2wwFuH8= -github.com/tetratelabs/wazero v1.10.1/go.mod h1:DRm5twOQ5Gr1AoEdSi0CLjDQF1J9ZAuyqFIjl1KKfQU= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= +github.com/tetratelabs/wazero v1.11.0/go.mod h1:eV28rsN8Q+xwjogd7f4/Pp4xFxO7uOGbLcD/LzB1wiU= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 h1:mJdDDPblDfPe7z7go8Dvv1AJQDI3eQ/5xith3q2mFlo= github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07/go.mod h1:Ak17IJ037caFp4jpCw/iQQ7/W74Sqpb1YuKJU6HTKfM= github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= @@ -221,6 +221,7 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -230,8 +231,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201203163018-be400aefbc4c/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= +golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -245,8 +246,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= -golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -259,8 +260,8 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -268,8 +269,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= +golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= +golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -287,16 +288,16 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8 h1:mepRgnBZa07I4TRuomDE4sTIYieg/osKmzIf4USdWS4= -google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= -google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda h1:+2XxjfsAu6vqFxwGBRcHiMaDCuZiqXGDUDVWVtrFAnE= +google.golang.org/genproto/googleapis/api v0.0.0-20251029180050-ab9386a59fda/go.mod h1:fDMmzKV90WSg1NbozdqrE64fkuTv6mlq2zxo9ad+3yo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/internal/analyzer/analyzer.go b/internal/analyzer/analyzer.go index 3d7e3a0287..674f283db9 100644 --- a/internal/analyzer/analyzer.go +++ b/internal/analyzer/analyzer.go @@ -110,7 +110,21 @@ func (c *CachedAnalyzer) Close(ctx context.Context) error { return c.a.Close(ctx) } +func (c *CachedAnalyzer) EnsureConn(ctx context.Context, migrations []string) error { + return c.a.EnsureConn(ctx, migrations) +} + +func (c *CachedAnalyzer) GetColumnNames(ctx context.Context, query string) ([]string, error) { + return c.a.GetColumnNames(ctx, query) +} + type Analyzer interface { Analyze(context.Context, ast.Node, string, []string, *named.ParamSet) (*analysis.Analysis, error) Close(context.Context) error + // EnsureConn initializes the database connection with the given migrations. + // This is required for database-only mode where we need to connect before analyzing queries. + EnsureConn(ctx context.Context, migrations []string) error + // GetColumnNames returns the column names for a query by preparing it against the database. + // This is used for star expansion in database-only mode. + GetColumnNames(ctx context.Context, query string) ([]string, error) } diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 93fd6bbeaa..80a167353e 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -30,6 +30,7 @@ func init() { initCmd.Flags().BoolP("v1", "", false, "generate v1 config yaml file") initCmd.Flags().BoolP("v2", "", true, "generate v2 config yaml file") initCmd.MarkFlagsMutuallyExclusive("v1", "v2") + parseCmd.Flags().StringP("dialect", "d", "", "SQL dialect to use (postgresql, mysql, or sqlite)") } // Do runs the command logic. @@ -44,6 +45,7 @@ func Do(args []string, stdin io.Reader, stdout io.Writer, stderr io.Writer) int rootCmd.AddCommand(diffCmd) rootCmd.AddCommand(genCmd) rootCmd.AddCommand(initCmd) + rootCmd.AddCommand(parseCmd) rootCmd.AddCommand(versionCmd) rootCmd.AddCommand(verifyCmd) rootCmd.AddCommand(pushCmd) @@ -136,10 +138,11 @@ var initCmd = &cobra.Command{ } type Env struct { - DryRun bool - Debug opts.Debug - Remote bool - NoRemote bool + DryRun bool + Debug opts.Debug + Experiment opts.Experiment + Remote bool + NoRemote bool } func ParseEnv(c *cobra.Command) Env { @@ -147,10 +150,11 @@ func ParseEnv(c *cobra.Command) Env { r := c.Flag("remote") nr := c.Flag("no-remote") return Env{ - DryRun: dr != nil && dr.Changed, - Debug: opts.DebugFromEnv(), - Remote: r != nil && r.Value.String() == "true", - NoRemote: nr != nil && nr.Value.String() == "true", + DryRun: dr != nil && dr.Changed, + Debug: opts.DebugFromEnv(), + Experiment: opts.ExperimentFromEnv(), + Remote: r != nil && r.Value.String() == "true", + NoRemote: nr != nil && nr.Value.String() == "true", } } diff --git a/internal/cmd/generate.go b/internal/cmd/generate.go index 00e8871c7e..05b5445ebb 100644 --- a/internal/cmd/generate.go +++ b/internal/cmd/generate.go @@ -295,7 +295,7 @@ func remoteGenerate(ctx context.Context, configPath string, conf *config.Config, func parse(ctx context.Context, name, dir string, sql config.SQL, combo config.CombinedSettings, parserOpts opts.Parser, stderr io.Writer) (*compiler.Result, bool) { defer trace.StartRegion(ctx, "parse").End() - c, err := compiler.NewCompiler(sql, combo) + c, err := compiler.NewCompiler(sql, combo, parserOpts) defer func() { if c != nil { c.Close(ctx) diff --git a/internal/cmd/parse.go b/internal/cmd/parse.go new file mode 100644 index 0000000000..b9e26c072e --- /dev/null +++ b/internal/cmd/parse.go @@ -0,0 +1,94 @@ +package cmd + +import ( + "encoding/json" + "fmt" + "io" + "os" + + "github.com/spf13/cobra" + + "github.com/sqlc-dev/sqlc/internal/engine/dolphin" + "github.com/sqlc-dev/sqlc/internal/engine/postgresql" + "github.com/sqlc-dev/sqlc/internal/engine/sqlite" + "github.com/sqlc-dev/sqlc/internal/sql/ast" +) + +var parseCmd = &cobra.Command{ + Use: "parse [file]", + Short: "Parse SQL and output the AST as JSON", + Long: `Parse SQL from a file or stdin and output the abstract syntax tree as JSON. + +Examples: + # Parse a SQL file with PostgreSQL dialect + sqlc parse --dialect postgresql schema.sql + + # Parse from stdin with MySQL dialect + echo "SELECT * FROM users" | sqlc parse --dialect mysql + + # Parse SQLite SQL + sqlc parse --dialect sqlite queries.sql`, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + dialect, err := cmd.Flags().GetString("dialect") + if err != nil { + return err + } + if dialect == "" { + return fmt.Errorf("--dialect flag is required (postgresql, mysql, or sqlite)") + } + + // Determine input source + var input io.Reader + if len(args) == 1 { + file, err := os.Open(args[0]) + if err != nil { + return fmt.Errorf("failed to open file: %w", err) + } + defer file.Close() + input = file + } else { + // Check if stdin has data + stat, err := os.Stdin.Stat() + if err != nil { + return fmt.Errorf("failed to stat stdin: %w", err) + } + if (stat.Mode() & os.ModeCharDevice) != 0 { + return fmt.Errorf("no input provided. Specify a file path or pipe SQL via stdin") + } + input = cmd.InOrStdin() + } + + // Parse SQL based on dialect + var stmts []ast.Statement + switch dialect { + case "postgresql", "postgres", "pg": + parser := postgresql.NewParser() + stmts, err = parser.Parse(input) + case "mysql": + parser := dolphin.NewParser() + stmts, err = parser.Parse(input) + case "sqlite": + parser := sqlite.NewParser() + stmts, err = parser.Parse(input) + default: + return fmt.Errorf("unsupported dialect: %s (use postgresql, mysql, or sqlite)", dialect) + } + if err != nil { + return fmt.Errorf("parse error: %w", err) + } + + // Output AST as JSON + stdout := cmd.OutOrStdout() + encoder := json.NewEncoder(stdout) + encoder.SetIndent("", " ") + + for _, stmt := range stmts { + if err := encoder.Encode(stmt.Raw); err != nil { + return fmt.Errorf("failed to encode AST: %w", err) + } + } + + return nil + }, +} diff --git a/internal/codegen/golang/result.go b/internal/codegen/golang/result.go index 515d0a654f..0820488f9d 100644 --- a/internal/codegen/golang/result.go +++ b/internal/codegen/golang/result.go @@ -141,9 +141,7 @@ func newGoEmbed(embed *plugin.Identifier, structs []Struct, defaultSchema string } fields := make([]Field, len(s.Fields)) - for i, f := range s.Fields { - fields[i] = f - } + copy(fields, s.Fields) return &goEmbed{ modelType: s.Name, diff --git a/internal/compiler/compile.go b/internal/compiler/compile.go index 84fbb20a3c..1a95b586f4 100644 --- a/internal/compiler/compile.go +++ b/internal/compiler/compile.go @@ -1,6 +1,7 @@ package compiler import ( + "context" "errors" "fmt" "io" @@ -39,11 +40,20 @@ func (c *Compiler) parseCatalog(schemas []string) error { } contents := migrations.RemoveRollbackStatements(string(blob)) c.schema = append(c.schema, contents) + + // In database-only mode, we parse the schema to validate syntax + // but don't update the catalog - the database will be the source of truth stmts, err := c.parser.Parse(strings.NewReader(contents)) if err != nil { merr.Add(filename, contents, 0, err) continue } + + // Skip catalog updates in database-only mode + if c.databaseOnlyMode { + continue + } + for i := range stmts { if err := c.catalog.Update(stmts[i], c); err != nil { merr.Add(filename, contents, stmts[i].Pos(), err) @@ -58,6 +68,15 @@ func (c *Compiler) parseCatalog(schemas []string) error { } func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) { + ctx := context.Background() + + // In database-only mode, initialize the database connection before parsing queries + if c.databaseOnlyMode && c.analyzer != nil { + if err := c.analyzer.EnsureConn(ctx, c.schema); err != nil { + return nil, fmt.Errorf("failed to initialize database connection: %w", err) + } + } + var q []*Query merr := multierr.New() set := map[string]struct{}{} @@ -113,6 +132,7 @@ func (c *Compiler) parseQueries(o opts.Parser) (*Result, error) { if len(q) == 0 { return nil, fmt.Errorf("no queries contained in paths %s", strings.Join(c.conf.Queries, ",")) } + return &Result{ Catalog: c.catalog, Queries: q, diff --git a/internal/compiler/engine.go b/internal/compiler/engine.go index 75749cd6df..64fdf3d5c7 100644 --- a/internal/compiler/engine.go +++ b/internal/compiler/engine.go @@ -14,6 +14,7 @@ import ( sqliteanalyze "github.com/sqlc-dev/sqlc/internal/engine/sqlite/analyzer" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/sql/catalog" + "github.com/sqlc-dev/sqlc/internal/x/expander" ) type Compiler struct { @@ -27,9 +28,15 @@ type Compiler struct { selector selector schema []string + + // databaseOnlyMode indicates that the compiler should use database-only analysis + // and skip building the internal catalog from schema files (analyzer.database: only) + databaseOnlyMode bool + // expander is used to expand SELECT * and RETURNING * in database-only mode + expander *expander.Expander } -func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, error) { +func NewCompiler(conf config.SQL, combo config.CombinedSettings, parserOpts opts.Parser) (*Compiler, error) { c := &Compiler{conf: conf, combo: combo} if conf.Database != nil && conf.Database.Managed { @@ -37,13 +44,33 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err c.client = client } + // Check for database-only mode (analyzer.database: only) + // This feature requires the analyzerv2 experiment to be enabled + databaseOnlyMode := conf.Analyzer.Database.IsOnly() && parserOpts.Experiment.AnalyzerV2 + switch conf.Engine { case config.EngineSQLite: - c.parser = sqlite.NewParser() + parser := sqlite.NewParser() + c.parser = parser c.catalog = sqlite.NewCatalog() c.selector = newSQLiteSelector() - if conf.Database != nil { - if conf.Analyzer.Database == nil || *conf.Analyzer.Database { + + if databaseOnlyMode { + // Database-only mode requires a database connection + if conf.Database == nil { + return nil, fmt.Errorf("analyzer.database: only requires database configuration") + } + if conf.Database.URI == "" && !conf.Database.Managed { + return nil, fmt.Errorf("analyzer.database: only requires database.uri or database.managed") + } + c.databaseOnlyMode = true + // Create the SQLite analyzer (implements Analyzer interface) + sqliteAnalyzer := sqliteanalyze.New(*conf.Database) + c.analyzer = analyzer.Cached(sqliteAnalyzer, combo.Global, *conf.Database) + // Create the expander using the analyzer as the column getter + c.expander = expander.New(c.analyzer, parser, parser) + } else if conf.Database != nil { + if conf.Analyzer.Database.IsEnabled() { c.analyzer = analyzer.Cached( sqliteanalyze.New(*conf.Database), combo.Global, @@ -56,11 +83,27 @@ func NewCompiler(conf config.SQL, combo config.CombinedSettings) (*Compiler, err c.catalog = dolphin.NewCatalog() c.selector = newDefaultSelector() case config.EnginePostgreSQL: - c.parser = postgresql.NewParser() + parser := postgresql.NewParser() + c.parser = parser c.catalog = postgresql.NewCatalog() c.selector = newDefaultSelector() - if conf.Database != nil { - if conf.Analyzer.Database == nil || *conf.Analyzer.Database { + + if databaseOnlyMode { + // Database-only mode requires a database connection + if conf.Database == nil { + return nil, fmt.Errorf("analyzer.database: only requires database configuration") + } + if conf.Database.URI == "" && !conf.Database.Managed { + return nil, fmt.Errorf("analyzer.database: only requires database.uri or database.managed") + } + c.databaseOnlyMode = true + // Create the PostgreSQL analyzer (implements Analyzer interface) + pgAnalyzer := pganalyze.New(c.client, *conf.Database) + c.analyzer = analyzer.Cached(pgAnalyzer, combo.Global, *conf.Database) + // Create the expander using the analyzer as the column getter + c.expander = expander.New(c.analyzer, parser, parser) + } else if conf.Database != nil { + if conf.Analyzer.Database.IsEnabled() { c.analyzer = analyzer.Cached( pganalyze.New(c.client, *conf.Database), combo.Global, diff --git a/internal/compiler/parse.go b/internal/compiler/parse.go index 681d291122..751cb3271a 100644 --- a/internal/compiler/parse.go +++ b/internal/compiler/parse.go @@ -71,7 +71,56 @@ func (c *Compiler) parseQuery(stmt ast.Node, src string, o opts.Parser) (*Query, } var anlys *analysis - if c.analyzer != nil { + if c.databaseOnlyMode && c.expander != nil { + // In database-only mode, use the expander for star expansion + // and rely entirely on the database analyzer for type resolution + expandedQuery, err := c.expander.Expand(ctx, rawSQL) + if err != nil { + return nil, fmt.Errorf("star expansion failed: %w", err) + } + + // Parse named parameters from the expanded query + expandedStmts, err := c.parser.Parse(strings.NewReader(expandedQuery)) + if err != nil { + return nil, fmt.Errorf("parsing expanded query failed: %w", err) + } + if len(expandedStmts) == 0 { + return nil, errors.New("no statements in expanded query") + } + expandedRaw := expandedStmts[0].Raw + + // Use the analyzer to get type information from the database + result, err := c.analyzer.Analyze(ctx, expandedRaw, expandedQuery, c.schema, nil) + if err != nil { + return nil, err + } + + // Convert the analyzer result to the internal analysis format + var cols []*Column + for _, col := range result.Columns { + cols = append(cols, convertColumn(col)) + } + var params []Parameter + for _, p := range result.Params { + params = append(params, Parameter{ + Number: int(p.Number), + Column: convertColumn(p.Column), + }) + } + + // Determine the insert table if applicable + var table *ast.TableName + if insert, ok := expandedRaw.Stmt.(*ast.InsertStmt); ok { + table, _ = ParseTableName(insert.Relation) + } + + anlys = &analysis{ + Table: table, + Columns: cols, + Parameters: params, + Query: expandedQuery, + } + } else if c.analyzer != nil { inference, _ := c.inferQuery(raw, rawSQL) if inference == nil { inference = &analysis{} diff --git a/internal/config/config.go b/internal/config/config.go index 0ff805fccd..d3e610ef05 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -122,8 +122,75 @@ type SQL struct { Analyzer Analyzer `json:"analyzer" yaml:"analyzer"` } +// AnalyzerDatabase represents the database analyzer setting. +// It can be a boolean (true/false) or the string "only" for database-only mode. +type AnalyzerDatabase struct { + value *bool // nil means not set, true/false for boolean values + isOnly bool // true when set to "only" +} + +// IsEnabled returns true if the database analyzer should be used. +// Returns true for both `true` and `"only"` settings. +func (a AnalyzerDatabase) IsEnabled() bool { + if a.isOnly { + return true + } + return a.value == nil || *a.value +} + +// IsOnly returns true if the analyzer is set to "only" mode. +func (a AnalyzerDatabase) IsOnly() bool { + return a.isOnly +} + +func (a *AnalyzerDatabase) UnmarshalJSON(data []byte) error { + // Try to unmarshal as boolean first + var b bool + if err := json.Unmarshal(data, &b); err == nil { + a.value = &b + a.isOnly = false + return nil + } + + // Try to unmarshal as string + var s string + if err := json.Unmarshal(data, &s); err == nil { + if s == "only" { + a.isOnly = true + a.value = nil + return nil + } + return errors.New("analyzer.database must be true, false, or \"only\"") + } + + return errors.New("analyzer.database must be true, false, or \"only\"") +} + +func (a *AnalyzerDatabase) UnmarshalYAML(unmarshal func(interface{}) error) error { + // Try to unmarshal as boolean first + var b bool + if err := unmarshal(&b); err == nil { + a.value = &b + a.isOnly = false + return nil + } + + // Try to unmarshal as string + var s string + if err := unmarshal(&s); err == nil { + if s == "only" { + a.isOnly = true + a.value = nil + return nil + } + return errors.New("analyzer.database must be true, false, or \"only\"") + } + + return errors.New("analyzer.database must be true, false, or \"only\"") +} + type Analyzer struct { - Database *bool `json:"database" yaml:"database"` + Database AnalyzerDatabase `json:"database" yaml:"database"` } // TODO: Figure out a better name for this diff --git a/internal/config/v_one.json b/internal/config/v_one.json index a0667a7c9c..e5ce9ec549 100644 --- a/internal/config/v_one.json +++ b/internal/config/v_one.json @@ -79,7 +79,10 @@ "type": "object", "properties": { "database": { - "type": "boolean" + "oneOf": [ + {"type": "boolean"}, + {"const": "only"} + ] } } }, diff --git a/internal/config/v_two.json b/internal/config/v_two.json index acf914997d..22591d7335 100644 --- a/internal/config/v_two.json +++ b/internal/config/v_two.json @@ -82,7 +82,10 @@ "type": "object", "properties": { "database": { - "type": "boolean" + "oneOf": [ + {"type": "boolean"}, + {"const": "only"} + ] } } }, diff --git a/internal/endtoend/CLAUDE.md b/internal/endtoend/CLAUDE.md new file mode 100644 index 0000000000..b9c995c9df --- /dev/null +++ b/internal/endtoend/CLAUDE.md @@ -0,0 +1,117 @@ +# End-to-End Tests - Native Database Setup + +This document describes how to set up MySQL and PostgreSQL for running end-to-end tests in environments without Docker, particularly when using an HTTP proxy. + +## Overview + +The end-to-end tests support three methods for connecting to databases: + +1. **Environment Variables**: Set `POSTGRESQL_SERVER_URI` and `MYSQL_SERVER_URI` directly +2. **Docker**: Automatically starts containers via the docker package +3. **Native Installation**: Starts existing database services on Linux + +## Installing Databases with HTTP Proxy + +In environments where DNS doesn't work directly but an HTTP proxy is available (e.g., some CI environments), you need to configure apt to use the proxy before installing packages. + +### Configure apt Proxy + +```bash +# Check if HTTP_PROXY is set +echo $HTTP_PROXY + +# Configure apt to use the proxy +sudo tee /etc/apt/apt.conf.d/99proxy << EOF +Acquire::http::Proxy "$HTTP_PROXY"; +Acquire::https::Proxy "$HTTPS_PROXY"; +EOF + +# Update package lists +sudo apt-get update -qq +``` + +### Install PostgreSQL + +```bash +# Install PostgreSQL +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y postgresql postgresql-contrib + +# Start the service +sudo service postgresql start + +# Set password for postgres user +sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + +# Configure pg_hba.conf for password authentication +# Find the hba_file location: +sudo -u postgres psql -t -c "SHOW hba_file;" + +# Add md5 authentication for localhost (add to the beginning of pg_hba.conf): +# host all all 127.0.0.1/32 md5 + +# Reload PostgreSQL +sudo service postgresql reload +``` + +### Install MySQL + +```bash +# Pre-configure MySQL root password +echo "mysql-server mysql-server/root_password password mysecretpassword" | sudo debconf-set-selections +echo "mysql-server mysql-server/root_password_again password mysecretpassword" | sudo debconf-set-selections + +# Install MySQL +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y mysql-server + +# Start the service +sudo service mysql start + +# Verify connection +mysql -uroot -pmysecretpassword -e "SELECT 1;" +``` + +## Expected Database Credentials + +The native database support expects the following credentials: + +### PostgreSQL +- **URI**: `postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable` +- **User**: `postgres` +- **Password**: `postgres` +- **Port**: `5432` + +### MySQL +- **URI**: `root:mysecretpassword@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true` +- **User**: `root` +- **Password**: `mysecretpassword` +- **Port**: `3306` + +## Running Tests + +```bash +# Run end-to-end tests +go test -v -run TestReplay -timeout 20m ./internal/endtoend/... + +# With verbose logging +go test -v -run TestReplay -timeout 20m ./internal/endtoend/... 2>&1 | tee test.log +``` + +## Troubleshooting + +### apt-get times out or fails +- Ensure HTTP proxy is configured in `/etc/apt/apt.conf.d/99proxy` +- Check that the proxy URL is correct: `echo $HTTP_PROXY` +- Try running `sudo apt-get update` first to verify connectivity + +### MySQL connection refused +- Check if MySQL is running: `sudo service mysql status` +- Verify the password: `mysql -uroot -pmysecretpassword -e "SELECT 1;"` +- Check if MySQL is listening on TCP: `netstat -tlnp | grep 3306` + +### PostgreSQL authentication failed +- Verify pg_hba.conf has md5 authentication for localhost +- Check password: `PGPASSWORD=postgres psql -h localhost -U postgres -c "SELECT 1;"` +- Reload PostgreSQL after config changes: `sudo service postgresql reload` + +### DNS resolution fails +This is expected in some environments. Configure apt proxy as shown above. diff --git a/internal/endtoend/endtoend_test.go b/internal/endtoend/endtoend_test.go index 537307e453..7634918446 100644 --- a/internal/endtoend/endtoend_test.go +++ b/internal/endtoend/endtoend_test.go @@ -18,6 +18,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/config" "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/sqltest/docker" + "github.com/sqlc-dev/sqlc/internal/sqltest/native" ) func lineEndings() cmp.Option { @@ -113,23 +114,63 @@ func TestReplay(t *testing.T) { ctx := context.Background() var mysqlURI, postgresURI string - if err := docker.Installed(); err == nil { - { - host, err := docker.StartPostgreSQLServer(ctx) - if err != nil { - t.Fatalf("starting postgresql failed: %s", err) + + // First, check environment variables + if uri := os.Getenv("POSTGRESQL_SERVER_URI"); uri != "" { + postgresURI = uri + } + if uri := os.Getenv("MYSQL_SERVER_URI"); uri != "" { + mysqlURI = uri + } + + // Try Docker for any missing databases + if postgresURI == "" || mysqlURI == "" { + if err := docker.Installed(); err == nil { + if postgresURI == "" { + host, err := docker.StartPostgreSQLServer(ctx) + if err != nil { + t.Logf("docker postgresql startup failed: %s", err) + } else { + postgresURI = host + } + } + if mysqlURI == "" { + host, err := docker.StartMySQLServer(ctx) + if err != nil { + t.Logf("docker mysql startup failed: %s", err) + } else { + mysqlURI = host + } } - postgresURI = host } - { - host, err := docker.StartMySQLServer(ctx) - if err != nil { - t.Fatalf("starting mysql failed: %s", err) + } + + // Try native installation for any missing databases (Linux only) + if postgresURI == "" || mysqlURI == "" { + if err := native.Supported(); err == nil { + if postgresURI == "" { + host, err := native.StartPostgreSQLServer(ctx) + if err != nil { + t.Logf("native postgresql startup failed: %s", err) + } else { + postgresURI = host + } + } + if mysqlURI == "" { + host, err := native.StartMySQLServer(ctx) + if err != nil { + t.Logf("native mysql startup failed: %s", err) + } else { + mysqlURI = host + } } - mysqlURI = host } } + // Log which databases are available + t.Logf("PostgreSQL available: %v (URI: %s)", postgresURI != "", postgresURI) + t.Logf("MySQL available: %v (URI: %s)", mysqlURI != "", mysqlURI) + contexts := map[string]textContext{ "base": { Mutate: func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) {} }, @@ -138,19 +179,20 @@ func TestReplay(t *testing.T) { "managed-db": { Mutate: func(t *testing.T, path string) func(*config.Config) { return func(c *config.Config) { + // Add all servers - tests will fail if database isn't available c.Servers = []config.Server{ { Name: "postgres", Engine: config.EnginePostgreSQL, URI: postgresURI, }, - { Name: "mysql", Engine: config.EngineMySQL, URI: mysqlURI, }, } + for i := range c.SQL { switch c.SQL[i].Engine { case config.EnginePostgreSQL: @@ -172,8 +214,8 @@ func TestReplay(t *testing.T) { } }, Enabled: func() bool { - err := docker.Installed() - return err == nil + // Enabled if at least one database URI is available + return postgresURI != "" || mysqlURI != "" }, }, } @@ -221,8 +263,9 @@ func TestReplay(t *testing.T) { opts := cmd.Options{ Env: cmd.Env{ - Debug: opts.DebugFromString(args.Env["SQLCDEBUG"]), - NoRemote: true, + Debug: opts.DebugFromString(args.Env["SQLCDEBUG"]), + Experiment: opts.ExperimentFromString(args.Env["SQLCEXPERIMENT"]), + NoRemote: true, }, Stderr: &stderr, MutateConfig: testctx.Mutate(t, path), diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..aaf587c793 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/exec.json @@ -0,0 +1,6 @@ +{ + "contexts": ["managed-db"], + "env": { + "SQLCEXPERIMENT": "analyzerv2" + } +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..90b88c3389 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/models.go @@ -0,0 +1,11 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +type Product struct { + ID int32 + Name string + Price string +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..8d31d41cdf --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,65 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" +) + +const getProductStats = `-- name: GetProductStats :one +WITH product_stats AS ( + SELECT COUNT(*) as total, AVG(price) as avg_price FROM products +) +SELECT total, avg_price FROM product_stats +` + +type GetProductStatsRow struct { + Total int64 + AvgPrice string +} + +func (q *Queries) GetProductStats(ctx context.Context) (GetProductStatsRow, error) { + row := q.db.QueryRowContext(ctx, getProductStats) + var i GetProductStatsRow + err := row.Scan(&i.Total, &i.AvgPrice) + return i, err +} + +const listExpensiveProducts = `-- name: ListExpensiveProducts :many +WITH expensive AS ( + SELECT id, name, price FROM products WHERE price > 100 +) +SELECT id, name, price FROM expensive +` + +type ListExpensiveProductsRow struct { + ID int32 + Name string + Price string +} + +func (q *Queries) ListExpensiveProducts(ctx context.Context) ([]ListExpensiveProductsRow, error) { + rows, err := q.db.QueryContext(ctx, listExpensiveProducts) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListExpensiveProductsRow + for rows.Next() { + var i ListExpensiveProductsRow + if err := rows.Scan(&i.ID, &i.Name, &i.Price); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..4626fe0f04 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/query.sql @@ -0,0 +1,11 @@ +-- name: ListExpensiveProducts :many +WITH expensive AS ( + SELECT * FROM products WHERE price > 100 +) +SELECT * FROM expensive; + +-- name: GetProductStats :one +WITH product_stats AS ( + SELECT COUNT(*) as total, AVG(price) as avg_price FROM products +) +SELECT * FROM product_stats; diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..17aaa6e650 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE products ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + price NUMERIC(10,2) NOT NULL +); diff --git a/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..629b01dea6 --- /dev/null +++ b/internal/endtoend/testdata/accurate_cte/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,13 @@ +version: "2" +sql: + - engine: postgresql + schema: "schema.sql" + queries: "query.sql" + database: + managed: true + analyzer: + database: "only" + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..aaf587c793 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/exec.json @@ -0,0 +1,6 @@ +{ + "contexts": ["managed-db"], + "env": { + "SQLCEXPERIMENT": "analyzerv2" + } +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..2b42787339 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/models.go @@ -0,0 +1,59 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "database/sql/driver" + "fmt" +) + +type Status string + +const ( + StatusPending Status = "pending" + StatusActive Status = "active" + StatusCompleted Status = "completed" +) + +func (e *Status) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = Status(s) + case string: + *e = Status(s) + default: + return fmt.Errorf("unsupported scan type for Status: %T", src) + } + return nil +} + +type NullStatus struct { + Status Status + Valid bool // Valid is true if Status is not NULL +} + +// Scan implements the Scanner interface. +func (ns *NullStatus) Scan(value interface{}) error { + if value == nil { + ns.Status, ns.Valid = "", false + return nil + } + ns.Valid = true + return ns.Status.Scan(value) +} + +// Value implements the driver Valuer interface. +func (ns NullStatus) Value() (driver.Value, error) { + if !ns.Valid { + return nil, nil + } + return string(ns.Status), nil +} + +type Task struct { + ID int32 + Title string + Status Status +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..263a6b6736 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,80 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" +) + +const createTask = `-- name: CreateTask :one +INSERT INTO tasks (title, status) VALUES ($1, $2) RETURNING id, title, status +` + +type CreateTaskParams struct { + Title string + Status Status +} + +func (q *Queries) CreateTask(ctx context.Context, arg CreateTaskParams) (Task, error) { + row := q.db.QueryRowContext(ctx, createTask, arg.Title, arg.Status) + var i Task + err := row.Scan(&i.ID, &i.Title, &i.Status) + return i, err +} + +const getTasksByStatus = `-- name: GetTasksByStatus :many +SELECT id, title, status FROM tasks WHERE status = $1 +` + +func (q *Queries) GetTasksByStatus(ctx context.Context, status Status) ([]Task, error) { + rows, err := q.db.QueryContext(ctx, getTasksByStatus, status) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Task + for rows.Next() { + var i Task + if err := rows.Scan(&i.ID, &i.Title, &i.Status); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listTasks = `-- name: ListTasks :many +SELECT id, title, status FROM tasks +` + +func (q *Queries) ListTasks(ctx context.Context) ([]Task, error) { + rows, err := q.db.QueryContext(ctx, listTasks) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Task + for rows.Next() { + var i Task + if err := rows.Scan(&i.ID, &i.Title, &i.Status); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..11dcd9bf48 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/query.sql @@ -0,0 +1,8 @@ +-- name: ListTasks :many +SELECT * FROM tasks; + +-- name: GetTasksByStatus :many +SELECT * FROM tasks WHERE status = $1; + +-- name: CreateTask :one +INSERT INTO tasks (title, status) VALUES ($1, $2) RETURNING *; diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..443ae9845f --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/schema.sql @@ -0,0 +1,7 @@ +CREATE TYPE status AS ENUM ('pending', 'active', 'completed'); + +CREATE TABLE tasks ( + id SERIAL PRIMARY KEY, + title TEXT NOT NULL, + status status NOT NULL DEFAULT 'pending' +); diff --git a/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..629b01dea6 --- /dev/null +++ b/internal/endtoend/testdata/accurate_enum/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,13 @@ +version: "2" +sql: + - engine: postgresql + schema: "schema.sql" + queries: "query.sql" + database: + managed: true + analyzer: + database: "only" + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/exec.json b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/exec.json new file mode 100644 index 0000000000..aaf587c793 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/exec.json @@ -0,0 +1,6 @@ +{ + "contexts": ["managed-db"], + "env": { + "SQLCEXPERIMENT": "analyzerv2" + } +} diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/db.go b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/models.go b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/models.go new file mode 100644 index 0000000000..eaf05e5c00 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/models.go @@ -0,0 +1,15 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "database/sql" +) + +type Author struct { + ID int64 + Name string + Bio sql.NullString +} diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/query.sql.go new file mode 100644 index 0000000000..203224ead2 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/go/query.sql.go @@ -0,0 +1,65 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const createAuthor = `-- name: CreateAuthor :one +INSERT INTO authors (name, bio) VALUES (?, ?) RETURNING id, name, bio +` + +type CreateAuthorParams struct { + Name string + Bio sql.NullString +} + +func (q *Queries) CreateAuthor(ctx context.Context, arg CreateAuthorParams) (Author, error) { + row := q.db.QueryRowContext(ctx, createAuthor, arg.Name, arg.Bio) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const getAuthor = `-- name: GetAuthor :one +SELECT id, name, bio FROM authors WHERE id = ? +` + +func (q *Queries) GetAuthor(ctx context.Context, id int64) (Author, error) { + row := q.db.QueryRowContext(ctx, getAuthor, id) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const listAuthors = `-- name: ListAuthors :many +SELECT id, name, bio FROM authors +` + +func (q *Queries) ListAuthors(ctx context.Context) ([]Author, error) { + rows, err := q.db.QueryContext(ctx, listAuthors) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Author + for rows.Next() { + var i Author + if err := rows.Scan(&i.ID, &i.Name, &i.Bio); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/query.sql b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/query.sql new file mode 100644 index 0000000000..8fe23a8600 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/query.sql @@ -0,0 +1,8 @@ +-- name: GetAuthor :one +SELECT * FROM authors WHERE id = ?; + +-- name: ListAuthors :many +SELECT * FROM authors; + +-- name: CreateAuthor :one +INSERT INTO authors (name, bio) VALUES (?, ?) RETURNING *; diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/schema.sql b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/schema.sql new file mode 100644 index 0000000000..22fc0993c1 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE authors ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL, + bio TEXT +); diff --git a/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/sqlc.yaml new file mode 100644 index 0000000000..d2da6c31b2 --- /dev/null +++ b/internal/endtoend/testdata/accurate_sqlite/sqlite/stdlib/sqlc.yaml @@ -0,0 +1,13 @@ +version: "2" +sql: + - engine: sqlite + schema: "schema.sql" + queries: "query.sql" + database: + managed: true + analyzer: + database: "only" + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/exec.json b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/exec.json new file mode 100644 index 0000000000..aaf587c793 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/exec.json @@ -0,0 +1,6 @@ +{ + "contexts": ["managed-db"], + "env": { + "SQLCEXPERIMENT": "analyzerv2" + } +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/db.go b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/db.go new file mode 100644 index 0000000000..3b320aa168 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/db.go @@ -0,0 +1,31 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "context" + "database/sql" +) + +type DBTX interface { + ExecContext(context.Context, string, ...interface{}) (sql.Result, error) + PrepareContext(context.Context, string) (*sql.Stmt, error) + QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error) + QueryRowContext(context.Context, string, ...interface{}) *sql.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx *sql.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/models.go b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/models.go new file mode 100644 index 0000000000..ec1cb8d670 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/models.go @@ -0,0 +1,15 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 + +package querytest + +import ( + "database/sql" +) + +type Author struct { + ID int32 + Name string + Bio sql.NullString +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/query.sql.go b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/query.sql.go new file mode 100644 index 0000000000..9e2820cdbd --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/go/query.sql.go @@ -0,0 +1,93 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.30.0 +// source: query.sql + +package querytest + +import ( + "context" + "database/sql" +) + +const createAuthor = `-- name: CreateAuthor :one +INSERT INTO authors (name, bio) VALUES ($1, $2) RETURNING id, name, bio +` + +type CreateAuthorParams struct { + Name string + Bio sql.NullString +} + +func (q *Queries) CreateAuthor(ctx context.Context, arg CreateAuthorParams) (Author, error) { + row := q.db.QueryRowContext(ctx, createAuthor, arg.Name, arg.Bio) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const deleteAuthor = `-- name: DeleteAuthor :one +DELETE FROM authors WHERE id = $1 RETURNING id, name, bio +` + +func (q *Queries) DeleteAuthor(ctx context.Context, id int32) (Author, error) { + row := q.db.QueryRowContext(ctx, deleteAuthor, id) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const getAuthor = `-- name: GetAuthor :one +SELECT id, name, bio FROM authors WHERE id = $1 +` + +func (q *Queries) GetAuthor(ctx context.Context, id int32) (Author, error) { + row := q.db.QueryRowContext(ctx, getAuthor, id) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} + +const listAuthors = `-- name: ListAuthors :many +SELECT id, name, bio FROM authors +` + +func (q *Queries) ListAuthors(ctx context.Context) ([]Author, error) { + rows, err := q.db.QueryContext(ctx, listAuthors) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Author + for rows.Next() { + var i Author + if err := rows.Scan(&i.ID, &i.Name, &i.Bio); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const updateAuthor = `-- name: UpdateAuthor :one +UPDATE authors SET name = $1, bio = $2 WHERE id = $3 RETURNING id, name, bio +` + +type UpdateAuthorParams struct { + Name string + Bio sql.NullString + ID int32 +} + +func (q *Queries) UpdateAuthor(ctx context.Context, arg UpdateAuthorParams) (Author, error) { + row := q.db.QueryRowContext(ctx, updateAuthor, arg.Name, arg.Bio, arg.ID) + var i Author + err := row.Scan(&i.ID, &i.Name, &i.Bio) + return i, err +} diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/query.sql b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/query.sql new file mode 100644 index 0000000000..e091a5eaef --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/query.sql @@ -0,0 +1,14 @@ +-- name: ListAuthors :many +SELECT * FROM authors; + +-- name: GetAuthor :one +SELECT * FROM authors WHERE id = $1; + +-- name: CreateAuthor :one +INSERT INTO authors (name, bio) VALUES ($1, $2) RETURNING *; + +-- name: UpdateAuthor :one +UPDATE authors SET name = $1, bio = $2 WHERE id = $3 RETURNING *; + +-- name: DeleteAuthor :one +DELETE FROM authors WHERE id = $1 RETURNING *; diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/schema.sql b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/schema.sql new file mode 100644 index 0000000000..ca6ad1e2cf --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/schema.sql @@ -0,0 +1,5 @@ +CREATE TABLE authors ( + id SERIAL PRIMARY KEY, + name TEXT NOT NULL, + bio TEXT +); diff --git a/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/sqlc.yaml b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/sqlc.yaml new file mode 100644 index 0000000000..629b01dea6 --- /dev/null +++ b/internal/endtoend/testdata/accurate_star_expansion/postgresql/stdlib/sqlc.yaml @@ -0,0 +1,13 @@ +version: "2" +sql: + - engine: postgresql + schema: "schema.sql" + queries: "query.sql" + database: + managed: true + analyzer: + database: "only" + gen: + go: + package: "querytest" + out: "go" diff --git a/internal/engine/postgresql/analyzer/analyze.go b/internal/engine/postgresql/analyzer/analyze.go index 5a08fa98ec..ee03e4d3c5 100644 --- a/internal/engine/postgresql/analyzer/analyze.go +++ b/internal/engine/postgresql/analyzer/analyze.go @@ -17,6 +17,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/shfmt" "github.com/sqlc-dev/sqlc/internal/sql/ast" + "github.com/sqlc-dev/sqlc/internal/sql/catalog" "github.com/sqlc-dev/sqlc/internal/sql/named" "github.com/sqlc-dev/sqlc/internal/sql/sqlerr" ) @@ -320,3 +321,227 @@ func (a *Analyzer) Close(_ context.Context) error { } return nil } + +// SQL queries for schema introspection +const introspectTablesQuery = ` +SELECT + n.nspname AS schema_name, + c.relname AS table_name, + a.attname AS column_name, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS data_type, + a.attnotnull AS not_null, + a.attndims AS array_dims, + COALESCE( + (SELECT true FROM pg_index i + WHERE i.indrelid = c.oid + AND i.indisprimary + AND a.attnum = ANY(i.indkey)), + false + ) AS is_primary_key +FROM + pg_catalog.pg_class c +JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace +JOIN + pg_catalog.pg_attribute a ON a.attrelid = c.oid +WHERE + c.relkind IN ('r', 'v', 'p') -- tables, views, partitioned tables + AND a.attnum > 0 -- skip system columns + AND NOT a.attisdropped + AND n.nspname = ANY($1) +ORDER BY + n.nspname, c.relname, a.attnum +` + +const introspectEnumsQuery = ` +SELECT + n.nspname AS schema_name, + t.typname AS type_name, + e.enumlabel AS enum_value +FROM + pg_catalog.pg_type t +JOIN + pg_catalog.pg_namespace n ON n.oid = t.typnamespace +JOIN + pg_catalog.pg_enum e ON e.enumtypid = t.oid +WHERE + t.typtype = 'e' + AND n.nspname = ANY($1) +ORDER BY + n.nspname, t.typname, e.enumsortorder +` + +type introspectedColumn struct { + SchemaName string `db:"schema_name"` + TableName string `db:"table_name"` + ColumnName string `db:"column_name"` + DataType string `db:"data_type"` + NotNull bool `db:"not_null"` + ArrayDims int `db:"array_dims"` + IsPrimaryKey bool `db:"is_primary_key"` +} + +type introspectedEnum struct { + SchemaName string `db:"schema_name"` + TypeName string `db:"type_name"` + EnumValue string `db:"enum_value"` +} + +// IntrospectSchema queries the database to build a catalog containing +// tables, columns, and enum types for the specified schemas. +func (a *Analyzer) IntrospectSchema(ctx context.Context, schemas []string) (*catalog.Catalog, error) { + if a.pool == nil { + return nil, fmt.Errorf("database connection not initialized") + } + + c, err := a.pool.Acquire(ctx) + if err != nil { + return nil, err + } + defer c.Release() + + // Query tables and columns + rows, err := c.Query(ctx, introspectTablesQuery, schemas) + if err != nil { + return nil, fmt.Errorf("introspect tables: %w", err) + } + columns, err := pgx.CollectRows(rows, pgx.RowToStructByName[introspectedColumn]) + if err != nil { + return nil, fmt.Errorf("collect table rows: %w", err) + } + + // Query enums + enumRows, err := c.Query(ctx, introspectEnumsQuery, schemas) + if err != nil { + return nil, fmt.Errorf("introspect enums: %w", err) + } + enums, err := pgx.CollectRows(enumRows, pgx.RowToStructByName[introspectedEnum]) + if err != nil { + return nil, fmt.Errorf("collect enum rows: %w", err) + } + + // Build catalog + cat := &catalog.Catalog{ + DefaultSchema: "public", + SearchPath: schemas, + } + + // Create schema map for quick lookup + schemaMap := make(map[string]*catalog.Schema) + for _, schemaName := range schemas { + schema := &catalog.Schema{Name: schemaName} + cat.Schemas = append(cat.Schemas, schema) + schemaMap[schemaName] = schema + } + + // Group columns by table + tableMap := make(map[string]*catalog.Table) + for _, col := range columns { + key := col.SchemaName + "." + col.TableName + tbl, exists := tableMap[key] + if !exists { + tbl = &catalog.Table{ + Rel: &ast.TableName{ + Schema: col.SchemaName, + Name: col.TableName, + }, + } + tableMap[key] = tbl + if schema, ok := schemaMap[col.SchemaName]; ok { + schema.Tables = append(schema.Tables, tbl) + } + } + + dt, isArray, dims := parseType(col.DataType) + tbl.Columns = append(tbl.Columns, &catalog.Column{ + Name: col.ColumnName, + Type: ast.TypeName{Name: dt}, + IsNotNull: col.NotNull, + IsArray: isArray || col.ArrayDims > 0, + ArrayDims: max(dims, col.ArrayDims), + }) + } + + // Group enum values by type + enumMap := make(map[string]*catalog.Enum) + for _, e := range enums { + key := e.SchemaName + "." + e.TypeName + enum, exists := enumMap[key] + if !exists { + enum = &catalog.Enum{ + Name: e.TypeName, + } + enumMap[key] = enum + if schema, ok := schemaMap[e.SchemaName]; ok { + schema.Types = append(schema.Types, enum) + } + } + enum.Vals = append(enum.Vals, e.EnumValue) + } + + return cat, nil +} + +// EnsureConn initializes the database connection pool if not already done. +// This is useful for database-only mode where we need to connect before analyzing queries. +func (a *Analyzer) EnsureConn(ctx context.Context, migrations []string) error { + if a.pool != nil { + return nil + } + + var uri string + if a.db.Managed { + if a.client == nil { + return fmt.Errorf("client is nil") + } + edb, err := a.client.CreateDatabase(ctx, &dbmanager.CreateDatabaseRequest{ + Engine: "postgresql", + Migrations: migrations, + }) + if err != nil { + return err + } + uri = edb.Uri + } else if a.dbg.OnlyManagedDatabases { + return fmt.Errorf("database: connections disabled via SQLCDEBUG=databases=managed") + } else { + uri = a.replacer.Replace(a.db.URI) + } + + conf, err := pgxpool.ParseConfig(uri) + if err != nil { + return err + } + pool, err := pgxpool.NewWithConfig(ctx, conf) + if err != nil { + return err + } + a.pool = pool + return nil +} + +// GetColumnNames implements the expander.ColumnGetter interface. +// It prepares a query and returns the column names from the result set description. +func (a *Analyzer) GetColumnNames(ctx context.Context, query string) ([]string, error) { + if a.pool == nil { + return nil, fmt.Errorf("database connection not initialized") + } + + conn, err := a.pool.Acquire(ctx) + if err != nil { + return nil, err + } + defer conn.Release() + + desc, err := conn.Conn().Prepare(ctx, "", query) + if err != nil { + return nil, err + } + + columns := make([]string, len(desc.Fields)) + for i, field := range desc.Fields { + columns[i] = field.Name + } + + return columns, nil +} diff --git a/internal/engine/sqlite/analyzer/analyze.go b/internal/engine/sqlite/analyzer/analyze.go index 3b526816f0..3af9f99a30 100644 --- a/internal/engine/sqlite/analyzer/analyze.go +++ b/internal/engine/sqlite/analyzer/analyze.go @@ -14,6 +14,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/opts" "github.com/sqlc-dev/sqlc/internal/shfmt" "github.com/sqlc-dev/sqlc/internal/sql/ast" + "github.com/sqlc-dev/sqlc/internal/sql/catalog" "github.com/sqlc-dev/sqlc/internal/sql/named" "github.com/sqlc-dev/sqlc/internal/sql/sqlerr" ) @@ -182,6 +183,144 @@ func (a *Analyzer) Close(_ context.Context) error { return nil } +// EnsureConn initializes the database connection if not already done. +// This is useful for database-only mode where we need to connect before analyzing queries. +func (a *Analyzer) EnsureConn(ctx context.Context, migrations []string) error { + a.mu.Lock() + defer a.mu.Unlock() + + if a.conn != nil { + return nil + } + + var uri string + applyMigrations := a.db.Managed + if a.db.Managed { + // For managed databases, create an in-memory database + uri = ":memory:" + } else if a.dbg.OnlyManagedDatabases { + return fmt.Errorf("database: connections disabled via SQLCDEBUG=databases=managed") + } else { + uri = a.replacer.Replace(a.db.URI) + // For in-memory databases, we need to apply migrations since the database starts empty + if isInMemoryDatabase(uri) { + applyMigrations = true + } + } + + conn, err := sqlite3.Open(uri) + if err != nil { + return fmt.Errorf("failed to open sqlite database: %w", err) + } + a.conn = conn + + // Apply migrations for managed or in-memory databases + if applyMigrations { + for _, m := range migrations { + if len(strings.TrimSpace(m)) == 0 { + continue + } + if err := a.conn.Exec(m); err != nil { + a.conn.Close() + a.conn = nil + return fmt.Errorf("migration failed: %s: %w", m, err) + } + } + } + + return nil +} + +// GetColumnNames implements the expander.ColumnGetter interface. +// It prepares a query and returns the column names from the result set description. +func (a *Analyzer) GetColumnNames(ctx context.Context, query string) ([]string, error) { + a.mu.Lock() + defer a.mu.Unlock() + + if a.conn == nil { + return nil, fmt.Errorf("database connection not initialized") + } + + stmt, _, err := a.conn.Prepare(query) + if err != nil { + return nil, err + } + defer stmt.Close() + + colCount := stmt.ColumnCount() + columns := make([]string, colCount) + for i := 0; i < colCount; i++ { + columns[i] = stmt.ColumnName(i) + } + + return columns, nil +} + +// IntrospectSchema queries the database to build a catalog containing +// tables and columns for the database. +func (a *Analyzer) IntrospectSchema(ctx context.Context, schemas []string) (*catalog.Catalog, error) { + a.mu.Lock() + defer a.mu.Unlock() + + if a.conn == nil { + return nil, fmt.Errorf("database connection not initialized") + } + + // Build catalog + cat := &catalog.Catalog{ + DefaultSchema: "main", + } + + // Create default schema + mainSchema := &catalog.Schema{Name: "main"} + cat.Schemas = append(cat.Schemas, mainSchema) + + // Query tables from sqlite_master + stmt, _, err := a.conn.Prepare("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'") + if err != nil { + return nil, fmt.Errorf("introspect tables: %w", err) + } + + tableNames := []string{} + for stmt.Step() { + tableName := stmt.ColumnText(0) + tableNames = append(tableNames, tableName) + } + stmt.Close() + + // For each table, get column information using PRAGMA table_info + for _, tableName := range tableNames { + tbl := &catalog.Table{ + Rel: &ast.TableName{ + Name: tableName, + }, + } + + pragmaStmt, _, err := a.conn.Prepare(fmt.Sprintf("PRAGMA table_info('%s')", tableName)) + if err != nil { + return nil, fmt.Errorf("pragma table_info for %s: %w", tableName, err) + } + + for pragmaStmt.Step() { + // PRAGMA table_info returns: cid, name, type, notnull, dflt_value, pk + colName := pragmaStmt.ColumnText(1) + colType := pragmaStmt.ColumnText(2) + notNull := pragmaStmt.ColumnInt(3) != 0 + + tbl.Columns = append(tbl.Columns, &catalog.Column{ + Name: colName, + Type: ast.TypeName{Name: normalizeType(colType)}, + IsNotNull: notNull, + }) + } + pragmaStmt.Close() + + mainSchema.Tables = append(mainSchema.Tables, tbl) + } + + return cat, nil +} + // isInMemoryDatabase checks if a SQLite URI refers to an in-memory database func isInMemoryDatabase(uri string) bool { if uri == ":memory:" || uri == "" { diff --git a/internal/opts/experiment.go b/internal/opts/experiment.go new file mode 100644 index 0000000000..45a1c11e05 --- /dev/null +++ b/internal/opts/experiment.go @@ -0,0 +1,105 @@ +package opts + +import ( + "os" + "strings" +) + +// The SQLCEXPERIMENT variable controls experimental features within sqlc. It +// is a comma-separated list of experiment names. Experiment names can be +// prefixed with "no" to explicitly disable them. +// +// This is modeled after Go's GOEXPERIMENT environment variable. For more +// information, see https://pkg.go.dev/internal/goexperiment +// +// Available experiments: +// +// analyzerv2 - enables database-only analyzer mode +// +// Example usage: +// +// SQLCEXPERIMENT=foo,bar # enable foo and bar experiments +// SQLCEXPERIMENT=nofoo # explicitly disable foo experiment +// SQLCEXPERIMENT=foo,nobar # enable foo, disable bar + +// Experiment holds the state of all experimental features. +// Add new experiments as boolean fields to this struct. +type Experiment struct { + // AnalyzerV2 enables the database-only analyzer mode (analyzer.database: only) + // which uses the database for all type resolution instead of parsing schema files. + AnalyzerV2 bool +} + +// ExperimentFromEnv returns an Experiment initialized from the SQLCEXPERIMENT +// environment variable. +func ExperimentFromEnv() Experiment { + return ExperimentFromString(os.Getenv("SQLCEXPERIMENT")) +} + +// ExperimentFromString parses a comma-separated list of experiment names +// and returns an Experiment with the appropriate flags set. +// +// Experiment names can be prefixed with "no" to explicitly disable them. +// Unknown experiment names are silently ignored. +func ExperimentFromString(val string) Experiment { + e := Experiment{} + if val == "" { + return e + } + + for _, name := range strings.Split(val, ",") { + name = strings.TrimSpace(name) + if name == "" { + continue + } + + // Check if this is a negation (noFoo) + enabled := true + if strings.HasPrefix(strings.ToLower(name), "no") && len(name) > 2 { + // Could be a negation, check if the rest is a valid experiment + possibleExp := name[2:] + if isKnownExperiment(possibleExp) { + name = possibleExp + enabled = false + } + // If not a known experiment, treat "no..." as a potential experiment name itself + } + + setExperiment(&e, name, enabled) + } + + return e +} + +// isKnownExperiment returns true if the given name (case-insensitive) is a +// known experiment. +func isKnownExperiment(name string) bool { + switch strings.ToLower(name) { + case "analyzerv2": + return true + default: + return false + } +} + +// setExperiment sets the experiment flag with the given name to the given value. +func setExperiment(e *Experiment, name string, enabled bool) { + switch strings.ToLower(name) { + case "analyzerv2": + e.AnalyzerV2 = enabled + } +} + +// Enabled returns a slice of all enabled experiment names. +func (e Experiment) Enabled() []string { + var enabled []string + if e.AnalyzerV2 { + enabled = append(enabled, "analyzerv2") + } + return enabled +} + +// String returns a comma-separated list of enabled experiments. +func (e Experiment) String() string { + return strings.Join(e.Enabled(), ",") +} diff --git a/internal/opts/experiment_test.go b/internal/opts/experiment_test.go new file mode 100644 index 0000000000..e9a8618e89 --- /dev/null +++ b/internal/opts/experiment_test.go @@ -0,0 +1,176 @@ +package opts + +import "testing" + +func TestExperimentFromString(t *testing.T) { + tests := []struct { + name string + input string + want Experiment + }{ + { + name: "empty string", + input: "", + want: Experiment{}, + }, + { + name: "whitespace only", + input: " ", + want: Experiment{}, + }, + { + name: "unknown experiment", + input: "unknownexperiment", + want: Experiment{}, + }, + { + name: "multiple unknown experiments", + input: "foo,bar,baz", + want: Experiment{}, + }, + { + name: "unknown with no prefix", + input: "nounknown", + want: Experiment{}, + }, + { + name: "whitespace around experiments", + input: " foo , bar , baz ", + want: Experiment{}, + }, + { + name: "empty items in list", + input: "foo,,bar", + want: Experiment{}, + }, + { + name: "enable analyzerv2", + input: "analyzerv2", + want: Experiment{AnalyzerV2: true}, + }, + { + name: "disable analyzerv2", + input: "noanalyzerv2", + want: Experiment{AnalyzerV2: false}, + }, + { + name: "enable then disable analyzerv2", + input: "analyzerv2,noanalyzerv2", + want: Experiment{AnalyzerV2: false}, + }, + { + name: "analyzerv2 case insensitive", + input: "AnalyzerV2", + want: Experiment{AnalyzerV2: true}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ExperimentFromString(tt.input) + if got != tt.want { + t.Errorf("ExperimentFromString(%q) = %+v, want %+v", tt.input, got, tt.want) + } + }) + } +} + +func TestExperimentEnabled(t *testing.T) { + tests := []struct { + name string + exp Experiment + want []string + }{ + { + name: "no experiments enabled", + exp: Experiment{}, + want: nil, + }, + { + name: "analyzerv2 enabled", + exp: Experiment{AnalyzerV2: true}, + want: []string{"analyzerv2"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.exp.Enabled() + if len(got) != len(tt.want) { + t.Errorf("Experiment.Enabled() = %v, want %v", got, tt.want) + return + } + for i := range got { + if got[i] != tt.want[i] { + t.Errorf("Experiment.Enabled()[%d] = %q, want %q", i, got[i], tt.want[i]) + } + } + }) + } +} + +func TestExperimentString(t *testing.T) { + tests := []struct { + name string + exp Experiment + want string + }{ + { + name: "no experiments", + exp: Experiment{}, + want: "", + }, + { + name: "analyzerv2 enabled", + exp: Experiment{AnalyzerV2: true}, + want: "analyzerv2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := tt.exp.String() + if got != tt.want { + t.Errorf("Experiment.String() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestIsKnownExperiment(t *testing.T) { + tests := []struct { + name string + input string + want bool + }{ + { + name: "unknown experiment", + input: "unknown", + want: false, + }, + { + name: "empty string", + input: "", + want: false, + }, + { + name: "analyzerv2 lowercase", + input: "analyzerv2", + want: true, + }, + { + name: "analyzerv2 mixed case", + input: "AnalyzerV2", + want: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isKnownExperiment(tt.input) + if got != tt.want { + t.Errorf("isKnownExperiment(%q) = %v, want %v", tt.input, got, tt.want) + } + }) + } +} diff --git a/internal/opts/parser.go b/internal/opts/parser.go index d6fb399552..2059d4f6a1 100644 --- a/internal/opts/parser.go +++ b/internal/opts/parser.go @@ -1,5 +1,6 @@ package opts type Parser struct { - Debug Debug + Debug Debug + Experiment Experiment } diff --git a/internal/sql/ast/select_stmt.go b/internal/sql/ast/select_stmt.go index 8c3606dd4d..62e6f1c9cf 100644 --- a/internal/sql/ast/select_stmt.go +++ b/internal/sql/ast/select_stmt.go @@ -37,9 +37,16 @@ func (n *SelectStmt) Format(buf *TrackedBuffer, d format.Dialect) { } if items(n.ValuesLists) { - buf.WriteString("VALUES (") - buf.astFormat(n.ValuesLists, d) - buf.WriteString(")") + buf.WriteString("VALUES ") + // ValuesLists is a list of rows, where each row is a List of values + for i, row := range n.ValuesLists.Items { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString("(") + buf.astFormat(row, d) + buf.WriteString(")") + } return } diff --git a/internal/sqltest/local/mysql.go b/internal/sqltest/local/mysql.go index dedd3dfd78..05733f6e8b 100644 --- a/internal/sqltest/local/mysql.go +++ b/internal/sqltest/local/mysql.go @@ -14,6 +14,7 @@ import ( migrate "github.com/sqlc-dev/sqlc/internal/migrations" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" "github.com/sqlc-dev/sqlc/internal/sqltest/docker" + "github.com/sqlc-dev/sqlc/internal/sqltest/native" ) var mysqlSync sync.Once @@ -31,8 +32,15 @@ func MySQL(t *testing.T, migrations []string) string { t.Fatal(err) } dburi = u + } else if ierr := native.Supported(); ierr == nil { + // Fall back to native installation when Docker is not available + u, err := native.StartMySQLServer(ctx) + if err != nil { + t.Fatal(err) + } + dburi = u } else { - t.Skip("MYSQL_SERVER_URI is empty") + t.Skip("MYSQL_SERVER_URI is empty and neither Docker nor native installation is available") } } diff --git a/internal/sqltest/local/postgres.go b/internal/sqltest/local/postgres.go index feda4cf7ac..243a7133ab 100644 --- a/internal/sqltest/local/postgres.go +++ b/internal/sqltest/local/postgres.go @@ -16,6 +16,7 @@ import ( "github.com/sqlc-dev/sqlc/internal/pgx/poolcache" "github.com/sqlc-dev/sqlc/internal/sql/sqlpath" "github.com/sqlc-dev/sqlc/internal/sqltest/docker" + "github.com/sqlc-dev/sqlc/internal/sqltest/native" ) var flight singleflight.Group @@ -41,8 +42,15 @@ func postgreSQL(t *testing.T, migrations []string, rw bool) string { t.Fatal(err) } dburi = u + } else if ierr := native.Supported(); ierr == nil { + // Fall back to native installation when Docker is not available + u, err := native.StartPostgreSQLServer(ctx) + if err != nil { + t.Fatal(err) + } + dburi = u } else { - t.Skip("POSTGRESQL_SERVER_URI is empty") + t.Skip("POSTGRESQL_SERVER_URI is empty and neither Docker nor native installation is available") } } diff --git a/internal/sqltest/native/enabled.go b/internal/sqltest/native/enabled.go new file mode 100644 index 0000000000..e5e12ccd80 --- /dev/null +++ b/internal/sqltest/native/enabled.go @@ -0,0 +1,20 @@ +package native + +import ( + "fmt" + "os/exec" + "runtime" +) + +// Supported returns nil if native database installation is supported on this platform. +// Currently only Linux (Ubuntu/Debian) is supported. +func Supported() error { + if runtime.GOOS != "linux" { + return fmt.Errorf("native database installation only supported on linux, got %s", runtime.GOOS) + } + // Check if apt-get is available (Debian/Ubuntu) + if _, err := exec.LookPath("apt-get"); err != nil { + return fmt.Errorf("apt-get not found: %w", err) + } + return nil +} diff --git a/internal/sqltest/native/mysql.go b/internal/sqltest/native/mysql.go new file mode 100644 index 0000000000..69482bace6 --- /dev/null +++ b/internal/sqltest/native/mysql.go @@ -0,0 +1,203 @@ +package native + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "os/exec" + "time" + + _ "github.com/go-sql-driver/mysql" + "golang.org/x/sync/singleflight" +) + +var mysqlFlight singleflight.Group +var mysqlURI string + +// StartMySQLServer starts an existing MySQL installation natively (without Docker). +func StartMySQLServer(ctx context.Context) (string, error) { + if err := Supported(); err != nil { + return "", err + } + if mysqlURI != "" { + return mysqlURI, nil + } + value, err, _ := mysqlFlight.Do("mysql", func() (interface{}, error) { + uri, err := startMySQLServer(ctx) + if err != nil { + return "", err + } + mysqlURI = uri + return uri, nil + }) + if err != nil { + return "", err + } + data, ok := value.(string) + if !ok { + return "", fmt.Errorf("returned value was not a string") + } + return data, nil +} + +func startMySQLServer(ctx context.Context) (string, error) { + // Standard URI for test MySQL + uri := "root:mysecretpassword@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true" + + // Try to connect first - it might already be running + if err := waitForMySQL(ctx, uri, 500*time.Millisecond); err == nil { + slog.Info("native/mysql", "status", "already running") + return uri, nil + } + + // Also try without password (default MySQL installation) + uriNoPassword := "root@tcp(localhost:3306)/mysql?multiStatements=true&parseTime=true" + if err := waitForMySQL(ctx, uriNoPassword, 500*time.Millisecond); err == nil { + slog.Info("native/mysql", "status", "already running (no password)") + // MySQL is running without password, try to set one + if err := setMySQLPassword(ctx); err != nil { + slog.Debug("native/mysql", "set-password-error", err) + // Return without password if we can't set one + return uriNoPassword, nil + } + // Try again with password + if err := waitForMySQL(ctx, uri, 1*time.Second); err == nil { + return uri, nil + } + // If password didn't work, use no password + return uriNoPassword, nil + } + + // Try to start existing MySQL service (might be installed but not running) + if _, err := exec.LookPath("mysqld"); err == nil { + slog.Info("native/mysql", "status", "starting existing service") + if err := startMySQLService(); err != nil { + slog.Debug("native/mysql", "start-error", err) + } else { + // Wait for MySQL to be ready + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + // Try with password first + if err := waitForMySQL(waitCtx, uri, 15*time.Second); err == nil { + return uri, nil + } + + // Try without password + if err := waitForMySQL(waitCtx, uriNoPassword, 15*time.Second); err == nil { + if err := setMySQLPassword(ctx); err != nil { + slog.Debug("native/mysql", "set-password-error", err) + return uriNoPassword, nil + } + if err := waitForMySQL(ctx, uri, 1*time.Second); err == nil { + return uri, nil + } + return uriNoPassword, nil + } + } + } + + return "", fmt.Errorf("MySQL is not installed or could not be started") +} + +func startMySQLService() error { + // Try systemctl first + cmd := exec.Command("sudo", "systemctl", "start", "mysql") + if err := cmd.Run(); err == nil { + // Give MySQL time to fully initialize + time.Sleep(2 * time.Second) + return nil + } + + // Try mysqld + cmd = exec.Command("sudo", "systemctl", "start", "mysqld") + if err := cmd.Run(); err == nil { + time.Sleep(2 * time.Second) + return nil + } + + // Try service command + cmd = exec.Command("sudo", "service", "mysql", "start") + if err := cmd.Run(); err == nil { + time.Sleep(2 * time.Second) + return nil + } + + cmd = exec.Command("sudo", "service", "mysqld", "start") + if err := cmd.Run(); err == nil { + time.Sleep(2 * time.Second) + return nil + } + + return fmt.Errorf("could not start MySQL service") +} + +func setMySQLPassword(ctx context.Context) error { + // Connect without password + db, err := sql.Open("mysql", "root@tcp(localhost:3306)/mysql") + if err != nil { + return err + } + defer db.Close() + + // Set root password using mysql_native_password for broader compatibility + _, err = db.ExecContext(ctx, "ALTER USER 'root'@'localhost' IDENTIFIED WITH mysql_native_password BY 'mysecretpassword';") + if err != nil { + // Try without specifying auth plugin + _, err = db.ExecContext(ctx, "ALTER USER 'root'@'localhost' IDENTIFIED BY 'mysecretpassword';") + if err != nil { + // Try older MySQL syntax + _, err = db.ExecContext(ctx, "SET PASSWORD FOR 'root'@'localhost' = PASSWORD('mysecretpassword');") + if err != nil { + return fmt.Errorf("could not set MySQL password: %w", err) + } + } + } + + // Flush privileges + _, _ = db.ExecContext(ctx, "FLUSH PRIVILEGES;") + + return nil +} + +func waitForMySQL(ctx context.Context, uri string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + // Make an immediate first attempt before waiting for the ticker + if err := tryMySQLConnection(ctx, uri); err == nil { + return nil + } + + var lastErr error + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w (last error: %v)", ctx.Err(), lastErr) + case <-ticker.C: + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for MySQL (last error: %v)", lastErr) + } + if err := tryMySQLConnection(ctx, uri); err != nil { + lastErr = err + continue + } + return nil + } + } +} + +func tryMySQLConnection(ctx context.Context, uri string) error { + db, err := sql.Open("mysql", uri) + if err != nil { + slog.Debug("native/mysql", "open-attempt", err) + return err + } + defer db.Close() + // Use a short timeout for ping to avoid hanging + pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return db.PingContext(pingCtx) +} diff --git a/internal/sqltest/native/postgres.go b/internal/sqltest/native/postgres.go new file mode 100644 index 0000000000..f805a40a1c --- /dev/null +++ b/internal/sqltest/native/postgres.go @@ -0,0 +1,221 @@ +package native + +import ( + "context" + "fmt" + "log/slog" + "os/exec" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "golang.org/x/sync/singleflight" +) + +var postgresFlight singleflight.Group +var postgresURI string + +// StartPostgreSQLServer starts an existing PostgreSQL installation natively (without Docker). +func StartPostgreSQLServer(ctx context.Context) (string, error) { + if err := Supported(); err != nil { + return "", err + } + if postgresURI != "" { + return postgresURI, nil + } + value, err, _ := postgresFlight.Do("postgresql", func() (interface{}, error) { + uri, err := startPostgreSQLServer(ctx) + if err != nil { + return "", err + } + postgresURI = uri + return uri, nil + }) + if err != nil { + return "", err + } + data, ok := value.(string) + if !ok { + return "", fmt.Errorf("returned value was not a string") + } + return data, nil +} + +func startPostgreSQLServer(ctx context.Context) (string, error) { + // Standard URI for test PostgreSQL + uri := "postgres://postgres:postgres@localhost:5432/postgres?sslmode=disable" + + // Try to connect first - it might already be running + if err := waitForPostgres(ctx, uri, 500*time.Millisecond); err == nil { + slog.Info("native/postgres", "status", "already running") + return uri, nil + } + + // Check if PostgreSQL is installed + if _, err := exec.LookPath("psql"); err != nil { + return "", fmt.Errorf("PostgreSQL is not installed (psql not found)") + } + + // Start PostgreSQL service + slog.Info("native/postgres", "status", "starting service") + + // Try systemctl first, fall back to pg_ctlcluster + if err := startPostgresService(); err != nil { + return "", fmt.Errorf("failed to start PostgreSQL: %w", err) + } + + // Configure PostgreSQL for password authentication + if err := configurePostgres(); err != nil { + return "", fmt.Errorf("failed to configure PostgreSQL: %w", err) + } + + // Wait for PostgreSQL to be ready + waitCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + if err := waitForPostgres(waitCtx, uri, 30*time.Second); err != nil { + return "", fmt.Errorf("timeout waiting for PostgreSQL: %w", err) + } + + return uri, nil +} + +func startPostgresService() error { + // Try systemctl first + cmd := exec.Command("sudo", "systemctl", "start", "postgresql") + if err := cmd.Run(); err == nil { + return nil + } + + // Try service command + cmd = exec.Command("sudo", "service", "postgresql", "start") + if err := cmd.Run(); err == nil { + return nil + } + + // Try pg_ctlcluster (Debian/Ubuntu specific) + // Find the installed PostgreSQL version + output, err := exec.Command("ls", "/etc/postgresql/").CombinedOutput() + if err != nil { + return fmt.Errorf("could not find PostgreSQL version: %w", err) + } + + versions := strings.Fields(string(output)) + if len(versions) == 0 { + return fmt.Errorf("no PostgreSQL version found in /etc/postgresql/") + } + + version := versions[0] + cmd = exec.Command("sudo", "pg_ctlcluster", version, "main", "start") + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("pg_ctlcluster start failed: %w\n%s", err, output) + } + + return nil +} + +func configurePostgres() error { + // Set password for postgres user using sudo -u postgres + cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER USER postgres PASSWORD 'postgres';") + if output, err := cmd.CombinedOutput(); err != nil { + // This might fail if password is already set, which is fine + slog.Debug("native/postgres", "set-password", string(output)) + } + + // Update pg_hba.conf to allow password authentication + // First, find the pg_hba.conf file + output, err := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SHOW hba_file;").CombinedOutput() + if err != nil { + return fmt.Errorf("could not find hba_file: %w", err) + } + + hbaFile := strings.TrimSpace(string(output)) + if hbaFile == "" { + return fmt.Errorf("empty hba_file path") + } + + // Check if we need to update pg_hba.conf + catOutput, err := exec.Command("sudo", "cat", hbaFile).CombinedOutput() + if err != nil { + return fmt.Errorf("could not read %s: %w", hbaFile, err) + } + + // If md5 or scram-sha-256 auth is not configured for local connections, add it + content := string(catOutput) + if !strings.Contains(content, "host all all 127.0.0.1/32 md5") && + !strings.Contains(content, "host all all 127.0.0.1/32 scram-sha-256") { + + // Prepend a rule for localhost password authentication + newRule := "host all all 127.0.0.1/32 md5\n" + + // Use sed to add the rule at the beginning (after comments) + cmd := exec.Command("sudo", "bash", "-c", + fmt.Sprintf(`echo '%s' | cat - %s > /tmp/pg_hba.conf.new && sudo mv /tmp/pg_hba.conf.new %s`, + newRule, hbaFile, hbaFile)) + if output, err := cmd.CombinedOutput(); err != nil { + slog.Debug("native/postgres", "update-hba-error", string(output)) + } + + // Reload PostgreSQL to apply changes + if err := reloadPostgres(); err != nil { + slog.Debug("native/postgres", "reload-error", err) + } + } + + return nil +} + +func reloadPostgres() error { + // Try systemctl reload + cmd := exec.Command("sudo", "systemctl", "reload", "postgresql") + if err := cmd.Run(); err == nil { + return nil + } + + // Try service reload + cmd = exec.Command("sudo", "service", "postgresql", "reload") + if err := cmd.Run(); err == nil { + return nil + } + + // Try pg_ctlcluster reload + output, _ := exec.Command("ls", "/etc/postgresql/").CombinedOutput() + versions := strings.Fields(string(output)) + if len(versions) > 0 { + cmd = exec.Command("sudo", "pg_ctlcluster", versions[0], "main", "reload") + return cmd.Run() + } + + return fmt.Errorf("could not reload PostgreSQL") +} + +func waitForPostgres(ctx context.Context, uri string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + var lastErr error + for { + select { + case <-ctx.Done(): + return fmt.Errorf("context cancelled: %w (last error: %v)", ctx.Err(), lastErr) + case <-ticker.C: + if time.Now().After(deadline) { + return fmt.Errorf("timeout waiting for PostgreSQL (last error: %v)", lastErr) + } + conn, err := pgx.Connect(ctx, uri) + if err != nil { + lastErr = err + slog.Debug("native/postgres", "connect-attempt", err) + continue + } + if err := conn.Ping(ctx); err != nil { + lastErr = err + conn.Close(ctx) + continue + } + conn.Close(ctx) + return nil + } + } +}