From 3c50bd462a9c924a91a7c408da54b48ec55c65cb Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Wed, 12 Oct 2022 17:58:31 -0700 Subject: [PATCH 1/2] docs: DB migration docs --- CONTRIBUTING.rst | 12 ++ ...3_0054-71b291295ae1_update_schema_types.py | 203 ++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100644 alembic/versions/2022_10_13_0054-71b291295ae1_update_schema_types.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 975dae1dc..6590623b2 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -299,3 +299,15 @@ Otherwise specify the package with a lower bound only: some-package>=1.2.3 Don't use upper bounds in the dependencies. We have nightly unit tests that test if newer versions of dependencies will break. + +Database migrations +=================== + +We use `Alembic `_ to manage schema migrations. If a PR introduces new models are changes existing ones a migration must be created. + +1. Run the Docker container with ``docker compose up``. +2. Enter the ``dj`` container with ``docker exec -it dj bash``. +3. Run ``alembic revision --autogenerate -m "Description of the migration"``. This will create a file in the repository, under ``alembic/versions/``. Verify the file, checking that the upgrade and the downgrade functions make sense. +4. Still inside the container, run ``alembic upgrade head``. This will update the database schema to match the models. +5. Now run ``alembic downgrade $SHA``, where ``$SHA`` is the previous migration. You can see the hash with ``alembic history``. +6. Once you've confirmed that both the upgrade and downgrade work, upgrade again and commit the file. diff --git a/alembic/versions/2022_10_13_0054-71b291295ae1_update_schema_types.py b/alembic/versions/2022_10_13_0054-71b291295ae1_update_schema_types.py new file mode 100644 index 000000000..07f2c0a6a --- /dev/null +++ b/alembic/versions/2022_10_13_0054-71b291295ae1_update_schema_types.py @@ -0,0 +1,203 @@ +"""Update schema types + +Revision ID: 71b291295ae1 +Revises: 37cd6f86330a +Create Date: 2022-10-13 00:54:24.422858+00:00 + +""" +# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module + +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "71b291295ae1" +down_revision = "37cd6f86330a" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("ix_column_dimension_column", table_name="column") + op.drop_index("ix_column_dimension_id", table_name="column") + op.drop_index("ix_column_id", table_name="column") + op.drop_index("ix_column_name", table_name="column") + op.alter_column( + "database", + "description", + existing_type=sa.VARCHAR(), + nullable=False, + ) + op.alter_column("database", "read_only", existing_type=sa.BOOLEAN(), nullable=False) + op.alter_column("database", "async", existing_type=sa.BOOLEAN(), nullable=False) + op.alter_column( + "database", + "cost", + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=False, + ) + op.drop_index("ix_database_URI", table_name="database") + op.drop_index("ix_database_async", table_name="database") + op.drop_index("ix_database_cost", table_name="database") + op.drop_index("ix_database_description", table_name="database") + op.drop_index("ix_database_id", table_name="database") + op.drop_index("ix_database_read_only", table_name="database") + op.alter_column("node", "description", existing_type=sa.VARCHAR(), nullable=False) + op.drop_index("ix_node_description", table_name="node") + op.drop_index("ix_node_id", table_name="node") + op.drop_index("ix_node_query", table_name="node") + op.drop_index("ix_nodecolumns_column_id", table_name="nodecolumns") + op.drop_index("ix_nodecolumns_node_id", table_name="nodecolumns") + op.drop_index("ix_noderelationship_child_id", table_name="noderelationship") + op.drop_index("ix_noderelationship_parent_id", table_name="noderelationship") + op.alter_column("query", "state", existing_type=sa.VARCHAR(), nullable=False) + op.alter_column( + "query", + "progress", + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=False, + ) + op.drop_index("ix_query_catalog", table_name="query") + op.drop_index("ix_query_database_id", table_name="query") + op.drop_index("ix_query_executed_query", table_name="query") + op.drop_index("ix_query_finished", table_name="query") + op.drop_index("ix_query_progress", table_name="query") + op.drop_index("ix_query_scheduled", table_name="query") + op.drop_index("ix_query_schema_", table_name="query") + op.drop_index("ix_query_started", table_name="query") + op.drop_index("ix_query_state", table_name="query") + op.drop_index("ix_query_submitted_query", table_name="query") + op.alter_column( + "table", + "cost", + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=False, + ) + op.drop_index("ix_table_catalog", table_name="table") + op.drop_index("ix_table_cost", table_name="table") + op.drop_index("ix_table_database_id", table_name="table") + op.drop_index("ix_table_id", table_name="table") + op.drop_index("ix_table_node_id", table_name="table") + op.drop_index("ix_table_schema_", table_name="table") + op.drop_index("ix_table_table", table_name="table") + op.drop_index("ix_tablecolumns_column_id", table_name="tablecolumns") + op.drop_index("ix_tablecolumns_table_id", table_name="tablecolumns") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index( + "ix_tablecolumns_table_id", + "tablecolumns", + ["table_id"], + unique=False, + ) + op.create_index( + "ix_tablecolumns_column_id", + "tablecolumns", + ["column_id"], + unique=False, + ) + op.create_index("ix_table_table", "table", ["table"], unique=False) + op.create_index("ix_table_schema_", "table", ["schema_"], unique=False) + op.create_index("ix_table_node_id", "table", ["node_id"], unique=False) + op.create_index("ix_table_id", "table", ["id"], unique=False) + op.create_index("ix_table_database_id", "table", ["database_id"], unique=False) + op.create_index("ix_table_cost", "table", ["cost"], unique=False) + op.create_index("ix_table_catalog", "table", ["catalog"], unique=False) + op.alter_column( + "table", + "cost", + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=True, + ) + op.create_index( + "ix_query_submitted_query", + "query", + ["submitted_query"], + unique=False, + ) + op.create_index("ix_query_state", "query", ["state"], unique=False) + op.create_index("ix_query_started", "query", ["started"], unique=False) + op.create_index("ix_query_schema_", "query", ["schema_"], unique=False) + op.create_index("ix_query_scheduled", "query", ["scheduled"], unique=False) + op.create_index("ix_query_progress", "query", ["progress"], unique=False) + op.create_index("ix_query_finished", "query", ["finished"], unique=False) + op.create_index( + "ix_query_executed_query", + "query", + ["executed_query"], + unique=False, + ) + op.create_index("ix_query_database_id", "query", ["database_id"], unique=False) + op.create_index("ix_query_catalog", "query", ["catalog"], unique=False) + op.alter_column( + "query", + "progress", + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=True, + ) + op.alter_column("query", "state", existing_type=sa.VARCHAR(), nullable=True) + op.create_index( + "ix_noderelationship_parent_id", + "noderelationship", + ["parent_id"], + unique=False, + ) + op.create_index( + "ix_noderelationship_child_id", + "noderelationship", + ["child_id"], + unique=False, + ) + op.create_index("ix_nodecolumns_node_id", "nodecolumns", ["node_id"], unique=False) + op.create_index( + "ix_nodecolumns_column_id", + "nodecolumns", + ["column_id"], + unique=False, + ) + op.create_index("ix_node_query", "node", ["query"], unique=False) + op.create_index("ix_node_id", "node", ["id"], unique=False) + op.create_index("ix_node_description", "node", ["description"], unique=False) + op.alter_column("node", "description", existing_type=sa.VARCHAR(), nullable=True) + op.create_index("ix_database_read_only", "database", ["read_only"], unique=False) + op.create_index("ix_database_id", "database", ["id"], unique=False) + op.create_index( + "ix_database_description", + "database", + ["description"], + unique=False, + ) + op.create_index("ix_database_cost", "database", ["cost"], unique=False) + op.create_index("ix_database_async", "database", ["async"], unique=False) + op.create_index("ix_database_URI", "database", ["URI"], unique=False) + op.alter_column( + "database", + "cost", + existing_type=postgresql.DOUBLE_PRECISION(precision=53), + nullable=True, + ) + op.alter_column("database", "async", existing_type=sa.BOOLEAN(), nullable=True) + op.alter_column("database", "read_only", existing_type=sa.BOOLEAN(), nullable=True) + op.alter_column( + "database", + "description", + existing_type=sa.VARCHAR(), + nullable=True, + ) + op.create_index("ix_column_name", "column", ["name"], unique=False) + op.create_index("ix_column_id", "column", ["id"], unique=False) + op.create_index("ix_column_dimension_id", "column", ["dimension_id"], unique=False) + op.create_index( + "ix_column_dimension_column", + "column", + ["dimension_column"], + unique=False, + ) + # ### end Alembic commands ### From fde1a466e49531e46dc16defaec207a63ae00b71 Mon Sep 17 00:00:00 2001 From: Beto Dealmeida Date: Wed, 12 Oct 2022 18:49:01 -0700 Subject: [PATCH 2/2] Address comments --- CONTRIBUTING.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 6590623b2..18b9d50e0 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -303,7 +303,7 @@ Don't use upper bounds in the dependencies. We have nightly unit tests that test Database migrations =================== -We use `Alembic `_ to manage schema migrations. If a PR introduces new models are changes existing ones a migration must be created. +We use `Alembic `_ to manage schema migrations. If a PR introduces new models or changes existing ones a migration must be created. 1. Run the Docker container with ``docker compose up``. 2. Enter the ``dj`` container with ``docker exec -it dj bash``.