docs: DB migration docs

CONTRIBUTING.rst

@@ -299,3 +299,15 @@ Otherwise specify the package with a lower bound only:
     some-package>=1.2.3
 
 Don't use upper bounds in the dependencies. We have nightly unit tests that test if newer versions of dependencies will break.
+
+Database migrations
+===================
+
+We use `Alembic <https://alembic.sqlalchemy.org/en/latest/index.html>`_ to manage schema migrations. If a PR introduces new models or changes existing ones a migration must be created.
+
+1. Run the Docker container with ``docker compose up``.
+2. Enter the ``dj`` container with ``docker exec -it dj bash``.
+3. Run ``alembic revision --autogenerate -m "Description of the migration"``. This will create a file in the repository, under ``alembic/versions/``. Verify the file, checking that the upgrade and the downgrade functions make sense.
+4. Still inside the container, run ``alembic upgrade head``. This will update the database schema to match the models.
+5. Now run ``alembic downgrade $SHA``, where ``$SHA`` is the previous migration. You can see the hash with ``alembic history``.
+6. Once you've confirmed that both the upgrade and downgrade work, upgrade again and commit the file.

alembic/versions/2022_10_13_0054-71b291295ae1_update_schema_types.py

@@ -0,0 +1,203 @@
+"""Update schema types
+
+Revision ID: 71b291295ae1
+Revises: 37cd6f86330a
+Create Date: 2022-10-13 00:54:24.422858+00:00
+
+"""
+# pylint: disable=no-member, invalid-name, missing-function-docstring, unused-import, no-name-in-module
+
+import sqlalchemy as sa
+import sqlmodel
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "71b291295ae1"
+down_revision = "37cd6f86330a"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index("ix_column_dimension_column", table_name="column")
+    op.drop_index("ix_column_dimension_id", table_name="column")
+    op.drop_index("ix_column_id", table_name="column")
+    op.drop_index("ix_column_name", table_name="column")
+    op.alter_column(
+        "database",
+        "description",
+        existing_type=sa.VARCHAR(),
+        nullable=False,
+    )
+    op.alter_column("database", "read_only", existing_type=sa.BOOLEAN(), nullable=False)
+    op.alter_column("database", "async", existing_type=sa.BOOLEAN(), nullable=False)
+    op.alter_column(
+        "database",
+        "cost",
+        existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+        nullable=False,
+    )
+    op.drop_index("ix_database_URI", table_name="database")
+    op.drop_index("ix_database_async", table_name="database")
+    op.drop_index("ix_database_cost", table_name="database")
+    op.drop_index("ix_database_description", table_name="database")
+    op.drop_index("ix_database_id", table_name="database")
+    op.drop_index("ix_database_read_only", table_name="database")
+    op.alter_column("node", "description", existing_type=sa.VARCHAR(), nullable=False)
+    op.drop_index("ix_node_description", table_name="node")
+    op.drop_index("ix_node_id", table_name="node")
+    op.drop_index("ix_node_query", table_name="node")
+    op.drop_index("ix_nodecolumns_column_id", table_name="nodecolumns")
+    op.drop_index("ix_nodecolumns_node_id", table_name="nodecolumns")
+    op.drop_index("ix_noderelationship_child_id", table_name="noderelationship")
+    op.drop_index("ix_noderelationship_parent_id", table_name="noderelationship")
+    op.alter_column("query", "state", existing_type=sa.VARCHAR(), nullable=False)
+    op.alter_column(
+        "query",
+        "progress",
+        existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+        nullable=False,
+    )
+    op.drop_index("ix_query_catalog", table_name="query")
+    op.drop_index("ix_query_database_id", table_name="query")
+    op.drop_index("ix_query_executed_query", table_name="query")
+    op.drop_index("ix_query_finished", table_name="query")
+    op.drop_index("ix_query_progress", table_name="query")
+    op.drop_index("ix_query_scheduled", table_name="query")
+    op.drop_index("ix_query_schema_", table_name="query")
+    op.drop_index("ix_query_started", table_name="query")
+    op.drop_index("ix_query_state", table_name="query")
+    op.drop_index("ix_query_submitted_query", table_name="query")
+    op.alter_column(
+        "table",
+        "cost",
+        existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+        nullable=False,
+    )
+    op.drop_index("ix_table_catalog", table_name="table")
+    op.drop_index("ix_table_cost", table_name="table")
+    op.drop_index("ix_table_database_id", table_name="table")
+    op.drop_index("ix_table_id", table_name="table")
+    op.drop_index("ix_table_node_id", table_name="table")
+    op.drop_index("ix_table_schema_", table_name="table")
+    op.drop_index("ix_table_table", table_name="table")
+    op.drop_index("ix_tablecolumns_column_id", table_name="tablecolumns")
+    op.drop_index("ix_tablecolumns_table_id", table_name="tablecolumns")
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_index(
+        "ix_tablecolumns_table_id",
+        "tablecolumns",
+        ["table_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_tablecolumns_column_id",
+        "tablecolumns",
+        ["column_id"],
+        unique=False,
+    )
+    op.create_index("ix_table_table", "table", ["table"], unique=False)
+    op.create_index("ix_table_schema_", "table", ["schema_"], unique=False)
+    op.create_index("ix_table_node_id", "table", ["node_id"], unique=False)
+    op.create_index("ix_table_id", "table", ["id"], unique=False)
+    op.create_index("ix_table_database_id", "table", ["database_id"], unique=False)
+    op.create_index("ix_table_cost", "table", ["cost"], unique=False)
+    op.create_index("ix_table_catalog", "table", ["catalog"], unique=False)
+    op.alter_column(
+        "table",
+        "cost",
+        existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+        nullable=True,
+    )
+    op.create_index(
+        "ix_query_submitted_query",
+        "query",
+        ["submitted_query"],
+        unique=False,
+    )
+    op.create_index("ix_query_state", "query", ["state"], unique=False)
+    op.create_index("ix_query_started", "query", ["started"], unique=False)
+    op.create_index("ix_query_schema_", "query", ["schema_"], unique=False)
+    op.create_index("ix_query_scheduled", "query", ["scheduled"], unique=False)
+    op.create_index("ix_query_progress", "query", ["progress"], unique=False)
+    op.create_index("ix_query_finished", "query", ["finished"], unique=False)
+    op.create_index(
+        "ix_query_executed_query",
+        "query",
+        ["executed_query"],
+        unique=False,
+    )
+    op.create_index("ix_query_database_id", "query", ["database_id"], unique=False)
+    op.create_index("ix_query_catalog", "query", ["catalog"], unique=False)
+    op.alter_column(
+        "query",
+        "progress",
+        existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+        nullable=True,
+    )
+    op.alter_column("query", "state", existing_type=sa.VARCHAR(), nullable=True)
+    op.create_index(
+        "ix_noderelationship_parent_id",
+        "noderelationship",
+        ["parent_id"],
+        unique=False,
+    )
+    op.create_index(
+        "ix_noderelationship_child_id",
+        "noderelationship",
+        ["child_id"],
+        unique=False,
+    )
+    op.create_index("ix_nodecolumns_node_id", "nodecolumns", ["node_id"], unique=False)
+    op.create_index(
+        "ix_nodecolumns_column_id",
+        "nodecolumns",
+        ["column_id"],
+        unique=False,
+    )
+    op.create_index("ix_node_query", "node", ["query"], unique=False)
+    op.create_index("ix_node_id", "node", ["id"], unique=False)
+    op.create_index("ix_node_description", "node", ["description"], unique=False)
+    op.alter_column("node", "description", existing_type=sa.VARCHAR(), nullable=True)
+    op.create_index("ix_database_read_only", "database", ["read_only"], unique=False)
+    op.create_index("ix_database_id", "database", ["id"], unique=False)
+    op.create_index(
+        "ix_database_description",
+        "database",
+        ["description"],
+        unique=False,
+    )
+    op.create_index("ix_database_cost", "database", ["cost"], unique=False)
+    op.create_index("ix_database_async", "database", ["async"], unique=False)
+    op.create_index("ix_database_URI", "database", ["URI"], unique=False)
+    op.alter_column(
+        "database",
+        "cost",
+        existing_type=postgresql.DOUBLE_PRECISION(precision=53),
+        nullable=True,
+    )
+    op.alter_column("database", "async", existing_type=sa.BOOLEAN(), nullable=True)
+    op.alter_column("database", "read_only", existing_type=sa.BOOLEAN(), nullable=True)
+    op.alter_column(
+        "database",
+        "description",
+        existing_type=sa.VARCHAR(),
+        nullable=True,
+    )
+    op.create_index("ix_column_name", "column", ["name"], unique=False)
+    op.create_index("ix_column_id", "column", ["id"], unique=False)
+    op.create_index("ix_column_dimension_id", "column", ["dimension_id"], unique=False)
+    op.create_index(
+        "ix_column_dimension_column",
+        "column",
+        ["dimension_column"],
+        unique=False,
+    )
+    # ### end Alembic commands ###