diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d380b35c..a104c6ebd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ The types of changes are: * `Security` in case of vulnerabilities. -## [Unreleased](https://github.com/ethyca/fidesops/compare/1.5.2...main) +## [Unreleased](https://github.com/ethyca/fidesops/compare/1.5.3...main) ### Added * Subject Request Details page [#563](https://github.com/ethyca/fidesops/pull/563) @@ -29,6 +29,14 @@ The types of changes are: * Refactor auth and enable static file serving [#577](https://github.com/ethyca/fidesops/pull/577) +## [1.5.3](https://github.com/ethyca/fidesops/compare/1.5.2...1.5.3) + +### Changed +* Database migrations now exist as part of the core `fidesops` package [#620](https://github.com/ethyca/fidesops/pull/620) + +### Removed +* The `[package]` config section no longer exists [#620](https://github.com/ethyca/fidesops/pull/620) + ## [1.5.2](https://github.com/ethyca/fidesops/compare/1.5.1...1.5.2) ### Added diff --git a/MANIFEST.in b/MANIFEST.in index 4d1955435..570a7a81e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,10 +3,10 @@ include README.md include requirements.txt include dev-requirements.txt include versioneer.py -include src/alembic.ini +include src/fidesops/alembic.ini include src/fidesops/_version.py -graft src/migrations -exclude src/migrations/README -exclude src/migrations/script.py.mako +graft src/fidesops/migrations +exclude src/fidesops/migrations/README +exclude src/fidesops/migrations/script.py.mako global-exclude *.pyc diff --git a/src/alembic.ini b/src/fidesops/alembic.ini similarity index 100% rename from src/alembic.ini rename to src/fidesops/alembic.ini diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index 4e44d8de6..1a3d46fb5 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -95,31 +95,6 @@ class Config: env_prefix = "FIDESOPS__EXECUTION__" -class PackageSettings(FidesSettings): - """Configuration settings for the fidesops package itself.""" - - PATH: Optional[str] = None - - @validator("PATH", pre=True) - def ensure_valid_package_path(cls, v: Optional[str]) -> str: - """ - Ensure a valid path to the fidesops src/ directory is provided. - - This is required to enable fidesops-plus to start successfully. - """ - - if isinstance(v, str) and os.path.isdir(v): - return ( - v if os.path.basename(v) == "fidesops" else os.path.join(v, "fidesops/") - ) - - current_dir = os.path.dirname(os.path.abspath(__file__)) - return os.path.normpath(os.path.join(current_dir, "../../")) - - class Config: - env_prefix = "FIDESOPS__PACKAGE__" - - class RedisSettings(FidesSettings): """Configuration settings for Redis.""" @@ -253,7 +228,6 @@ class FidesopsConfig(FidesSettings): """Configuration variables for the FastAPI project""" database: DatabaseSettings - package = PackageSettings() redis: RedisSettings security: SecuritySettings execution: ExecutionSettings diff --git a/src/fidesops/db/database.py b/src/fidesops/db/database.py index a972bfd42..64453def6 100644 --- a/src/fidesops/db/database.py +++ b/src/fidesops/db/database.py @@ -1,29 +1,27 @@ """ Contains all of the logic for spinning up/tearing down the database. """ -import os +from os import path from alembic import command from alembic.config import Config from alembic.migration import MigrationContext from pydantic import PostgresDsn -from fidesops.core.config import config from fidesops.db.session import get_db_engine from .base import Base -def get_alembic_config( - database_url: str, - package_source_dir: str = config.package.PATH, -) -> Config: +def get_alembic_config(database_url: str) -> Config: """ Do lots of magic to make alembic work programmatically. """ - migrations_dir = os.path.join(package_source_dir, "migrations/") - alembic_config = Config(os.path.join(package_source_dir, "alembic.ini")) + current_dir = path.dirname(path.abspath(__file__)) + migrations_dir = path.abspath(path.join(current_dir, "../migrations")) + + alembic_config = Config(path.abspath(path.join(current_dir, "../alembic.ini"))) alembic_config.set_main_option("script_location", migrations_dir.replace("%", "%%")) alembic_config.set_main_option("sqlalchemy.url", database_url) return alembic_config @@ -35,14 +33,11 @@ def upgrade_db(alembic_config: Config, revision: str = "head") -> None: command.upgrade(alembic_config, revision) -def init_db( - database_url: PostgresDsn, - package_source_dir: str = config.package.PATH, -) -> None: +def init_db(database_url: PostgresDsn) -> None: """ Runs the migrations and creates all of the database objects. """ - alembic_config = get_alembic_config(database_url, package_source_dir) + alembic_config = get_alembic_config(database_url) upgrade_db(alembic_config) diff --git a/src/fidesops/main.py b/src/fidesops/main.py index 3e8fe5c01..a1caf0b69 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -56,7 +56,7 @@ def start_webserver() -> None: if config.database.ENABLED: logger.info("Running any pending DB migrations...") - init_db(config.database.SQLALCHEMY_DATABASE_URI, config.package.PATH) + init_db(config.database.SQLALCHEMY_DATABASE_URI) scheduler.start() diff --git a/src/migrations/README b/src/fidesops/migrations/README similarity index 100% rename from src/migrations/README rename to src/fidesops/migrations/README diff --git a/src/migrations/__init__.py b/src/fidesops/migrations/__init__.py similarity index 100% rename from src/migrations/__init__.py rename to src/fidesops/migrations/__init__.py diff --git a/src/migrations/env.py b/src/fidesops/migrations/env.py similarity index 100% rename from src/migrations/env.py rename to src/fidesops/migrations/env.py diff --git a/src/migrations/script.py.mako b/src/fidesops/migrations/script.py.mako similarity index 100% rename from src/migrations/script.py.mako rename to src/fidesops/migrations/script.py.mako diff --git a/src/migrations/versions/0210948a8147_initial.py b/src/fidesops/migrations/versions/0210948a8147_initial.py similarity index 100% rename from src/migrations/versions/0210948a8147_initial.py rename to src/fidesops/migrations/versions/0210948a8147_initial.py diff --git a/src/migrations/versions/07014ff34eb2_add_mariadb.py b/src/fidesops/migrations/versions/07014ff34eb2_add_mariadb.py similarity index 100% rename from src/migrations/versions/07014ff34eb2_add_mariadb.py rename to src/fidesops/migrations/versions/07014ff34eb2_add_mariadb.py diff --git a/src/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py b/src/fidesops/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py similarity index 100% rename from src/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py rename to src/fidesops/migrations/versions/1dfc5a2d30e7_add_saas_config_to_connection_config.py diff --git a/src/migrations/versions/1ff88b7bd579_add_authorization_request.py b/src/fidesops/migrations/versions/1ff88b7bd579_add_authorization_request.py similarity index 100% rename from src/migrations/versions/1ff88b7bd579_add_authorization_request.py rename to src/fidesops/migrations/versions/1ff88b7bd579_add_authorization_request.py diff --git a/src/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py b/src/fidesops/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py similarity index 100% rename from src/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py rename to src/fidesops/migrations/versions/27fe9da9d0f9_privacy_request_stopped.py diff --git a/src/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py b/src/fidesops/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py similarity index 100% rename from src/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py rename to src/fidesops/migrations/versions/29a7d707163a_adds_first_name_and_last_name_to_user_.py diff --git a/src/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py b/src/fidesops/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py similarity index 100% rename from src/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py rename to src/fidesops/migrations/versions/3a7c5fb119c9_add_manual_connector_type.py diff --git a/src/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py b/src/fidesops/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py similarity index 100% rename from src/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py rename to src/fidesops/migrations/versions/5078badb90b9_adds_drp_action_to_policy.py diff --git a/src/migrations/versions/530fb8533ca4_test.py b/src/fidesops/migrations/versions/530fb8533ca4_test.py similarity index 100% rename from src/migrations/versions/530fb8533ca4_test.py rename to src/fidesops/migrations/versions/530fb8533ca4_test.py diff --git a/src/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py b/src/fidesops/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py similarity index 100% rename from src/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py rename to src/fidesops/migrations/versions/5a966cd643d7_fidesops_user_and_client_detail.py diff --git a/src/migrations/versions/90070db16d05_add_fidesops_user_permissions.py b/src/fidesops/migrations/versions/90070db16d05_add_fidesops_user_permissions.py similarity index 100% rename from src/migrations/versions/90070db16d05_add_fidesops_user_permissions.py rename to src/fidesops/migrations/versions/90070db16d05_add_fidesops_user_permissions.py diff --git a/src/migrations/versions/906d7198df28_privacy_request_approve.py b/src/fidesops/migrations/versions/906d7198df28_privacy_request_approve.py similarity index 100% rename from src/migrations/versions/906d7198df28_privacy_request_approve.py rename to src/fidesops/migrations/versions/906d7198df28_privacy_request_approve.py diff --git a/src/migrations/versions/__init__.py b/src/fidesops/migrations/versions/__init__.py similarity index 100% rename from src/migrations/versions/__init__.py rename to src/fidesops/migrations/versions/__init__.py diff --git a/src/migrations/versions/c5336b841d70_add_policy_webhooks.py b/src/fidesops/migrations/versions/c5336b841d70_add_policy_webhooks.py similarity index 100% rename from src/migrations/versions/c5336b841d70_add_policy_webhooks.py rename to src/fidesops/migrations/versions/c5336b841d70_add_policy_webhooks.py diff --git a/src/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py b/src/fidesops/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py similarity index 100% rename from src/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py rename to src/fidesops/migrations/versions/c9759675b5d3_add_user_login_and_password_reset_dates.py diff --git a/src/migrations/versions/c98da12d76f8_add_audit_log.py b/src/fidesops/migrations/versions/c98da12d76f8_add_audit_log.py similarity index 100% rename from src/migrations/versions/c98da12d76f8_add_audit_log.py rename to src/fidesops/migrations/versions/c98da12d76f8_add_audit_log.py diff --git a/src/migrations/versions/d65e7e921814_add_privacy_request_status.py b/src/fidesops/migrations/versions/d65e7e921814_add_privacy_request_status.py similarity index 100% rename from src/migrations/versions/d65e7e921814_add_privacy_request_status.py rename to src/fidesops/migrations/versions/d65e7e921814_add_privacy_request_status.py diff --git a/src/migrations/versions/e55a51b354e3_add_bigquery.py b/src/fidesops/migrations/versions/e55a51b354e3_add_bigquery.py similarity index 100% rename from src/migrations/versions/e55a51b354e3_add_bigquery.py rename to src/fidesops/migrations/versions/e55a51b354e3_add_bigquery.py diff --git a/src/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py b/src/fidesops/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py similarity index 100% rename from src/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py rename to src/fidesops/migrations/versions/f206d4e7574d_add_redshift_and_snowflake_support.py diff --git a/src/migrations/versions/f3841942d90c_add_mssql.py b/src/fidesops/migrations/versions/f3841942d90c_add_mssql.py similarity index 100% rename from src/migrations/versions/f3841942d90c_add_mssql.py rename to src/fidesops/migrations/versions/f3841942d90c_add_mssql.py