From 313f48ec70b49a7eab0b73ff74b763543a1d822b Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 21:09:18 +0300 Subject: [PATCH 01/15] rework build 2 --- dp-client/README.md | 72 ++++++++ dp-client/dp_client/__init__.py | 11 ++ dp-client/dp_client/api/__init__.py | 4 + dp-client/dp_client/api/health.py | 21 +++ dp-client/dp_client/api/users.py | 34 ++++ dp-client/dp_client/client.py | 56 ++++++ dp-client/dp_client/clients/__init__.py | 3 + dp-client/dp_client/clients/metadata.py | 28 +++ dp-client/dp_client/db/__init__.py | 3 + dp-client/dp_client/db/drivers/__init__.py | 3 + dp-client/dp_client/db/drivers/postgres.py | 50 +++++ dp-client/dp_client/db/pg.py | 113 +++++++++++ dp-client/pyproject.toml | 21 +++ dp-client/requirements.txt | 3 + mypy.ini | 14 +- scripts/build_and_install_dp_client.sh | 46 +++++ scripts/build_dp_client.sh | 46 +++++ scripts/precommit.sh | 3 +- tests/component/test_user_model_component.py | 185 +++++++++++++++++++ tests/conftest.py | 58 ++++++ tests/unit/test_user_model.py | 28 +-- 21 files changed, 771 insertions(+), 31 deletions(-) create mode 100644 dp-client/README.md create mode 100644 dp-client/dp_client/__init__.py create mode 100644 dp-client/dp_client/api/__init__.py create mode 100644 dp-client/dp_client/api/health.py create mode 100644 dp-client/dp_client/api/users.py create mode 100644 dp-client/dp_client/client.py create mode 100644 dp-client/dp_client/clients/__init__.py create mode 100644 dp-client/dp_client/clients/metadata.py create mode 100644 dp-client/dp_client/db/__init__.py create mode 100644 dp-client/dp_client/db/drivers/__init__.py create mode 100644 dp-client/dp_client/db/drivers/postgres.py create mode 100644 dp-client/dp_client/db/pg.py create mode 100644 dp-client/pyproject.toml create mode 100644 dp-client/requirements.txt create mode 100755 scripts/build_and_install_dp_client.sh create mode 100755 scripts/build_dp_client.sh create mode 100644 tests/component/test_user_model_component.py create mode 100644 tests/conftest.py diff --git a/dp-client/README.md b/dp-client/README.md new file mode 100644 index 0000000..ab02d23 --- /dev/null +++ b/dp-client/README.md @@ -0,0 +1,72 @@ +# dp-client + +A unified, test-friendly client to interact with the platform during component/integration tests. + +- Lives in its own folder (dp-client) +- Built and versioned similarly to metadata-client +- Depends on the generated `metadata-client` +- Has its own requirements file (requirements.txt) + +## Installation (development) + +Option A: Install from requirements + +```bash +pip install -r dp-client/requirements.txt +``` + +Option B: Build with Poetry and install the wheel + +```bash +./scripts/build_dp_client.sh +pip install ./dp-client/dist/*.whl +``` + +## Usage + +```python +from dp_client import DPClient + +client = DPClient(base_url="http://localhost:8000") + +# Backward-compatible helpers: +client.health_check() +client.create_user({"id": "...", "name": "...", "phone": "+972...", "address": "..."}) +client.get_user("...") +client.list_users() + +# Structured access: +# Underlying generated client (metadata-client): +client.MetaDataServerAPIClient + +# API domains: +client.HealthAPI.health_check() +client.UsersApi.create_user({...}) + +# Optional DB helper (uses Django ORM if available in the environment): +client.PGDBClient.get_user_by_id("...") +client.PGDBClient.delete_users_by_ids(["...", "..."]) +``` + + +## Running DB-backed component tests locally + +You have two options to make DB checks work outside Docker: + +1) Use docker-compose (recommended): + - Ensure the Postgres service is up and reachable as host name `db` within the network. + - From repo root: `docker-compose up -d` + - Run tests; the `.env` sets POSTGRES_HOST=db and tests will connect via that. + +2) Use a local Postgres instance: + - Export env vars to point at your local instance, for example: + - `export POSTGRES_HOST=localhost` + - `export POSTGRES_PORT=5432` + - `export POSTGRES_DB=postgres` + - `export POSTGRES_USER=postgres` + - `export POSTGRES_PASSWORD=postgres` + - Or, if your `.env` uses POSTGRES_HOST=db and you don’t want to change it, you can opt-in to a safe fallback: + - `export POSTGRES_ALLOW_LOCAL_FALLBACK=true` + - When the host `db` is not resolvable, dp-client will transparently fall back to `localhost`. + +If the DB is not reachable, component tests that require direct DB access will be skipped with a helpful message. diff --git a/dp-client/dp_client/__init__.py b/dp-client/dp_client/__init__.py new file mode 100644 index 0000000..f3ba3aa --- /dev/null +++ b/dp-client/dp_client/__init__.py @@ -0,0 +1,11 @@ +from importlib import metadata as _metadata + +try: + __version__ = _metadata.version("dp-client") +except _metadata.PackageNotFoundError: # pragma: no cover + __version__ = "0.0.0" + +from .api import HealthAPI, UsersAPI # noqa: F401 +from .client import DPClient # noqa: F401 +from .clients import MetaDataServerAPIClientFactory # noqa: F401 +from .db import PGDBClient # noqa: F401 diff --git a/dp-client/dp_client/api/__init__.py b/dp-client/dp_client/api/__init__.py new file mode 100644 index 0000000..a4753c9 --- /dev/null +++ b/dp-client/dp_client/api/__init__.py @@ -0,0 +1,4 @@ +from .health import HealthAPI +from .users import UsersAPI + +__all__ = ["HealthAPI", "UsersAPI"] diff --git a/dp-client/dp_client/api/health.py b/dp-client/dp_client/api/health.py new file mode 100644 index 0000000..6681f6c --- /dev/null +++ b/dp-client/dp_client/api/health.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from typing import Any + + +class HealthAPI: + """Health endpoints wrapper using generated metadata_client api module.""" + + def __init__(self, client: Any) -> None: + try: + from metadata_client.api.health import health_retrieve as _health_retrieve # noqa: WPS433 + except Exception as exc: # pragma: no cover + raise RuntimeError( + "metadata-client is required but not installed or failed to import. " + f"Original error: {exc}. Install it or run: pip install -r dp-client/requirements.txt" + ) from exc + self._client = client + self._health_retrieve = _health_retrieve + + def health_check(self): + return self._health_retrieve.sync_detailed(client=self._client) diff --git a/dp-client/dp_client/api/users.py b/dp-client/dp_client/api/users.py new file mode 100644 index 0000000..d1a322f --- /dev/null +++ b/dp-client/dp_client/api/users.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from typing import Any, Union + + +class UsersAPI: + """Users endpoints wrapper using generated metadata_client api module.""" + + def __init__(self, client: Any) -> None: + try: + from metadata_client.api.users import users_create as _users_create + from metadata_client.api.users import users_list as _users_list + from metadata_client.api.users import users_retrieve as _users_retrieve + from metadata_client.models import User as _User + except Exception as exc: # pragma: no cover + raise RuntimeError( + "metadata-client is required but not installed or failed to import. " + f"Original error: {exc}. Install it or run: pip install -r dp-client/requirements.txt" + ) from exc + self._client = client + self._users_create = _users_create + self._users_list = _users_list + self._users_retrieve = _users_retrieve + self._User = _User + + def create_user(self, user: Union[Any, dict]): + body = self._User.from_dict(user) if isinstance(user, dict) else user + return self._users_create.sync_detailed(client=self._client, body=body) + + def get_user(self, user_id: str): + return self._users_retrieve.sync_detailed(client=self._client, id=user_id) + + def list_users(self): + return self._users_list.sync_detailed(client=self._client) diff --git a/dp-client/dp_client/client.py b/dp-client/dp_client/client.py new file mode 100644 index 0000000..e8184ff --- /dev/null +++ b/dp-client/dp_client/client.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from typing import Any, Optional, Union + +from .api.health import HealthAPI +from .api.users import UsersAPI +from .clients.metadata import MetaDataServerAPIClientFactory +from .db.pg import PGDBClient + + +class DPClient: + """High-level helper that composes separated concerns. + + Exposes: + - self.MetaDataServerAPIClient: underlying generated client instance (Client or AuthenticatedClient) + - self.UsersApi: operations around Users endpoints + - self.HealthAPI: operations around Health endpoints + - self.PGDBClient: optional DB helper for direct DB assertions/cleanup + + Backward compatibility: keeps thin wrapper methods (health_check, create_user, get_user, list_users). + """ + + def __init__( + self, + base_url: str, + token: Optional[str] = None, + *, + prefix: str = "Bearer", + timeout: float = 10.0, + ) -> None: + # Build the underlying metadata server client + self._client_factory = MetaDataServerAPIClientFactory() + self.MetaDataServerAPIClient: Any = self._client_factory.build(base_url=base_url, token=token, prefix=prefix) + + # Compose API domains + self.UsersApi = UsersAPI(self.MetaDataServerAPIClient) + self.HealthAPI = HealthAPI(self.MetaDataServerAPIClient) + + # DB helper (optional, raises clear error if ORM not available when used) + self.PGDBClient = PGDBClient() + + # Store timeout for potential future use (e.g., retries) + self._timeout = timeout + + # Backward-compatible thin wrappers + def health_check(self): + return self.HealthAPI.health_check() + + def create_user(self, user: Union[Any, dict[str, Any]]): + return self.UsersApi.create_user(user) + + def get_user(self, user_id: str): + return self.UsersApi.get_user(user_id) + + def list_users(self): + return self.UsersApi.list_users() diff --git a/dp-client/dp_client/clients/__init__.py b/dp-client/dp_client/clients/__init__.py new file mode 100644 index 0000000..6180080 --- /dev/null +++ b/dp-client/dp_client/clients/__init__.py @@ -0,0 +1,3 @@ +from .metadata import MetaDataServerAPIClientFactory + +__all__ = ["MetaDataServerAPIClientFactory"] diff --git a/dp-client/dp_client/clients/metadata.py b/dp-client/dp_client/clients/metadata.py new file mode 100644 index 0000000..5ac1afb --- /dev/null +++ b/dp-client/dp_client/clients/metadata.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from typing import Any, Optional + + +class MetaDataServerAPIClientFactory: + """Factory to construct metadata_client Client/AuthenticatedClient consistently. + + Mirrors how unit/component tests construct the generated client. + """ + + def __init__(self) -> None: + # Import lazily to keep dp-client importable even if metadata-client is missing + try: + from metadata_client import AuthenticatedClient as _AuthenticatedClient # type: ignore + from metadata_client import Client as _Client + except Exception as exc: # pragma: no cover + raise RuntimeError( + "metadata-client is required but not installed or failed to import. " + f"Original error: {exc}. Install it or run: pip install -r dp-client/requirements.txt" + ) from exc + self._AuthenticatedClient = _AuthenticatedClient + self._Client = _Client + + def build(self, base_url: str, token: Optional[str] = None, prefix: str = "Bearer") -> Any: + if token: + return self._AuthenticatedClient(base_url=base_url, token=token, prefix=prefix) + return self._Client(base_url=base_url) diff --git a/dp-client/dp_client/db/__init__.py b/dp-client/dp_client/db/__init__.py new file mode 100644 index 0000000..5b4e2f2 --- /dev/null +++ b/dp-client/dp_client/db/__init__.py @@ -0,0 +1,3 @@ +from .pg import PGDBClient + +__all__ = ["PGDBClient"] diff --git a/dp-client/dp_client/db/drivers/__init__.py b/dp-client/dp_client/db/drivers/__init__.py new file mode 100644 index 0000000..870df18 --- /dev/null +++ b/dp-client/dp_client/db/drivers/__init__.py @@ -0,0 +1,3 @@ +from .postgres import PostgresDriver + +__all__ = ["PostgresDriver"] diff --git a/dp-client/dp_client/db/drivers/postgres.py b/dp-client/dp_client/db/drivers/postgres.py new file mode 100644 index 0000000..0ec2484 --- /dev/null +++ b/dp-client/dp_client/db/drivers/postgres.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from typing import Any, Optional, Tuple + +try: + import psycopg2 +except Exception as exc: # pragma: no cover + raise RuntimeError( + "psycopg2-binary is required for Postgres driver. Install it via dp-client dependencies." + ) from exc + + +class PostgresDriver: + """Simple PostgreSQL driver using psycopg2. No Django coupling. + + Connection parameters must be provided explicitly by the caller (no hardcoded defaults). + """ + + def __init__( + self, + *, + host: str, + port: int, + dbname: str, + user: str, + password: str, + ) -> None: + self._conn_params = dict(host=host, port=port, dbname=dbname, user=user, password=password) + + def _conn(self): + return psycopg2.connect(**self._conn_params) + + def fetch_one(self, query: str, params: Optional[Tuple[Any, ...]] = None) -> Optional[tuple]: + with self._conn() as conn: + with conn.cursor() as cur: + cur.execute(query, params or ()) + return cur.fetchone() + + def fetch_value(self, query: str, params: Optional[Tuple[Any, ...]] = None) -> Optional[Any]: + row = self.fetch_one(query, params) + return row[0] if row is not None and len(row) > 0 else None + + def execute(self, query: str, params: Optional[Tuple[Any, ...]] = None) -> int: + with self._conn() as conn: + with conn.cursor() as cur: + cur.execute(query, params or ()) + # psycopg2 rowcount: number of rows affected by last command + affected = cur.rowcount + conn.commit() + return int(affected) diff --git a/dp-client/dp_client/db/pg.py b/dp-client/dp_client/db/pg.py new file mode 100644 index 0000000..244594c --- /dev/null +++ b/dp-client/dp_client/db/pg.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import os +import socket +from typing import Iterable, Optional + +from .drivers.postgres import PostgresDriver + + +class PGDBClient: + """PostgreSQL Database client using a dedicated driver (no Django dependencies). + + Reads connection parameters from arguments or environment variables (no hardcoded defaults): + - POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_USER, POSTGRES_PASSWORD + """ + + def __init__( + self, + *, + host: Optional[str] = None, + port: Optional[int] = None, + dbname: Optional[str] = None, + user: Optional[str] = None, + password: Optional[str] = None, + table: str = "metadata_manager_user", + ) -> None: + # Allow explicit args to override env; otherwise read from env without fallback defaults + env_host = os.environ.get("POSTGRES_HOST") + env_port = os.environ.get("POSTGRES_PORT") + env_db = os.environ.get("POSTGRES_DB") + env_user = os.environ.get("POSTGRES_USER") + env_password = os.environ.get("POSTGRES_PASSWORD") + + final_host = host or env_host + final_port = port if port is not None else (int(env_port) if env_port else None) + final_db = dbname or env_db + final_user = user or env_user + final_password = password or env_password + + missing = [] + if not final_host: + missing.append("POSTGRES_HOST or host") + if final_port is None: + missing.append("POSTGRES_PORT or port") + if not final_db: + missing.append("POSTGRES_DB or dbname") + if not final_user: + missing.append("POSTGRES_USER or user") + if not final_password: + missing.append("POSTGRES_PASSWORD or password") + + if missing: + raise RuntimeError( + "PGDBClient configuration missing: " + + ", ".join(missing) + + ". Provide env vars in .env or pass parameters explicitly." + ) + + # Optional DNS resolution fallback for local runs outside Docker networks. + # If POSTGRES_HOST is not resolvable and POSTGRES_ALLOW_LOCAL_FALLBACK is truthy, + # transparently switch to localhost. This preserves env-driven config with no hardcoded defaults + # unless explicitly opted-in by the environment. + allow_fallback = os.environ.get("POSTGRES_ALLOW_LOCAL_FALLBACK", "").strip().lower() in { + "1", + "true", + "yes", + "on", + } + resolved_host = final_host # type: ignore[assignment] + try: + # Try to resolve the host name; if it fails, consider fallback. + socket.gethostbyname(str(final_host)) + except Exception: + if allow_fallback: + resolved_host = "localhost" + + self._driver = PostgresDriver( + host=resolved_host, # type: ignore[arg-type] + port=int(final_port), # type: ignore[arg-type] + dbname=final_db, # type: ignore[arg-type] + user=final_user, # type: ignore[arg-type] + password=final_password, # type: ignore[arg-type] + ) + self._table = table + + def get_user_by_id(self, user_id: str): + row = self._driver.fetch_one( + f"SELECT id, name, phone, address FROM {self._table} WHERE id = %s", + (user_id,), + ) + if not row: + return None + # Return a simple dict to avoid coupling + return {"id": row[0], "name": row[1], "phone": row[2], "address": row[3]} + + def users_exist(self, user_ids: Iterable[str]) -> bool: + ids = list(user_ids) + if not ids: + return True + count = self._driver.fetch_value( + f"SELECT COUNT(*) FROM {self._table} WHERE id = ANY(%s)", + (ids,), + ) + return int(count or 0) == len(ids) + + def delete_users_by_ids(self, user_ids: Iterable[str]) -> int: + ids = list(user_ids) + if not ids: + return 0 + return self._driver.execute( + f"DELETE FROM {self._table} WHERE id = ANY(%s)", + (ids,), + ) diff --git a/dp-client/pyproject.toml b/dp-client/pyproject.toml new file mode 100644 index 0000000..f3ad9f5 --- /dev/null +++ b/dp-client/pyproject.toml @@ -0,0 +1,21 @@ +[tool.poetry] +name = "dp-client" +version = "0.0.0" # overwritten by build script +description = "Unified test-friendly client for platform interactions." +authors = ["Your Team "] +packages = [{ include = "dp_client" }] +readme = "README.md" +license = "MIT" + +[tool.poetry.dependencies] +python = ">=3.10,<4.0" +# This will be rewritten by the build script to an exact version from setup.py +metadata-client = "*" +psycopg2-binary = ">=2.9,<3.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.3.0" + +[build-system] +requires = ["poetry-core>=1.8.0"] +build-backend = "poetry.core.masonry.api" diff --git a/dp-client/requirements.txt b/dp-client/requirements.txt new file mode 100644 index 0000000..c46ca15 --- /dev/null +++ b/dp-client/requirements.txt @@ -0,0 +1,3 @@ +metadata-client +psycopg2-binary>=2.9,<3.0 + diff --git a/mypy.ini b/mypy.ini index 1bf779c..f65f93e 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -files = **/*.py +files = src, tests exclude = ^(dist|\.git|\.venv|venv|\.pytest_cache|\.mypy_cache|\.tox|\.eggs|.*egg-info|build|client|metadata-client|src/metadata_manager/migrations)/ warn_unused_ignores = True warn_unreachable = True @@ -7,14 +7,14 @@ allow_redefinition = True warn_unused_configs = True strict = False plugins = mypy_django_plugin.main -mypy_path = src -no_namespace_packages = True +explicit_package_bases = True +namespace_packages = True [mypy-setuptools.*] ignore_missing_imports = True [mypy.plugins.django-stubs] -django_settings_module = "src.server.settings" +django_settings_module = "server.settings" strict_settings = False [mypy_django_plugin] @@ -34,3 +34,9 @@ ignore_missing_imports = True [mypy-rest_framework.routers] ignore_missing_imports = True + +[mypy-dp_client.*] +ignore_missing_imports = True + +[mypy-metadata_manager.*] +ignore_missing_imports = True diff --git a/scripts/build_and_install_dp_client.sh b/scripts/build_and_install_dp_client.sh new file mode 100755 index 0000000..5d699c1 --- /dev/null +++ b/scripts/build_and_install_dp_client.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -euo pipefail + +# 1) Build and install metadata-client first (dp-client depends on it) +echo "[dp-client] Building and installing metadata-client (dependency) ..." +./scripts/build_and_install_open_api_client.sh + +echo "[dp-client] Building package..." +./scripts/build_dp_client.sh + +echo "[dp-client] Locating built wheel..." +WHEEL=$(ls -t ./dp-client/dist/dp_client-*.whl 2>/dev/null | head -n1) +if [[ -z "${WHEEL}" ]]; then + echo "[dp-client] Error: No dp_client wheel found in ./dp-client/dist/. Build may have failed." + exit 1 +fi +echo "[dp-client] Found wheel: ${WHEEL}" + +PYVER=$(python -V 2>/dev/null || true) +if [[ -n "${PYVER}" ]]; then + echo "[dp-client] Installing into Python: ${PYVER}" +else + echo "[dp-client] Installing wheel (Python version unknown)" +fi +# Install without resolving dependencies since metadata-client was already installed locally above +pip install --no-deps "${WHEEL}" --force-reinstall + +echo "[dp-client] Verifying installation..." +python - <<'PY' +import sys +try: + from importlib import metadata as m + import dp_client + ver = m.version("dp-client") + print(f"[dp-client] Installed successfully: dp-client=={ver}") + print(f"[dp-client] Module location: {dp_client.__file__}") +except Exception as e: + print(f"[dp-client] Verification failed: {e}", file=sys.stderr) + sys.exit(1) +PY + +echo "[dp-client] Done. Example quick check:" +echo " python - <<'PY'" +echo "from dp_client import DPClient" +echo "print('dp-client import OK, version:', getattr(__import__('dp_client'), '__version__', 'unknown'))" +echo "PY" diff --git a/scripts/build_dp_client.sh b/scripts/build_dp_client.sh new file mode 100755 index 0000000..c401b7e --- /dev/null +++ b/scripts/build_dp_client.sh @@ -0,0 +1,46 @@ +#!/bin/bash +set -euo pipefail + +PKG_DIR="dp-client" +PKG_NAME="dp-client" + +# Ensure Poetry is present +if ! command -v poetry >/dev/null 2>&1; then + echo "[dp-client] Poetry not found. Installing via pip..." + pip install poetry +fi + +# Read version from repo root setup.py (align with metadata-client) +VERSION=$(grep "__version__" setup.py | awk -F '"' '{print $2}') +if [[ -z "${VERSION}" ]]; then + echo "[dp-client] Could not determine version from setup.py" + exit 1 +fi + +pushd "$PKG_DIR" >/dev/null + +# Optionally pin metadata-client dependency. +# If version is 0.0.0 (local-only), avoid exact pin to prevent resolver hitting remote index. +if grep -q '^metadata-client' pyproject.toml; then + if [[ "${VERSION}" == "0.0.0" ]]; then + echo "[dp-client] Detected local development version (0.0.0). Using unpinned dependency for metadata-client." + sed -e "s/^metadata-client *=.*$/metadata-client = \"*\"/" pyproject.toml > pyproject.toml.tmp + else + echo "[dp-client] Pinning metadata-client dependency to =${VERSION}" + sed -e "s/^metadata-client *=.*$/metadata-client = \"=${VERSION}\"/" pyproject.toml > pyproject.toml.tmp + fi + mv pyproject.toml.tmp pyproject.toml +fi + +# Set the package version +poetry version "${VERSION}" + +echo "[dp-client] Installing build dependencies (poetry-core, etc.)" +# Poetry will manage build env; ensure lock is up-to-date is not required for build + +echo "[dp-client] Building the dp-client wheel with Poetry" +poetry build + +popd >/dev/null + +echo "[dp-client] Build complete. Artifacts in ${PKG_DIR}/dist/" \ No newline at end of file diff --git a/scripts/precommit.sh b/scripts/precommit.sh index 7d03cc2..0ba562a 100755 --- a/scripts/precommit.sh +++ b/scripts/precommit.sh @@ -6,5 +6,6 @@ isort . black . flake8 . pycodestyle . -PYTHONPATH=src mypy . +# Let mypy read targets from mypy.ini (avoids duplicate module names like src.package vs package) +PYTHONPATH=src mypy yamllint . --no-warnings \ No newline at end of file diff --git a/tests/component/test_user_model_component.py b/tests/component/test_user_model_component.py new file mode 100644 index 0000000..96035d0 --- /dev/null +++ b/tests/component/test_user_model_component.py @@ -0,0 +1,185 @@ +from http import HTTPStatus +from typing import Generator, List + +import pytest +from dp_client import DPClient + +from src.tools import generate_israeli_id, generate_random_phone_number + +# Define a tuple of psycopg2-related exceptions if psycopg2 is available. +# This lets us catch specific DB connectivity/driver errors without a blanket Exception. +# Always keep a consistent, typed value to satisfy mypy. +PSYCOPG2_ERRORS: tuple[type[BaseException], ...] = tuple() +try: # pragma: no cover - optional dependency for tests + import psycopg2 # type: ignore + + PSYCOPG2_ERRORS = ( + psycopg2.OperationalError, + psycopg2.InterfaceError, + psycopg2.Error, + ) +except Exception: # psycopg2 not installed or failed to import in this context + # Leave PSYCOPG2_ERRORS as an empty tuple of exception types + pass + + +@pytest.fixture +def non_authenticated_client(base_url: str) -> DPClient: + return DPClient(base_url=base_url) + + +@pytest.fixture +def client(base_url: str, auth_token: str) -> DPClient: + return DPClient(base_url=base_url, token=auth_token, prefix="Bearer") + + +@pytest.fixture +def require_db(client: DPClient): + """Skip tests that require direct DB access if the DB is not reachable. + + Tries a simple query via PGDBClient; on failure, skips the test. + """ + try: + # Trigger a DB connection with a harmless lookup + client.PGDBClient.get_user_by_id("__nonexistent__") + except Exception as exc: + pytest.skip( + "Database not available for component DB checks: " + f"{exc}\n" + "Hints: set POSTGRES_HOST to a reachable host (e.g., localhost), " + "or export POSTGRES_ALLOW_LOCAL_FALLBACK=true to fall back to localhost when 'db' is not resolvable, " + "or run `docker-compose up -d` so the service name 'db' is resolvable." + ) + + +@pytest.fixture(scope="function") +def created_user_ids(client: DPClient) -> Generator[List[str], None, None]: + ids: List[str] = [] + # Pre-test cleanup (defensive): ensure none of these ids already exist (normally random, but be safe) + if ids: + try: + client.PGDBClient.delete_users_by_ids(ids) + except PSYCOPG2_ERRORS + (RuntimeError,): + # DB not available/misconfigured; ignore cleanup in pre + pass + yield ids + # Post-test cleanup: remove created users directly from DB since there is no DELETE API + if ids: + try: + client.PGDBClient.delete_users_by_ids(ids) + except PSYCOPG2_ERRORS + (RuntimeError,): + # DB not available/misconfigured; ignore cleanup in post + pass + + +def test_health_check_component(non_authenticated_client: DPClient): + response = non_authenticated_client.health_check() + assert response.status_code == 200 + assert response.status_code == HTTPStatus.OK + + +def test_create_user_valid_component(client: DPClient, created_user_ids, require_db): + payload = { + "id": generate_israeli_id(), + "name": "Test User", + "phone": generate_random_phone_number(), + "address": "Test Street 1", + } + created_user_ids.append(payload["id"]) + + response = client.create_user(payload) + assert response.status_code == HTTPStatus.CREATED + assert response.parsed.id == payload["id"] + assert response.parsed.name == payload["name"] + + # Verify in DB via dp-client DB driver + db_user = client.PGDBClient.get_user_by_id(payload["id"]) + assert db_user is not None, "User should exist in DB after creation" + assert db_user["name"] == payload["name"] + assert db_user["phone"] == payload["phone"] + assert db_user["address"] == payload["address"] + + +def test_create_user_invalid_id_component(client: DPClient, require_db): + payload = { + "id": "123789456", # invalid checksum + "name": "Test User", + "phone": generate_random_phone_number(), + "address": "Test Street 1", + } + response = client.create_user(payload) + assert response.status_code == 400 + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.parsed is None + + # Verify not in DB + assert client.PGDBClient.get_user_by_id(payload["id"]) is None + + +def test_create_user_invalid_phone_component(client: DPClient, require_db): + payload = { + "id": generate_israeli_id(), + "name": "Test User", + "phone": "0501234567", # invalid format + "address": "Test Street 1", + } + response = client.create_user(payload) + assert response.status_code == 400 + assert response.status_code == HTTPStatus.BAD_REQUEST + assert response.parsed is None + + # Verify not in DB + assert client.PGDBClient.get_user_by_id(payload["id"]) is None + + +def test_retrieve_user_component(client: DPClient, created_user_ids, require_db): + payload = { + "id": generate_israeli_id(), + "name": "Test User", + "phone": generate_random_phone_number(), + "address": "Test Street 1", + } + created_user_ids.append(payload["id"]) + + create_resp = client.create_user(payload) + assert create_resp.status_code == HTTPStatus.CREATED + + # Verify in DB first via dp-client + assert client.PGDBClient.users_exist([payload["id"]]) + + response = client.get_user(payload["id"]) + assert response.status_code == HTTPStatus.OK + assert response.parsed.id == payload["id"] + assert response.parsed.name == payload["name"] + + +def test_list_users_component(client: DPClient, created_user_ids, require_db): + users_data = [ + { + "id": generate_israeli_id(), + "name": "Test User", + "phone": generate_random_phone_number(), + "address": "Street 1", + }, + { + "id": generate_israeli_id(), + "name": "Test User", + "phone": generate_random_phone_number(), + "address": "Street 2", + }, + ] + for u in users_data: + created_user_ids.append(u["id"]) + resp = client.create_user(u) + assert resp.status_code == HTTPStatus.CREATED + + # DB cross-check: all created users present (via dp-client DB driver) + ids = [u["id"] for u in users_data] + assert client.PGDBClient.users_exist(ids) + + # API list still OK + list_resp = client.list_users() + assert list_resp.status_code == HTTPStatus.OK + returned_ids_api = {u.id for u in list_resp.parsed} + for u in users_data: + assert u["id"] in returned_ids_api diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..65e9c28 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +try: + # Enable local hostname fallback for Postgres unless explicitly disabled + # This lets component tests run locally when POSTGRES_HOST (e.g., 'db') isn't resolvable. + os.environ.setdefault("POSTGRES_ALLOW_LOCAL_FALLBACK", "true") + + # Load environment variables from repository .env so tests and CI have required config + from dotenv import load_dotenv + + _repo_root = Path(__file__).resolve().parents[1] + load_dotenv(dotenv_path=_repo_root / ".env", override=False) +except Exception: + # If python-dotenv is not available, silently continue; CI should provide envs + pass + + +@pytest.fixture +def base_url() -> str: + """Resolve API base URL from environment with sensible defaults. + + Prefers API_BASE_URL; otherwise builds from METADATA_HOST and METADATA_PORT. + """ + return os.environ.get( + "API_BASE_URL", + f"http://{os.environ.get('METADATA_HOST', 'localhost')}:{os.environ.get('METADATA_PORT', '8000')}", + ) + + +@pytest.fixture +def auth_token(base_url: str) -> str: + """Get an access token for authenticated API tests. + + Uses API_TOKEN if set; otherwise performs username/password login against /api/token/. + """ + token = os.environ.get("API_TOKEN") + if token: + return token + + import requests # imported here to avoid hard dependency if not needed + + username = os.environ.get("TEST_USERNAME") + password = os.environ.get("TEST_PASSWORD") + if not username or not password: + raise RuntimeError("TEST_USERNAME/TEST_PASSWORD or API_TOKEN must be set for authenticated tests") + + resp = requests.post( + f"{base_url}/api/token/", + json={"username": username, "password": password}, + timeout=10, + ) + resp.raise_for_status() + return resp.json()["access"] diff --git a/tests/unit/test_user_model.py b/tests/unit/test_user_model.py index 52d8eec..a082564 100644 --- a/tests/unit/test_user_model.py +++ b/tests/unit/test_user_model.py @@ -1,8 +1,6 @@ -import os from http import HTTPStatus import pytest -import requests from metadata_client import AuthenticatedClient, Client from metadata_client.api.health import health_retrieve from metadata_client.api.users import ( @@ -15,36 +13,14 @@ from src.tools import generate_israeli_id, generate_random_phone_number -@pytest.fixture -def base_url() -> str: - return os.environ.get( - "API_BASE_URL", - f"http://{os.environ.get('METADATA_HOST', 'localhost')}:{os.environ.get('METADATA_PORT', '8000')}", - ) - - @pytest.fixture def non_authenticated_client(base_url: str): return Client(base_url=base_url) @pytest.fixture -def client(base_url: str): - token = os.environ.get("API_TOKEN") - if not token: - username = os.environ.get("TEST_USERNAME") - password = os.environ.get("TEST_PASSWORD") - if not username or not password: - raise RuntimeError("TEST_USERNAME/TEST_PASSWORD or API_TOKEN must be set for authenticated tests") - resp = requests.post( - f"{base_url}/api/token/", - json={"username": username, "password": password}, - timeout=10, - ) - resp.raise_for_status() - token = resp.json()["access"] - - return AuthenticatedClient(base_url=base_url, token=token, prefix="Bearer") +def client(base_url: str, auth_token: str): + return AuthenticatedClient(base_url=base_url, token=auth_token, prefix="Bearer") def test_health_check(non_authenticated_client): From 723dd5363a033740f1300bf7d6383edcc138974f Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:08:33 +0300 Subject: [PATCH 02/15] wip1 --- .github/workflows/ci_cd.yml | 12 ++ .github/workflows/component_tests.yml | 163 ++++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 .github/workflows/component_tests.yml diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 56b273b..bbb4196 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -28,3 +28,15 @@ jobs: file: "Dockerfile" image_name: "metadata_server" secrets: inherit + + component-tests: + name: 🧩 🔍 Component Tests + needs: build-server-image + uses: sidkos/metadata_server/.github/workflows/component_tests.yml@main + with: + image_name: "metadata_server" + image_tag: "latest" + tests_path: "./tests/component" + tests_command: "pytest" + build_and_install_script: "./scripts/build_and_install_dp_client.sh" + secrets: inherit diff --git a/.github/workflows/component_tests.yml b/.github/workflows/component_tests.yml new file mode 100644 index 0000000..5c649f6 --- /dev/null +++ b/.github/workflows/component_tests.yml @@ -0,0 +1,163 @@ +name: 🧩 🔍 🧱 Component Tests +run-name: "Image: ${{ inputs.image_name }}:${{ inputs.image_tag }} | Command: ${{ inputs.tests_command }} Context: ${{ inputs.tests_path }}" + +on: + workflow_call: + inputs: + image_name: + description: 'Container image name (without registry and owner)' + required: false + type: string + default: 'metadata_server' + image_tag: + description: 'Container image tag' + required: false + type: string + default: 'latest' + tests_path: + description: 'Path to component tests' + required: false + type: string + default: './tests/component' + tests_command: + description: 'Command to run the tests' + required: false + type: string + default: 'pytest' + build_and_install_script: + description: 'Script to build and install client packages (dp-client)' + required: false + type: string + default: './scripts/build_and_install_dp_client.sh' + workflow_dispatch: + inputs: + branch: + description: 'Branch to run the workflow on' + required: true + type: string + image_name: + description: 'Container image name (without registry and owner)' + required: false + type: string + default: 'metadata_server' + image_tag: + description: 'Container image tag' + required: false + type: string + default: 'latest' + tests_path: + description: 'Path to component tests' + required: false + type: string + default: './tests/component' + tests_command: + description: 'Command to run the tests' + required: false + type: string + default: 'pytest' + build_and_install_script: + description: 'Script to build and install client packages (dp-client)' + required: false + type: string + default: './scripts/build_and_install_dp_client.sh' + +permissions: + id-token: write + contents: write + +jobs: + component-tests: + name: "🧩 🔍 Component Tests" + runs-on: ubuntu-latest + timeout-minutes: 20 + + defaults: + run: + shell: bash + + services: + postgres: + image: postgres:15 + env: + POSTGRES_DB: ${{ vars.POSTGRES_DB }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ vars.POSTGRES_PASSWORD }} + options: >- + --health-cmd="pg_isready -U ${{ vars.POSTGRES_USER }}" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + + env: + DJANGO_SECRET_KEY: ${{ secrets.DJANGO_SECRET_KEY }} + DJANGO_DEBUG: ${{ vars.DJANGO_DEBUG }} + DJANGO_ALLOWED_HOSTS: ${{ vars.DJANGO_ALLOWED_HOSTS }} + TEST_USERNAME: ${{ vars.TEST_USERNAME }} + TEST_PASSWORD: ${{ secrets.TEST_PASSWORD }} + METADATA_HOST: localhost + METADATA_PORT: 8000 + POSTGRES_HOST: localhost + POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_DB: ${{ vars.POSTGRES_DB }} + POSTGRES_USER: ${{ vars.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ vars.POSTGRES_PASSWORD }} + + steps: + - name: SetUp Python Project + uses: sidkos/metadata_server/.github/actions/setup_python_project@main + + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Print runner env + run: env | sort + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Pull server image + run: | + IMAGE_REF="ghcr.io/${{ github.repository }}/${{ inputs.image_name }}:${{ inputs.image_tag }}" + echo "Pulling $IMAGE_REF" + docker pull "$IMAGE_REF" + echo "IMAGE_REF=$IMAGE_REF" >> $GITHUB_ENV + + - name: Run server container + run: | + DB_PORT=${{ job.services.postgres.ports[5432] }} + docker run -d --name metadata-server \ + --add-host=host.docker.internal:host-gateway \ + -p 8000:8000 \ + -e DJANGO_SECRET_KEY="${{ env.DJANGO_SECRET_KEY }}" \ + -e DJANGO_DEBUG="${{ env.DJANGO_DEBUG }}" \ + -e DJANGO_ALLOWED_HOSTS="${{ env.DJANGO_ALLOWED_HOSTS }}" \ + -e POSTGRES_DB="${{ env.POSTGRES_DB }}" \ + -e POSTGRES_USER="${{ env.POSTGRES_USER }}" \ + -e POSTGRES_PASSWORD="${{ env.POSTGRES_PASSWORD }}" \ + -e POSTGRES_HOST=host.docker.internal \ + -e POSTGRES_PORT="$DB_PORT" \ + -e TEST_USERNAME="${{ env.TEST_USERNAME }}" \ + -e TEST_PASSWORD="${{ env.TEST_PASSWORD }}" \ + "$IMAGE_REF" \ + bash -lc 'python src/manage.py migrate && if [ -n "$TEST_USERNAME" ] && [ -n "$TEST_PASSWORD" ]; then python src/manage.py shell -c "from django.contrib.auth import get_user_model; import os; U=get_user_model(); u=os.environ.get(\"TEST_USERNAME\"); p=os.environ.get(\"TEST_PASSWORD\"); user, created = U.objects.get_or_create(username=u, defaults={\"is_staff\": True, \"is_superuser\": True}); user.set_password(p); user.save(); print(\"Test user ensured:\", user.username)"; else echo "TEST_USERNAME/TEST_PASSWORD not set; skipping test user creation"; fi && python src/manage.py runserver 0.0.0.0:8000' + + - name: Wait for server to be ready + run: | + for i in {1..30}; do + if curl -fsS "http://${{ env.METADATA_HOST }}:${{ env.METADATA_PORT }}/api/health/" >/dev/null; then + echo "Server is ready"; exit 0 + fi + echo "Waiting for server... ($i)"; sleep 2 + done + echo "Server did not become ready in time"; docker logs metadata-server || true; exit 1 + + - name: Build and install dp-client + if: ${{ inputs.build_and_install_script }} + run: ${{ inputs.build_and_install_script }} + + - name: Run Component Tests + run: "python -m ${{ inputs.tests_command }} ${{ inputs.tests_path }} -s -v" From f198be98853b6d1f6a4c91f14c060146f66ce63a Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:09:08 +0300 Subject: [PATCH 03/15] change branch pointer --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index bbb4196..02755e1 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -32,7 +32,7 @@ jobs: component-tests: name: 🧩 🔍 Component Tests needs: build-server-image - uses: sidkos/metadata_server/.github/workflows/component_tests.yml@main + uses: sidkos/metadata_server/.github/workflows/component_tests.yml@add-dp-client with: image_name: "metadata_server" image_tag: "latest" From 19d8a02883b5e2f9987ef3deb7e06a528963d5a1 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:27:13 +0300 Subject: [PATCH 04/15] fix PG port --- .github/workflows/component_tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/component_tests.yml b/.github/workflows/component_tests.yml index 5c649f6..cf9683a 100644 --- a/.github/workflows/component_tests.yml +++ b/.github/workflows/component_tests.yml @@ -82,6 +82,8 @@ jobs: POSTGRES_DB: ${{ vars.POSTGRES_DB }} POSTGRES_USER: ${{ vars.POSTGRES_USER }} POSTGRES_PASSWORD: ${{ vars.POSTGRES_PASSWORD }} + ports: + - 5432:5432 options: >- --health-cmd="pg_isready -U ${{ vars.POSTGRES_USER }}" --health-interval=10s @@ -97,7 +99,7 @@ jobs: METADATA_HOST: localhost METADATA_PORT: 8000 POSTGRES_HOST: localhost - POSTGRES_PORT: ${{ job.services.postgres.ports[5432] }} + POSTGRES_PORT: "5432" POSTGRES_DB: ${{ vars.POSTGRES_DB }} POSTGRES_USER: ${{ vars.POSTGRES_USER }} POSTGRES_PASSWORD: ${{ vars.POSTGRES_PASSWORD }} @@ -128,7 +130,7 @@ jobs: - name: Run server container run: | - DB_PORT=${{ job.services.postgres.ports[5432] }} + DB_PORT=${{ env.METADATA_PORT }} docker run -d --name metadata-server \ --add-host=host.docker.internal:host-gateway \ -p 8000:8000 \ From acc8dde5cdb77c2908cf5d3176620480bb093026 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:33:28 +0300 Subject: [PATCH 05/15] fix install client --- .github/workflows/ci.yml | 2 +- dp-client/dp_client/__init__.py | 2 +- dp-client/dp_client/api/health.py | 6 +++--- dp-client/dp_client/api/users.py | 2 +- dp-client/dp_client/clients/metadata.py | 2 +- dp-client/dp_client/db/drivers/postgres.py | 2 +- tests/component/test_user_model_component.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1ac8ea1..c16d2aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: static_code_analysis: uses: sidkos/metadata_server/.github/workflows/static_code_analysis.yml@main with: - build_and_install_script: "./scripts/build_and_install_open_api_client.sh" + build_and_install_script: "./scripts/build_and_install_dp_client.sh" secrets: inherit openapi-validation: diff --git a/dp-client/dp_client/__init__.py b/dp-client/dp_client/__init__.py index f3ba3aa..470e894 100644 --- a/dp-client/dp_client/__init__.py +++ b/dp-client/dp_client/__init__.py @@ -2,7 +2,7 @@ try: __version__ = _metadata.version("dp-client") -except _metadata.PackageNotFoundError: # pragma: no cover +except _metadata.PackageNotFoundError: __version__ = "0.0.0" from .api import HealthAPI, UsersAPI # noqa: F401 diff --git a/dp-client/dp_client/api/health.py b/dp-client/dp_client/api/health.py index 6681f6c..552683e 100644 --- a/dp-client/dp_client/api/health.py +++ b/dp-client/dp_client/api/health.py @@ -8,10 +8,10 @@ class HealthAPI: def __init__(self, client: Any) -> None: try: - from metadata_client.api.health import health_retrieve as _health_retrieve # noqa: WPS433 - except Exception as exc: # pragma: no cover + from metadata_client.api.health import health_retrieve as _health_retrieve + except Exception as exc: raise RuntimeError( - "metadata-client is required but not installed or failed to import. " + "Metadata-client is required but not installed or failed to import. " f"Original error: {exc}. Install it or run: pip install -r dp-client/requirements.txt" ) from exc self._client = client diff --git a/dp-client/dp_client/api/users.py b/dp-client/dp_client/api/users.py index d1a322f..9f4b61e 100644 --- a/dp-client/dp_client/api/users.py +++ b/dp-client/dp_client/api/users.py @@ -12,7 +12,7 @@ def __init__(self, client: Any) -> None: from metadata_client.api.users import users_list as _users_list from metadata_client.api.users import users_retrieve as _users_retrieve from metadata_client.models import User as _User - except Exception as exc: # pragma: no cover + except Exception as exc: raise RuntimeError( "metadata-client is required but not installed or failed to import. " f"Original error: {exc}. Install it or run: pip install -r dp-client/requirements.txt" diff --git a/dp-client/dp_client/clients/metadata.py b/dp-client/dp_client/clients/metadata.py index 5ac1afb..e66b271 100644 --- a/dp-client/dp_client/clients/metadata.py +++ b/dp-client/dp_client/clients/metadata.py @@ -14,7 +14,7 @@ def __init__(self) -> None: try: from metadata_client import AuthenticatedClient as _AuthenticatedClient # type: ignore from metadata_client import Client as _Client - except Exception as exc: # pragma: no cover + except Exception as exc: raise RuntimeError( "metadata-client is required but not installed or failed to import. " f"Original error: {exc}. Install it or run: pip install -r dp-client/requirements.txt" diff --git a/dp-client/dp_client/db/drivers/postgres.py b/dp-client/dp_client/db/drivers/postgres.py index 0ec2484..9d6a01b 100644 --- a/dp-client/dp_client/db/drivers/postgres.py +++ b/dp-client/dp_client/db/drivers/postgres.py @@ -4,7 +4,7 @@ try: import psycopg2 -except Exception as exc: # pragma: no cover +except Exception as exc: raise RuntimeError( "psycopg2-binary is required for Postgres driver. Install it via dp-client dependencies." ) from exc diff --git a/tests/component/test_user_model_component.py b/tests/component/test_user_model_component.py index 96035d0..3f36c45 100644 --- a/tests/component/test_user_model_component.py +++ b/tests/component/test_user_model_component.py @@ -10,7 +10,7 @@ # This lets us catch specific DB connectivity/driver errors without a blanket Exception. # Always keep a consistent, typed value to satisfy mypy. PSYCOPG2_ERRORS: tuple[type[BaseException], ...] = tuple() -try: # pragma: no cover - optional dependency for tests +try: import psycopg2 # type: ignore PSYCOPG2_ERRORS = ( From 378f0184f109d8617c10c49518df7e128e41e804 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:45:42 +0300 Subject: [PATCH 06/15] mypy --- mypy.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index f65f93e..8d52b03 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,6 +1,6 @@ [mypy] files = src, tests -exclude = ^(dist|\.git|\.venv|venv|\.pytest_cache|\.mypy_cache|\.tox|\.eggs|.*egg-info|build|client|metadata-client|src/metadata_manager/migrations)/ +exclude = ^(dist|\.git|\.venv|venv|\.pytest_cache|\.mypy_cache|\.tox|\.eggs|.*egg-info|build|client|metadata-client|src/metadata_manager/migrations)/|src/tools\.py$|src/manage\.py$|src/server/|src/metadata_manager/|src/__init__\.py$ warn_unused_ignores = True warn_unreachable = True allow_redefinition = True @@ -9,6 +9,7 @@ strict = False plugins = mypy_django_plugin.main explicit_package_bases = True namespace_packages = True +mypy_path = src [mypy-setuptools.*] ignore_missing_imports = True From 0b460dbbcfb5b786f7e78c9c4abca921e351db70 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:50:33 +0300 Subject: [PATCH 07/15] mypy styling 2 --- dp-client/dp_client/clients/metadata.py | 2 +- dp-client/dp_client/db/pg.py | 23 ++++++++++++++++------- mypy.ini | 3 +++ 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/dp-client/dp_client/clients/metadata.py b/dp-client/dp_client/clients/metadata.py index e66b271..49dc7fa 100644 --- a/dp-client/dp_client/clients/metadata.py +++ b/dp-client/dp_client/clients/metadata.py @@ -12,7 +12,7 @@ class MetaDataServerAPIClientFactory: def __init__(self) -> None: # Import lazily to keep dp-client importable even if metadata-client is missing try: - from metadata_client import AuthenticatedClient as _AuthenticatedClient # type: ignore + from metadata_client import AuthenticatedClient as _AuthenticatedClient from metadata_client import Client as _Client except Exception as exc: raise RuntimeError( diff --git a/dp-client/dp_client/db/pg.py b/dp-client/dp_client/db/pg.py index 244594c..03d1747 100644 --- a/dp-client/dp_client/db/pg.py +++ b/dp-client/dp_client/db/pg.py @@ -66,20 +66,29 @@ def __init__( "yes", "on", } - resolved_host = final_host # type: ignore[assignment] + # At this point, required parameters are present; narrow Optional types for type-checker + from typing import cast + + final_host_str = cast(str, final_host) + final_port_int = cast(int, final_port) + final_db_str = cast(str, final_db) + final_user_str = cast(str, final_user) + final_password_str = cast(str, final_password) + + resolved_host: str = final_host_str try: # Try to resolve the host name; if it fails, consider fallback. - socket.gethostbyname(str(final_host)) + socket.gethostbyname(final_host_str) except Exception: if allow_fallback: resolved_host = "localhost" self._driver = PostgresDriver( - host=resolved_host, # type: ignore[arg-type] - port=int(final_port), # type: ignore[arg-type] - dbname=final_db, # type: ignore[arg-type] - user=final_user, # type: ignore[arg-type] - password=final_password, # type: ignore[arg-type] + host=resolved_host, + port=final_port_int, + dbname=final_db_str, + user=final_user_str, + password=final_password_str, ) self._table = table diff --git a/mypy.ini b/mypy.ini index 8d52b03..a467363 100644 --- a/mypy.ini +++ b/mypy.ini @@ -41,3 +41,6 @@ ignore_missing_imports = True [mypy-metadata_manager.*] ignore_missing_imports = True + +[mypy-psycopg2.*] +ignore_missing_imports = True From 3545cf6d872b43bed8c748c76235a3b6dd51bda3 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 22:57:14 +0300 Subject: [PATCH 08/15] mypy styling 3 --- dp-client/dp_client/client.py | 8 -------- dp-client/dp_client/db/pg.py | 5 +---- scripts/build_and_install_dp_client.sh | 3 +-- scripts/build_dp_client.sh | 4 ---- tests/component/test_user_model_component.py | 2 +- 5 files changed, 3 insertions(+), 19 deletions(-) diff --git a/dp-client/dp_client/client.py b/dp-client/dp_client/client.py index e8184ff..9a235e5 100644 --- a/dp-client/dp_client/client.py +++ b/dp-client/dp_client/client.py @@ -28,21 +28,13 @@ def __init__( prefix: str = "Bearer", timeout: float = 10.0, ) -> None: - # Build the underlying metadata server client self._client_factory = MetaDataServerAPIClientFactory() self.MetaDataServerAPIClient: Any = self._client_factory.build(base_url=base_url, token=token, prefix=prefix) - - # Compose API domains self.UsersApi = UsersAPI(self.MetaDataServerAPIClient) self.HealthAPI = HealthAPI(self.MetaDataServerAPIClient) - - # DB helper (optional, raises clear error if ORM not available when used) self.PGDBClient = PGDBClient() - - # Store timeout for potential future use (e.g., retries) self._timeout = timeout - # Backward-compatible thin wrappers def health_check(self): return self.HealthAPI.health_check() diff --git a/dp-client/dp_client/db/pg.py b/dp-client/dp_client/db/pg.py index 03d1747..c63f552 100644 --- a/dp-client/dp_client/db/pg.py +++ b/dp-client/dp_client/db/pg.py @@ -2,7 +2,7 @@ import os import socket -from typing import Iterable, Optional +from typing import Iterable, Optional, cast from .drivers.postgres import PostgresDriver @@ -67,8 +67,6 @@ def __init__( "on", } # At this point, required parameters are present; narrow Optional types for type-checker - from typing import cast - final_host_str = cast(str, final_host) final_port_int = cast(int, final_port) final_db_str = cast(str, final_db) @@ -99,7 +97,6 @@ def get_user_by_id(self, user_id: str): ) if not row: return None - # Return a simple dict to avoid coupling return {"id": row[0], "name": row[1], "phone": row[2], "address": row[3]} def users_exist(self, user_ids: Iterable[str]) -> bool: diff --git a/scripts/build_and_install_dp_client.sh b/scripts/build_and_install_dp_client.sh index 5d699c1..f7440b0 100755 --- a/scripts/build_and_install_dp_client.sh +++ b/scripts/build_and_install_dp_client.sh @@ -1,7 +1,6 @@ #!/bin/bash set -euo pipefail -# 1) Build and install metadata-client first (dp-client depends on it) echo "[dp-client] Building and installing metadata-client (dependency) ..." ./scripts/build_and_install_open_api_client.sh @@ -22,7 +21,7 @@ if [[ -n "${PYVER}" ]]; then else echo "[dp-client] Installing wheel (Python version unknown)" fi -# Install without resolving dependencies since metadata-client was already installed locally above + pip install --no-deps "${WHEEL}" --force-reinstall echo "[dp-client] Verifying installation..." diff --git a/scripts/build_dp_client.sh b/scripts/build_dp_client.sh index c401b7e..8aa35eb 100755 --- a/scripts/build_dp_client.sh +++ b/scripts/build_dp_client.sh @@ -4,13 +4,11 @@ set -euo pipefail PKG_DIR="dp-client" PKG_NAME="dp-client" -# Ensure Poetry is present if ! command -v poetry >/dev/null 2>&1; then echo "[dp-client] Poetry not found. Installing via pip..." pip install poetry fi -# Read version from repo root setup.py (align with metadata-client) VERSION=$(grep "__version__" setup.py | awk -F '"' '{print $2}') if [[ -z "${VERSION}" ]]; then echo "[dp-client] Could not determine version from setup.py" @@ -19,8 +17,6 @@ fi pushd "$PKG_DIR" >/dev/null -# Optionally pin metadata-client dependency. -# If version is 0.0.0 (local-only), avoid exact pin to prevent resolver hitting remote index. if grep -q '^metadata-client' pyproject.toml; then if [[ "${VERSION}" == "0.0.0" ]]; then echo "[dp-client] Detected local development version (0.0.0). Using unpinned dependency for metadata-client." diff --git a/tests/component/test_user_model_component.py b/tests/component/test_user_model_component.py index 3f36c45..cfe5e43 100644 --- a/tests/component/test_user_model_component.py +++ b/tests/component/test_user_model_component.py @@ -11,7 +11,7 @@ # Always keep a consistent, typed value to satisfy mypy. PSYCOPG2_ERRORS: tuple[type[BaseException], ...] = tuple() try: - import psycopg2 # type: ignore + import psycopg2 PSYCOPG2_ERRORS = ( psycopg2.OperationalError, From 46bb745b365d60109a3e9b0f1b61a557147e29ef Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 23:06:50 +0300 Subject: [PATCH 09/15] fix path to image --- .github/workflows/component_tests.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/component_tests.yml b/.github/workflows/component_tests.yml index cf9683a..bd16902 100644 --- a/.github/workflows/component_tests.yml +++ b/.github/workflows/component_tests.yml @@ -108,9 +108,6 @@ jobs: - name: SetUp Python Project uses: sidkos/metadata_server/.github/actions/setup_python_project@main - - name: Checkout repository - uses: actions/checkout@v4 - - name: Print runner env run: env | sort @@ -123,14 +120,14 @@ jobs: - name: Pull server image run: | - IMAGE_REF="ghcr.io/${{ github.repository }}/${{ inputs.image_name }}:${{ inputs.image_tag }}" + IMAGE_REF="ghcr.io/${{ github.repository_owner }}/${{ inputs.image_name }}:${{ inputs.image_tag }}" echo "Pulling $IMAGE_REF" docker pull "$IMAGE_REF" echo "IMAGE_REF=$IMAGE_REF" >> $GITHUB_ENV - name: Run server container run: | - DB_PORT=${{ env.METADATA_PORT }} + DB_PORT=${{ env.POSTGRES_PORT }} docker run -d --name metadata-server \ --add-host=host.docker.internal:host-gateway \ -p 8000:8000 \ From 17d216ba533cbc89b42f2e020e1e6d34430ef72b Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 23:17:06 +0300 Subject: [PATCH 10/15] fix start server --- .github/workflows/component_tests.yml | 3 ++- Dockerfile | 4 +++- scripts/start_server_in_container.sh | 32 +++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 scripts/start_server_in_container.sh diff --git a/.github/workflows/component_tests.yml b/.github/workflows/component_tests.yml index bd16902..bbb3349 100644 --- a/.github/workflows/component_tests.yml +++ b/.github/workflows/component_tests.yml @@ -141,8 +141,9 @@ jobs: -e POSTGRES_PORT="$DB_PORT" \ -e TEST_USERNAME="${{ env.TEST_USERNAME }}" \ -e TEST_PASSWORD="${{ env.TEST_PASSWORD }}" \ + -v "${{ github.workspace }}/scripts:/app/scripts:ro" \ "$IMAGE_REF" \ - bash -lc 'python src/manage.py migrate && if [ -n "$TEST_USERNAME" ] && [ -n "$TEST_PASSWORD" ]; then python src/manage.py shell -c "from django.contrib.auth import get_user_model; import os; U=get_user_model(); u=os.environ.get(\"TEST_USERNAME\"); p=os.environ.get(\"TEST_PASSWORD\"); user, created = U.objects.get_or_create(username=u, defaults={\"is_staff\": True, \"is_superuser\": True}); user.set_password(p); user.save(); print(\"Test user ensured:\", user.username)"; else echo "TEST_USERNAME/TEST_PASSWORD not set; skipping test user creation"; fi && python src/manage.py runserver 0.0.0.0:8000' + bash /app/scripts/start_server_in_container.sh - name: Wait for server to be ready run: | diff --git a/Dockerfile b/Dockerfile index a34aaa7..fbd7b43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,8 @@ COPY requirements.txt . RUN pip install --upgrade pip && pip install -r requirements.txt COPY src/ ./src/ +COPY scripts/start_server_in_container.sh /app/scripts/start_server_in_container.sh +RUN chmod +x /app/scripts/start_server_in_container.sh -CMD ["bash", "-c", "python src/manage.py migrate && python src/manage.py runserver 0.0.0.0:8000"] +CMD ["bash", "/app/scripts/start_server_in_container.sh"] diff --git a/scripts/start_server_in_container.sh b/scripts/start_server_in_container.sh new file mode 100644 index 0000000..5c43362 --- /dev/null +++ b/scripts/start_server_in_container.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +# This script runs inside the app container. It expects the following env vars: +# - DJANGO_* for Django config +# - POSTGRES_* for DB connectivity +# - TEST_USERNAME / TEST_PASSWORD (optional) to ensure a test superuser exists + +cd /app + +echo "[container] Running migrations..." +python src/manage.py migrate --noinput + +if [[ -n "${TEST_USERNAME:-}" && -n "${TEST_PASSWORD:-}" ]]; then + echo "[container] Ensuring test user ${TEST_USERNAME} exists..." + python - <<'PY' +from django.contrib.auth import get_user_model +import os +U = get_user_model() +u = os.environ.get('TEST_USERNAME') +p = os.environ.get('TEST_PASSWORD') +user, created = U.objects.get_or_create(username=u, defaults={'is_staff': True, 'is_superuser': True}) +user.set_password(p) +user.save() +print('Test user ensured:', user.username) +PY +else + echo "[container] TEST_USERNAME/TEST_PASSWORD not set; skipping test user creation" +fi + +echo "[container] Starting Django server..." +exec python src/manage.py runserver 0.0.0.0:8000 From 5aca27dbef68ed996b47e8cfa89bd5f7f19f24de Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 23:24:28 +0300 Subject: [PATCH 11/15] revert me --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 02755e1..ff16652 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -31,7 +31,7 @@ jobs: component-tests: name: 🧩 🔍 Component Tests - needs: build-server-image +# needs: build-server-image uses: sidkos/metadata_server/.github/workflows/component_tests.yml@add-dp-client with: image_name: "metadata_server" From 97a0a5d5ed0629005e05313ca2b3cf4cc2bad58d Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 23:25:21 +0300 Subject: [PATCH 12/15] fix run server --- .github/workflows/component_tests.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/component_tests.yml b/.github/workflows/component_tests.yml index bbb3349..0262638 100644 --- a/.github/workflows/component_tests.yml +++ b/.github/workflows/component_tests.yml @@ -131,16 +131,16 @@ jobs: docker run -d --name metadata-server \ --add-host=host.docker.internal:host-gateway \ -p 8000:8000 \ - -e DJANGO_SECRET_KEY="${{ env.DJANGO_SECRET_KEY }}" \ - -e DJANGO_DEBUG="${{ env.DJANGO_DEBUG }}" \ - -e DJANGO_ALLOWED_HOSTS="${{ env.DJANGO_ALLOWED_HOSTS }}" \ - -e POSTGRES_DB="${{ env.POSTGRES_DB }}" \ - -e POSTGRES_USER="${{ env.POSTGRES_USER }}" \ - -e POSTGRES_PASSWORD="${{ env.POSTGRES_PASSWORD }}" \ + -e DJANGO_SECRET_KEY \ + -e DJANGO_DEBUG \ + -e DJANGO_ALLOWED_HOSTS \ + -e POSTGRES_DB \ + -e POSTGRES_USER \ + -e POSTGRES_PASSWORD \ -e POSTGRES_HOST=host.docker.internal \ - -e POSTGRES_PORT="$DB_PORT" \ - -e TEST_USERNAME="${{ env.TEST_USERNAME }}" \ - -e TEST_PASSWORD="${{ env.TEST_PASSWORD }}" \ + -e POSTGRES_PORT=$DB_PORT \ + -e TEST_USERNAME \ + -e TEST_PASSWORD \ -v "${{ github.workspace }}/scripts:/app/scripts:ro" \ "$IMAGE_REF" \ bash /app/scripts/start_server_in_container.sh From 4723636af844116d8746dbf0683f8567b33df54c Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 23:35:40 +0300 Subject: [PATCH 13/15] fix run server 2 --- scripts/start_server_in_container.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/start_server_in_container.sh b/scripts/start_server_in_container.sh index 5c43362..ed1f527 100644 --- a/scripts/start_server_in_container.sh +++ b/scripts/start_server_in_container.sh @@ -14,8 +14,12 @@ python src/manage.py migrate --noinput if [[ -n "${TEST_USERNAME:-}" && -n "${TEST_PASSWORD:-}" ]]; then echo "[container] Ensuring test user ${TEST_USERNAME} exists..." python - <<'PY' -from django.contrib.auth import get_user_model import os +# Ensure Django is configured for standalone script execution +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'server.settings') +import django +django.setup() +from django.contrib.auth import get_user_model U = get_user_model() u = os.environ.get('TEST_USERNAME') p = os.environ.get('TEST_PASSWORD') From c54cb8d7ee6def0fd0f37b37d50d31ad607b2517 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sat, 4 Oct 2025 23:38:40 +0300 Subject: [PATCH 14/15] Revert "revert me" This reverts commit 5aca27dbef68ed996b47e8cfa89bd5f7f19f24de. --- .github/workflows/ci_cd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index ff16652..02755e1 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -31,7 +31,7 @@ jobs: component-tests: name: 🧩 🔍 Component Tests -# needs: build-server-image + needs: build-server-image uses: sidkos/metadata_server/.github/workflows/component_tests.yml@add-dp-client with: image_name: "metadata_server" From 6acdb729a1aa431dc1e5d6c8d7d815effd01ff88 Mon Sep 17 00:00:00 2001 From: sidkos Date: Sun, 5 Oct 2025 00:31:48 +0300 Subject: [PATCH 15/15] done --- .github/workflows/ci_cd.yml | 2 +- README.md | 233 +++++++++++++++++++++------ dp-client/README.md | 131 ++++++++++----- scripts/install_all_requirements.sh | 43 +++++ scripts/start_server_in_container.sh | 0 5 files changed, 320 insertions(+), 89 deletions(-) create mode 100755 scripts/install_all_requirements.sh mode change 100644 => 100755 scripts/start_server_in_container.sh diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 02755e1..bbb4196 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -32,7 +32,7 @@ jobs: component-tests: name: 🧩 🔍 Component Tests needs: build-server-image - uses: sidkos/metadata_server/.github/workflows/component_tests.yml@add-dp-client + uses: sidkos/metadata_server/.github/workflows/component_tests.yml@main with: image_name: "metadata_server" image_tag: "latest" diff --git a/README.md b/README.md index 33d11d4..87799ae 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,191 @@ -# Authentication (JWT) — Quick Guide +# Project overview +A small, production-ready example of a REST API server with an OpenAPI schema, a generated low‑level client, and a higher‑level testing client. It demonstrates API design, validation, JWT auth, containerization, CI/CD, and tests (unit and component). -Short overview of the authentication we implemented. +- Server endpoints: `POST /api/users/`, `GET /api/users/{id}/`, `GET /api/users/`, `GET /api/health/`. +- Persists users in PostgreSQL with input validation and clear error responses. -- Approach: JWT-only API auth using Django REST Framework + SimpleJWT. - - Global permission is IsAuthenticated (all endpoints require a token). - - Only exception: /api/health/ is public for probes. - - OpenAPI via drf-spectacular advertises bearerAuth (JWT) globally. - - Client usage relies on AuthenticatedClient with Authorization: Bearer . +# Deliverables and separation +- metadata_server (this repository): Django REST Framework server and Docker image. +- metadata_client: Generated Python client from the server’s OpenAPI (thin, endpoint‑oriented). Used in unit tests. +- dp-client: Higher‑level Python client intended for component tests. It composes the generated metadata_client plus optional DB checks via its own DB drivers. -References +Notes on separation and SDLC: +- metadata_server, metadata_client, and dp-client should each have an independent SDLC: separate repositories, versioning, release pipelines, and artifacts. +- Best practice: split dp-client into its own repository and manage it there independently. +- Best practice: publish both metadata_client and dp-client to a package registry (e.g., PyPI, GH Packages, CodeArtifact) and consume them by version in downstream projects. + +# Versioning status +- Versioning of published deliverables is not yet implemented here; currently we use on‑the‑fly builds or "latest" artifacts during CI and local development. +- Should be introduced: semantic versioning for metadata_server, metadata_client, and dp-client, published releases, and version pinning in consumers for reproducible builds. + +# Technologies used and why +- Django + Django REST Framework: rapid API development, serializers, validation, permissions. +- SimpleJWT (DRF): standard JWT‑based authentication. +- drf-spectacular: OpenAPI generation for accurate client generation. +- PostgreSQL: reliable relational database with first‑class support on GitHub Actions. +- Docker: consistent runtime for the server. +- GitHub Actions: automated CI/CD with reusable workflows. + +When to use what: +- Use metadata_client for thin, OpenAPI‑aligned calls (used in unit tests). +- Use dp-client for higher‑level test flows including optional DB verification (used in component tests). + +# API surface (summary) +- POST /api/users/ — Create a user +- GET /api/users/{id}/ — Retrieve user by ID +- GET /api/users/ — List all user IDs +- GET /api/health/ — Public health check (no auth) + +Validation rules: +- id: Valid Israeli ID (checksum validated) +- phone: International format starting with `+` +- name: Required +- address: Required + +# Authentication (JWT) +Approach: JWT‑only API auth using Django REST Framework + SimpleJWT. +- Global permission is IsAuthenticated (all endpoints require a token) except `/api/health/`. +- OpenAPI advertises global `bearerAuth` security scheme. + +Usage flow: +- Obtain token at `POST /api/token/` with JSON `{"username":"", "password":"

"}` → `{"access","refresh"}`. +- Call protected endpoints with header `Authorization: Bearer `. +- Refresh access tokens at `POST /api/token/refresh/` with `{"refresh"}`. + +References: - DRF: https://www.django-rest-framework.org/ - SimpleJWT: https://django-rest-framework-simplejwt.readthedocs.io/en/latest/ - drf-spectacular: https://drf-spectacular.readthedocs.io/en/latest/ -## How it works in production -- Clients obtain tokens at POST /api/token/ with JSON {"username", "password"}; response contains {"access", "refresh"}. -- Include the access token in requests: Authorization: Bearer . -- Refresh short-lived access tokens at POST /api/token/refresh/ with {"refresh"}. -- Bootstrap an admin once (e.g., createsuperuser at deploy time) to manage users. Do not use DJANGO_SECRET_KEY as a token; it remains server-only for signing. - -Minimal example -- Get a token: - - curl -s -X POST "$BASE_URL/api/token/" -H 'Content-Type: application/json' -d '{"username":"","password":"

"}' -- Call API with token: - - curl -H "Authorization: Bearer " "$BASE_URL/api/users/" - -## How it works in CI/CD -- Workflow: .github/workflows/unit_tests_on_docker.yml - - Exposes TEST_USERNAME and TEST_PASSWORD to the app container. - - docker-compose ensures a matching Django user exists at startup (superuser by default for tests). - - Tests obtain a JWT by calling /api/token/ and pass it with AuthenticatedClient; health probe remains unauthenticated. - -## How to run manually (local) -1) Configure .env (repo root). At minimum: - - POSTGRES_* vars, METADATA_HOST/PORT (provided), and optionally: - - TEST_USERNAME=your_user - - TEST_PASSWORD=your_password -2) Start stack: - - docker compose up --build - - The app migrates and creates/updates the test user if TEST_* vars are set. -3) Get a JWT: - - curl -s -X POST "http://localhost:8000/api/token/" -H 'Content-Type: application/json' -d '{"username":"your_user","password":"your_password"}' -4) Call protected endpoints with Authorization: Bearer . -5) Public healthcheck: - - curl http://localhost:8000/api/health/ - -## Using the generated Python client -- Install the latest client you built/published, then: - - from metadata_client import AuthenticatedClient - - client = AuthenticatedClient(base_url=BASE_URL, token=ACCESS_TOKEN, prefix="Bearer") -- See the generated client docs or source for operation calls (users_create, users_list, etc.). - -Notes -- All endpoints except /api/health/ require a valid JWT. -- Token/signing configuration is managed by SimpleJWT; adjust lifetimes/keys via settings if needed (see docs above). +# Logging in the server +- Uses Django’s logging framework. By default, logs go to stdout/stderr (captured by Docker and CI logs). +- You can tune the LOGGING dict in settings.py for JSON output, correlation IDs, and routing to a centralized sink via container stdout. +- Defaults provide level‑tagged lines suitable for `docker compose logs` or `docker logs `. + +# Requirements files and why they differ +Different contexts need different dependency sets. This repository intentionally separates them: + +- requirements.txt (server runtime): dependencies to run the Django app in production or a container. Typical libs: Django, DRF, JWT, PostgreSQL driver, drf‑spectacular. + +- dp-client/requirements.txt (dp‑client runtime): dependencies needed to use dp-client as a standalone package, including its DB driver(s) for component tests (e.g., psycopg2-binary). Keeps client usage decoupled from server dependencies. + +- requirements-build-client.txt (client build toolchain): tools required to generate and build the OpenAPI client(s). For example: + - openapi-python-client — generates the metadata_client from the OpenAPI schema. + - poetry — builds the Python packages (e.g., dp-client). + - typer and click — CLI dependencies some tooling relies on. + Use case: CI jobs or local scripts that generate and build the client packages without installing full dev tooling. + +- requirements-dev.txt (development toolchain): editor/test/quality tools for contributors: + - Linters/formatters: black, flake8, isort, yamllint. + - Typing: mypy, django-stubs, djangorestframework-stubs. + - Testing: pytest, pytest-django. + - Release tooling: python-semantic-release (future use when versioning is introduced). + - Utilities: dotenv for loading `.env` in tests and scripts. + Use case: local development and CI quality gates. + +This separation ensures minimal, context‑appropriate installs. For example, a developer running only the server doesn’t need OpenAPI generators; a test runner using only dp-client doesn’t need the entire server stack. + +# Local development and examples +## .env example +``` +DJANGO_SECRET_KEY=change-me +DJANGO_DEBUG=True +DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1 +POSTGRES_DB=postgres +POSTGRES_USER=postgres +POSTGRES_PASSWORD=postgres +POSTGRES_HOST=db +POSTGRES_PORT=5432 +METADATA_PORT=8000 +METADATA_HOST=localhost +TEST_USERNAME=test_username +TEST_PASSWORD=1qazxsw@ +``` + +## Install requirements +You can install everything needed for this repository in one go: + +``` +./scripts/install_all_requirements.sh +``` + +Tip: run inside a virtual environment first: +``` +python -m venv .venv && source .venv/bin/activate +``` + +Or pick the set that matches your task: +- Server runtime: + pip install -r requirements.txt +- dp‑client runtime: + pip install -r dp-client/requirements.txt +- Client build toolchain (for generating/building clients): + pip install -r requirements-build-client.txt +- Full development toolchain (linting, typing, testing, etc.): + pip install -r requirements-dev.txt + +## Run locally with docker‑compose (recommended) +``` +docker compose up --build +# Server becomes available at: +# http://localhost:8000/api/health/ +``` + +Healthcheck example: +``` +curl http://localhost:8000/api/health/ +``` + +## Client builds and local install +Build and install both clients locally when developing: +``` +# Build and install the generated OpenAPI client (metadata_client) +./scripts/build_and_install_open_api_client.sh + +# Build and install dp-client (will build metadata_client first; installs without fetching deps) +./scripts/build_and_install_dp_client.sh +``` + +## Tests +- Unit tests (use metadata_client, typically with compose up): + pytest tests/unit -s -v +- Component tests (use dp-client and verify DB persistence): + # Optionally enable fallback if your .env uses POSTGRES_HOST=db and you're outside compose + export POSTGRES_ALLOW_LOCAL_FALLBACK=true + + ./scripts/build_and_install_dp_client.sh + pytest tests/component -s -v + +# OpenAPI and clients +- We validate that the OpenAPI schema is up‑to‑date and that the generated client is not drifting using repository scripts (see scripts/): + - scripts/validate_open_api_file_up_to_date.sh + - scripts/verify_open_api_client_is_up_to_date.sh +- metadata_client mirrors the API one‑to‑one and is ideal for unit tests and simple integrations. +- dp-client composes: + - a factory for constructing authenticated/unauthenticated metadata_client instances + - separated API wrappers (e.g., UsersAPI, HealthAPI) + - a DB helper (PGDBClient) with driver‑based design (Postgres provided), fully decoupled from Django ORM. + +# CI/CD overview +Entry point: CI‑CD Pipelines interface — https://github.com/sidkos/metadata_server/actions/workflows/ci_cd.yml + +Pipeline summary: +- Quality gates: + - Lint/format + - mypy with Django plugin and stubs configuration + - OpenAPI validation and client drift checks + - Unit tests (reusable workflow analogous to unit_tests_on_docker.yml) +- Build server image: + - Builds and pushes ghcr.io//metadata_server:latest. +- Component tests (reusable component_tests.yml): + - Spins up a GitHub Actions PostgreSQL service. + - Pulls the server image built in the previous step and runs it. + - Waits for `/api/health/` readiness. + - Builds/installs dp-client on the runner and executes pytest against tests/component. + +# Notes on release strategy +- Current workflows build and use the latest images and locally built clients; version pinning and artifact publishing are not yet implemented. +- Recommended next steps: + - Introduce semantic versioning for all deliverables. + - Publish metadata_client and dp-client to a registry and pin versions in tests and downstream projects. + - Keep dp-client in a separate repository with its own SDLC, releases, and changelog. diff --git a/dp-client/README.md b/dp-client/README.md index ab02d23..4a67b36 100644 --- a/dp-client/README.md +++ b/dp-client/README.md @@ -1,72 +1,125 @@ # dp-client -A unified, test-friendly client to interact with the platform during component/integration tests. +A higher-level, test-friendly Python client for the Metadata Server, designed for component/end‑to‑end tests. It provides a structured API façade over the server’s endpoints and optional DB checks via driver-based adapters. -- Lives in its own folder (dp-client) -- Built and versioned similarly to metadata-client -- Depends on the generated `metadata-client` -- Has its own requirements file (requirements.txt) +- Lives in its own folder (`dp-client/`) +- Manages its own runtime requirements (`dp-client/requirements.txt`) +- Built via Poetry; helper scripts provided -## Installation (development) +## When to use dp-client +- Use `dp-client` for component tests and higher-level flows (create → verify via API and DB). It provides a stable façade and hides low-level API details. -Option A: Install from requirements +## Architecture +`DPClient` composes distinct parts with strict separation of concerns: +- MetaDataServerAPIClient — factory‑built HTTP client for the Metadata Server (authenticated or not). +- HealthAPI — methods for `/api/health/`. +- UsersApi — methods for `/api/users/` (create/get/list), aligned with unit tests usage. +- PGDBClient — optional Postgres DB helper used in tests to verify persistence and cleanup. It is decoupled from Django ORM and uses driver(s) under `dp_client.db.drivers`. + +This design keeps HTTP API usage and DB verification separate, while allowing simple orchestration from tests. + +## Installation +Option A: install runtime requirements for dp-client only ```bash pip install -r dp-client/requirements.txt ``` -Option B: Build with Poetry and install the wheel +Option B: build and install the package (recommended for local dev; also used in CI) ```bash -./scripts/build_dp_client.sh -pip install ./dp-client/dist/*.whl +# From repo root +./scripts/build_and_install_dp_client.sh ``` -## Usage +Notes: +- The installer resolves and installs prerequisites automatically, then builds and installs `dp-client` using `pip install --no-deps` to avoid fetching from remote indexes. +- Versioning across deliverables is not implemented yet; current flows use latest/on‑the‑fly builds. Introducing semantic versioning is recommended. + +## Configuration +dp-client reads configuration from arguments and environment variables. + +API base URL and auth: +- Base URL is passed to `DPClient(base_url=...)`. +- For authenticated usage, pass `token` (a JWT from `POST /api/token/`) and optionally `prefix="Bearer"` if not default. + +Database connection (for PGDBClient): +- POSTGRES_HOST +- POSTGRES_PORT +- POSTGRES_DB +- POSTGRES_USER +- POSTGRES_PASSWORD +- POSTGRES_ALLOW_LOCAL_FALLBACK (optional, truthy values enable fallback to `localhost` when the host, e.g. `db`, is not resolvable outside Docker networks.) + +## Usage examples +Basic (no auth required for health): ```python from dp_client import DPClient client = DPClient(base_url="http://localhost:8000") +res = client.HealthAPI.health_check() +print(res.status_code, res.parsed) +``` + +Authenticated usage (JWT): + +```python +import os +from dp_client import DPClient -# Backward-compatible helpers: +base_url = os.environ.get("API_BASE_URL", "http://localhost:8000") +access_token = os.environ["ACCESS_TOKEN"] # obtain via POST /api/token/ + +client = DPClient(base_url=base_url, token=access_token) + +# Create a user via API +payload = {"id": "123456782", "name": "Alice", "phone": "+972501234567", "address": "Street 1"} +create_resp = client.UsersApi.create_user(payload) +assert create_resp.status_code == 201 +user_id = create_resp.parsed.id + +# Verify via API +get_resp = client.UsersApi.get_user(user_id) +assert get_resp.status_code == 200 + +# Optional: verify in DB and cleanup +if client.PGDBClient is not None: + db_user = client.PGDBClient.get_user_by_id(user_id) + assert db_user and db_user["id"] == user_id + client.PGDBClient.delete_users_by_ids([user_id]) +``` + +Backwards‑compatible helpers (delegating to structured APIs): + +```python client.health_check() client.create_user({"id": "...", "name": "...", "phone": "+972...", "address": "..."}) client.get_user("...") client.list_users() +``` -# Structured access: -# Underlying generated client (metadata-client): -client.MetaDataServerAPIClient - -# API domains: -client.HealthAPI.health_check() -client.UsersApi.create_user({...}) +## Using dp-client in component tests +- Component tests in this repo import `DPClient` and use it to call the server and to check the DB (when available). +- If your `.env` uses `POSTGRES_HOST=db` (Docker service name) and you run tests outside Docker, enable safe fallback: -# Optional DB helper (uses Django ORM if available in the environment): -client.PGDBClient.get_user_by_id("...") -client.PGDBClient.delete_users_by_ids(["...", "..."]) +```bash +export POSTGRES_ALLOW_LOCAL_FALLBACK=true ``` +- When the database is not reachable, DB‑dependent assertions in tests may be skipped with an explanatory message. -## Running DB-backed component tests locally +## Requirements +Runtime dependencies for dp-client are declared in: -You have two options to make DB checks work outside Docker: +- `dp-client/requirements.txt` (includes the DB driver for Postgres) -1) Use docker-compose (recommended): - - Ensure the Postgres service is up and reachable as host name `db` within the network. - - From repo root: `docker-compose up -d` - - Run tests; the `.env` sets POSTGRES_HOST=db and tests will connect via that. +Install them directly, or rely on `./scripts/build_and_install_dp_client.sh` which builds and installs compatible wheels locally. -2) Use a local Postgres instance: - - Export env vars to point at your local instance, for example: - - `export POSTGRES_HOST=localhost` - - `export POSTGRES_PORT=5432` - - `export POSTGRES_DB=postgres` - - `export POSTGRES_USER=postgres` - - `export POSTGRES_PASSWORD=postgres` - - Or, if your `.env` uses POSTGRES_HOST=db and you don’t want to change it, you can opt-in to a safe fallback: - - `export POSTGRES_ALLOW_LOCAL_FALLBACK=true` - - When the host `db` is not resolvable, dp-client will transparently fall back to `localhost`. +## Scripts +- `scripts/build_dp_client.sh` — builds the dp-client wheel with Poetry and syncs its version to the repository when applicable. +- `scripts/build_and_install_dp_client.sh` — builds and installs `dp-client` (using `--no-deps`), prints diagnostics, and verifies import. -If the DB is not reachable, component tests that require direct DB access will be skipped with a helpful message. +## Notes on repository layout and releases +- While dp-client is colocated here for convenience, best practice is to host it in a separate repository with its own SDLC and publish it to a package registry. Downstream projects can then depend on versioned releases. +- For now, CI and local development build `dp-client` on the fly and use the latest artifacts. diff --git a/scripts/install_all_requirements.sh b/scripts/install_all_requirements.sh new file mode 100755 index 0000000..3859a3e --- /dev/null +++ b/scripts/install_all_requirements.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Install all requirement sets used in this repository in one go. +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" + +PY="${PYTHON_BINARY:-$(command -v python || command -v python3)}" +PIP="$PY -m pip" + +if [ -z "$PY" ]; then + echo "[ERROR] Python interpreter not found in PATH" >&2 + exit 1 +fi + +echo "[env] Python: $($PY --version 2>&1)" +echo "[env] Pip: $($PIP --version 2>&1)" + +# Optional: keep pip modern to reduce resolver issues +if [ "${UPGRADE_PIP:-1}" = "1" ]; then + echo "[step] Upgrading pip (can be disabled with UPGRADE_PIP=0)" + $PIP install --upgrade pip >/dev/null +fi + +install_requirements() { + local file="$1" + if [ -f "$file" ]; then + echo "[install] $file" + $PIP install -r "$file" + else + echo "[skip] $file (not found)" + fi +} + +install_requirements "$ROOT/requirements-build-client.txt" + +install_requirements "$ROOT/requirements-dev.txt" + +install_requirements "$ROOT/requirements.txt" + +install_requirements "$ROOT/dp-client/requirements.txt" + +echo "[done] All requirement sets processed." \ No newline at end of file diff --git a/scripts/start_server_in_container.sh b/scripts/start_server_in_container.sh old mode 100644 new mode 100755