F-r-ldraledigplannerare/backend/tests/integration/test_schema_parity.py

"""Mechanism test: Alembic migrations vs. `_FULL_SCHEMA` must describe one schema.

Production boots via `run_migrations()` (the real Alembic chain in
`backend/migrations/versions/`). All 518+ existing tests instead get their schema
from `ensure_schema()`, which applies `_FULL_SCHEMA` — a hand-maintained DDL string
in `app/db/connection.py` — directly, bypassing Alembic entirely (see
`tests/conftest.py::connection`).

Nothing has ever compared the two. They have drifted before (see
`PLANNING_ENGINE_SPEC.md` §10.5: "Do not forget `_FULL_SCHEMA`" is called out
precisely because someone forgot it once already). When they drift, every test
in the suite keeps passing against a schema production does not have, and the
gap is only discovered when production breaks.

This test builds one SQLite database via each path and asserts the resulting
schemas are structurally equivalent: same tables, same columns (name, type,
nullability, default, primary-key membership), same CHECK constraints, same
foreign keys, same indexes/uniqueness. Column *order* and cosmetic DDL
formatting are deliberately ignored — `_FULL_SCHEMA` writes each table's final
column layout by hand, while the migration chain appends columns via `ALTER
TABLE ... ADD COLUMN`, so the two legitimately end up in different column
orders for the same set of columns. That is not a bug and must not fail this
test.
"""

from __future__ import annotations

import asyncio
import re
import sqlite3
from pathlib import Path

import aiosqlite
import pytest

from app.db.connection import ensure_schema, run_migrations

# Bookkeeping tables that exist only because of *how* a schema was applied,
# never part of the application's own data model.
_IGNORED_TABLES = {"sqlite_sequence", "alembic_version"}


def _is_app_table(name: str) -> bool:
    return name not in _IGNORED_TABLES and not name.startswith("sqlite_")


def _table_names(cur: sqlite3.Cursor) -> set[str]:
    rows = cur.execute("SELECT name FROM sqlite_master WHERE type = 'table'").fetchall()
    return {row[0] for row in rows if _is_app_table(row[0])}


def _normalize_type(sql_type: str | None) -> str:
    return (sql_type or "").strip().upper()


def _normalize_default(default: object) -> str | None:
    if default is None:
        return None
    return str(default).strip()


def _columns(cur: sqlite3.Cursor, table: str) -> dict[str, tuple[str, bool, str | None, int]]:
    """name -> (normalized type, notnull, normalized default, pk ordinal)."""
    rows = cur.execute(f'PRAGMA table_info("{table}")').fetchall()
    # row: cid, name, type, notnull, dflt_value, pk
    return {
        row[1]: (_normalize_type(row[2]), bool(row[3]), _normalize_default(row[4]), row[5])
        for row in rows
    }


def _foreign_keys(cur: sqlite3.Cursor, table: str) -> set[tuple[str, str, str, str, str]]:
    rows = cur.execute(f'PRAGMA foreign_key_list("{table}")').fetchall()
    # row: id, seq, table, from, to, on_update, on_delete, match
    return {(row[2], row[3], row[4] or "", row[5], row[6]) for row in rows}


def _indexes(cur: sqlite3.Cursor, table: str) -> set[tuple[bool, tuple[str, ...]]]:
    """(is_unique, ordered column names) per index — index *names* are not compared,
    they are SQLite-generated (`sqlite_autoindex_...`) and carry no meaning of their own.
    """
    index_rows = cur.execute(f'PRAGMA index_list("{table}")').fetchall()
    # row: seq, name, unique, origin, partial
    result: set[tuple[bool, tuple[str, ...]]] = set()
    for row in index_rows:
        index_name, is_unique = row[1], bool(row[2])
        info_rows = cur.execute(f'PRAGMA index_info("{index_name}")').fetchall()
        # row: seqno, cid, name — order by seqno for a stable column tuple.
        cols = tuple(c[2] for c in sorted(info_rows, key=lambda c: c[0]))
        result.add((is_unique, cols))
    return result


def _extract_checks(create_table_sql: str) -> list[str]:
    """Pull out each balanced `CHECK ( ... )` clause's inner text, raw."""
    checks: list[str] = []
    upper = create_table_sql.upper()
    pos = 0
    while True:
        found = upper.find("CHECK", pos)
        if found == -1:
            break
        paren_start = create_table_sql.find("(", found)
        if paren_start == -1:
            break
        depth = 0
        end = paren_start
        for end in range(paren_start, len(create_table_sql)):
            char = create_table_sql[end]
            if char == "(":
                depth += 1
            elif char == ")":
                depth -= 1
                if depth == 0:
                    break
        checks.append(create_table_sql[paren_start + 1 : end])
        pos = end + 1
    return checks


def _normalize_check(check_text: str) -> str:
    return re.sub(r"\s+", " ", check_text).strip()


def _checks(cur: sqlite3.Cursor, table: str) -> frozenset[str]:
    row = cur.execute(
        "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", (table,)
    ).fetchone()
    sql = row[0] if row and row[0] else ""
    return frozenset(_normalize_check(c) for c in _extract_checks(sql))


def _diff_schemas(migrated: sqlite3.Connection, full_schema: sqlite3.Connection) -> list[str]:
    """Return a list of human-readable mismatch descriptions; empty means equivalent."""
    diffs: list[str] = []
    migrated_cur = migrated.cursor()
    full_schema_cur = full_schema.cursor()

    migrated_tables = _table_names(migrated_cur)
    full_schema_tables = _table_names(full_schema_cur)

    only_in_migrations = migrated_tables - full_schema_tables
    only_in_full_schema = full_schema_tables - migrated_tables
    if only_in_migrations:
        diffs.append(
            f"tables only produced by run_migrations() (missing from _FULL_SCHEMA): "
            f"{sorted(only_in_migrations)}"
        )
    if only_in_full_schema:
        diffs.append(
            f"tables only produced by _FULL_SCHEMA (missing from the migration chain): "
            f"{sorted(only_in_full_schema)}"
        )

    for table in sorted(migrated_tables & full_schema_tables):
        migrated_cols = _columns(migrated_cur, table)
        full_schema_cols = _columns(full_schema_cur, table)

        missing_in_full_schema = migrated_cols.keys() - full_schema_cols.keys()
        missing_in_migrations = full_schema_cols.keys() - migrated_cols.keys()
        if missing_in_full_schema:
            diffs.append(
                f"table '{table}': column(s) {sorted(missing_in_full_schema)} exist after "
                f"run_migrations() but are missing from _FULL_SCHEMA"
            )
        if missing_in_migrations:
            diffs.append(
                f"table '{table}': column(s) {sorted(missing_in_migrations)} exist in "
                f"_FULL_SCHEMA but the migration chain never creates them"
            )

        for col in sorted(migrated_cols.keys() & full_schema_cols.keys()):
            m_type, m_notnull, m_default, m_pk = migrated_cols[col]
            f_type, f_notnull, f_default, f_pk = full_schema_cols[col]
            if (m_type, m_notnull, m_default, m_pk) != (f_type, f_notnull, f_default, f_pk):
                diffs.append(
                    f"table '{table}' column '{col}': migrations gives "
                    f"(type={m_type!r}, notnull={m_notnull}, default={m_default!r}, pk={m_pk}) "
                    f"but _FULL_SCHEMA gives "
                    f"(type={f_type!r}, notnull={f_notnull}, default={f_default!r}, pk={f_pk})"
                )

        m_checks = _checks(migrated_cur, table)
        f_checks = _checks(full_schema_cur, table)
        if m_checks != f_checks:
            diffs.append(
                f"table '{table}': CHECK constraints differ — "
                f"migrations only: {sorted(m_checks - f_checks)}; "
                f"_FULL_SCHEMA only: {sorted(f_checks - m_checks)}"
            )

        m_fks = _foreign_keys(migrated_cur, table)
        f_fks = _foreign_keys(full_schema_cur, table)
        if m_fks != f_fks:
            diffs.append(
                f"table '{table}': foreign keys differ (ref_table, from_col, to_col, "
                f"on_update, on_delete) — "
                f"migrations only: {sorted(m_fks - f_fks)}; "
                f"_FULL_SCHEMA only: {sorted(f_fks - m_fks)}"
            )

        m_idx = _indexes(migrated_cur, table)
        f_idx = _indexes(full_schema_cur, table)
        if m_idx != f_idx:
            diffs.append(
                f"table '{table}': indexes differ (is_unique, columns) — "
                f"migrations only: {sorted(m_idx - f_idx)}; "
                f"_FULL_SCHEMA only: {sorted(f_idx - m_idx)}"
            )

    return diffs


async def _apply_full_schema(db_path: Path) -> None:
    conn = await aiosqlite.connect(db_path)
    try:
        await ensure_schema(conn)
    finally:
        await conn.close()


def test_alembic_migrations_and_full_schema_agree(tmp_path: Path) -> None:
    """`run_migrations()` (production) and `ensure_schema()` (tests) must build the
    same schema. A failure here means the 518+ tests using `_FULL_SCHEMA` are
    exercising a schema production does not actually have — see module docstring.
    """
    migrated_db = tmp_path / "via_migrations.db"
    full_schema_db = tmp_path / "via_full_schema.db"

    # Alembic chain, exactly as production's app lifespan runs it.
    run_migrations(f"sqlite:///{migrated_db}")

    # `ensure_schema()` is async (an aiosqlite connection); `run_migrations()` above
    # must run to completion first since it drives its own `asyncio.run()` internally
    # and cannot be called from within a running event loop.
    asyncio.run(_apply_full_schema(full_schema_db))

    migrated_conn = sqlite3.connect(migrated_db)
    full_schema_conn = sqlite3.connect(full_schema_db)
    try:
        diffs = _diff_schemas(migrated_conn, full_schema_conn)
    finally:
        migrated_conn.close()
        full_schema_conn.close()

    if diffs:
        pytest.fail(
            "Schema drift between Alembic migrations (production) and _FULL_SCHEMA "
            "(tests) in backend/app/db/connection.py:\n- " + "\n- ".join(diffs),
            pytrace=False,
        )