ai-exchange/tests/unit/test_documentation_consistency.py

import re
import unittest
from pathlib import Path


ROOT = Path(__file__).resolve().parents[2]
TRAINING_PRODUCT_DOC = ROOT / "docs" / "training_case_engine_m1_m6_product_readme.md"
API_SERVER = ROOT / "src" / "api" / "server.py"


def _training_api_routes_from_product_doc() -> tuple[tuple[str, str], ...]:
    text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
    routes = []
    for method, path in re.findall(r"\|\s*(GET|POST)\s*\|\s*`([^`]+)`\s*\|", text):
        if path.startswith("/training/"):
            routes.append((method, path))
    return tuple(routes)


def _supported_training_routes_from_route_request() -> tuple[tuple[str, str], ...]:
    source = _route_request_source()
    routes = list(
        re.findall(
            r'if method == "(GET|POST)" and path == "(/training/[^"]+)":',
            source,
        )
    )

    for pattern, route in (
        (
            r'if method == "GET" and len\(path_parts\) == 3 and path_parts\[:2\] == \["training", "cases"\]:\s+case_id = _parse_path_int\(path_parts\[2\], "case_id"\)',
            ("GET", "/training/cases/{case_id}"),
        ),
        (
            r'if method == "POST" and len\(path_parts\) == 4 and path_parts\[:2\] == \["training", "attempts"\] and path_parts\[3\] == "annotations":\s+attempt_id = _parse_path_int\(path_parts\[2\], "attempt_id"\)',
            ("POST", "/training/attempts/{attempt_id}/annotations"),
        ),
        (
            r'if method == "POST" and len\(path_parts\) == 4 and path_parts\[:2\] == \["training", "attempts"\] and path_parts\[3\] == "submit":\s+attempt_id = _parse_path_int\(path_parts\[2\], "attempt_id"\)',
            ("POST", "/training/attempts/{attempt_id}/submit"),
        ),
    ):
        if re.search(pattern, source):
            routes.append(route)
    return tuple(routes)


def _route_request_source() -> str:
    source = API_SERVER.read_text(encoding="utf-8")
    match = re.search(r"^def route_request\(.*?(?=^class APIRequestHandler)", source, re.DOTALL | re.MULTILINE)
    if match is None:
        raise AssertionError("src/api/server.py is missing route_request")
    return match.group(0)


def _training_validation_section() -> str:
    text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
    match = re.search(r"## Validation And Errors\n(?P<section>.*?)(?:\n## |\Z)", text, re.DOTALL)
    if match is None:
        raise AssertionError("Training product README is missing the Validation And Errors section")
    return match.group("section")


class DocumentationConsistencyTests(unittest.TestCase):
    def test_readme_lists_script_output_json_environment_variables(self) -> None:
        script_env_vars = set()
        for path in (ROOT / "scripts").glob("*.py"):
            script_env_vars.update(re.findall(r'"([A-Z0-9_]+OUTPUT_JSON)"', path.read_text(encoding="utf-8")))
        readme = (ROOT / "README.md").read_text(encoding="utf-8")

        missing = sorted(name for name in script_env_vars if name not in readme)

        self.assertEqual(missing, [])

    def test_training_case_product_doc_lists_public_m1_m6_surface(self) -> None:
        text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")

        for snippet in (
            "Training panel",
            "/training/cases",
            "/training/cases/{case_id}",
            "/training/cases/export",
            "/training/cases/import",
            "/training/attempts",
            "/training/attempts/{attempt_id}/annotations",
            "/training/attempts/{attempt_id}/submit",
            "/training/progress",
            ".venv/bin/python scripts/run_training_case_package.py",
            ".venv/bin/python scripts/run_offline_checks.py",
            "internal solution key",
            "Data Isolation Guarantees",
        ):
            with self.subTest(snippet=snippet):
                self.assertIn(snippet, text)

    def test_training_case_product_doc_api_routes_match_server_route_request(self) -> None:
        documented_routes = _training_api_routes_from_product_doc()
        server_routes = _supported_training_routes_from_route_request()

        self.assertEqual(
            len(documented_routes),
            len(set(documented_routes)),
            "Training API docs contain duplicate route entries",
        )
        self.assertEqual(
            sorted(documented_routes),
            sorted(server_routes),
        )

    def test_training_case_product_doc_names_public_error_semantics(self) -> None:
        section = _training_validation_section()

        for snippet in (
            "`400` with `invalid_json_body`",
            "`400` with `invalid_path_param:<field>`",
            "`400` with `invalid_limit_query_param:limit`",
            "`404` with `not_found`",
            "`500` with `internal_server_error`",
            "do not echo raw internals",
        ):
            with self.subTest(snippet=snippet):
                self.assertIn(snippet, section)

        self.assertNotIn("answer" + "_key", section)

    def test_training_case_product_doc_avoids_private_literal_shapes(self) -> None:
        text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8").lower()

        for term in (
            "data" + "base",
            "data" + "base_url",
            "post" + "gresql",
            "psy" + "copg",
            "pass" + "word",
            "pass" + "wd",
            "tok" + "en",
            "sec" + "ret",
            "api" + "_key",
            "answer" + "_key",
        ):
            with self.subTest(term=term):
                self.assertNotIn(term, text)

        self.assertIsNone(re.search(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", text))


if __name__ == "__main__":
    unittest.main()