917 lines
37 KiB
Python
917 lines
37 KiB
Python
import ast
|
|
import re
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PROJECT_README = ROOT / "README.md"
|
|
TRAINING_PRODUCT_DOC = ROOT / "docs" / "training_case_engine_m1_m6_product_readme.md"
|
|
TRAINING_PACKAGE_SMOKE_DOC = ROOT / "docs" / "examples" / "training_case_package_smoke_m6.md"
|
|
TRAINING_DOCS_CLI_TEST = ROOT / "tests" / "unit" / "test_training_case_docs_cli_m10.py"
|
|
TRAINING_DASHBOARD_HTML = ROOT / "src" / "web" / "index.html"
|
|
TRAINING_DASHBOARD_APP = ROOT / "src" / "web" / "app.js"
|
|
API_SERVER = ROOT / "src" / "api" / "server.py"
|
|
OFFLINE_CHECKS_SCRIPT = ROOT / "scripts" / "run_offline_checks.py"
|
|
CURRENT_TRAINING_DOC_MATURITY_END = 40
|
|
CURRENT_TRAINING_DOC_MATURITY_MARKER = f"M1-M{CURRENT_TRAINING_DOC_MATURITY_END}"
|
|
CURRENT_TRAINING_DOC_MATURITY_SPAN = f"through M{CURRENT_TRAINING_DOC_MATURITY_END}"
|
|
CURRENT_TRAINING_DOC_MATURITY_PHRASE = (
|
|
f"current product maturity {CURRENT_TRAINING_DOC_MATURITY_SPAN}"
|
|
)
|
|
TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY = {
|
|
"m1_m40_maturity_wording": (
|
|
"test_training_case_product_doc_reflects_current_m40_maturity",
|
|
),
|
|
"route_surface": (
|
|
"test_training_case_product_doc_api_routes_match_server_route_request",
|
|
"test_training_case_public_route_registry_is_documented_and_supported",
|
|
"test_training_case_product_doc_api_surface_wording_matches_public_routes",
|
|
"test_training_dashboard_routes_are_documented_public_api_routes",
|
|
),
|
|
"dashboard_public_docs_privacy": (
|
|
"test_training_readmes_name_dashboard_privacy_and_offline_registry_coverage",
|
|
"test_training_dashboard_and_public_docs_keep_privacy_wording_exclusionary",
|
|
),
|
|
"offline_registry_coverage": (
|
|
"test_training_readmes_name_dashboard_privacy_and_offline_registry_coverage",
|
|
),
|
|
}
|
|
TRAINING_M24_M29_PACKAGE_DOCS_INVENTORY_TEST = (
|
|
"test_m24_m29_package_docs_privacy_test_inventory_is_current"
|
|
)
|
|
TRAINING_CASE_ACCEPTANCE_TEST_MANIFEST = (
|
|
"test_training_case_engine_m1",
|
|
"test_training_api_m2",
|
|
"test_training_dashboard_m3",
|
|
"test_training_api_m4",
|
|
"test_training_case_import_export_m5",
|
|
"test_training_public_api_contract_m8",
|
|
"test_training_api_error_contract_m9",
|
|
"test_training_case_docs_cli_m10",
|
|
"test_training_public_export_privacy_m15",
|
|
"test_documentation_consistency",
|
|
)
|
|
TRAINING_CASE_REQUIRED_ACCEPTANCE_TEST_MODULES = (
|
|
"test_training_case_engine_m1",
|
|
"test_training_api_m2",
|
|
"test_training_dashboard_m3",
|
|
"test_training_api_m4",
|
|
"test_training_case_import_export_m5",
|
|
"test_training_public_api_contract_m8",
|
|
"test_training_api_error_contract_m9",
|
|
"test_training_case_docs_cli_m10",
|
|
"test_training_public_export_privacy_m15",
|
|
"test_documentation_consistency",
|
|
)
|
|
TRAINING_PUBLIC_API_ROUTE_REGISTRY = (
|
|
("GET", "/training/cases"),
|
|
("POST", "/training/cases/import"),
|
|
("GET", "/training/cases/export"),
|
|
("GET", "/training/cases/{case_id}"),
|
|
("POST", "/training/attempts"),
|
|
("POST", "/training/attempts/{attempt_id}/annotations"),
|
|
("POST", "/training/attempts/{attempt_id}/submit"),
|
|
("GET", "/training/progress"),
|
|
)
|
|
TRAINING_DASHBOARD_ROUTE_REGISTRY = (
|
|
("GET", "/training/cases"),
|
|
("GET", "/training/progress"),
|
|
("POST", "/training/attempts"),
|
|
("POST", "/training/attempts/{attempt_id}/annotations"),
|
|
("POST", "/training/attempts/{attempt_id}/submit"),
|
|
)
|
|
TRAINING_ROUTE_SURFACE_WORDING = {
|
|
("GET", "/training/cases"): ("list cases", "status", "skill tag", "instrument", "limit filters"),
|
|
("POST", "/training/cases/import"): ("import a versioned public package", "cases plus tasks"),
|
|
("GET", "/training/cases/export"): ("export a public package", "same case filters"),
|
|
("GET", "/training/cases/{case_id}"): ("read one public case",),
|
|
("POST", "/training/attempts"): ("start an attempt", "case"),
|
|
("POST", "/training/attempts/{attempt_id}/annotations"): ("save an annotation payload", "attempt"),
|
|
("POST", "/training/attempts/{attempt_id}/submit"): ("submit an attempt", "optional score"),
|
|
("GET", "/training/progress"): ("list progress", "user", "skill tag", "limit filters"),
|
|
}
|
|
TRAINING_DASHBOARD_PUBLIC_WORDING = (
|
|
"available cases",
|
|
"user progress",
|
|
"attempt start",
|
|
"annotation note save",
|
|
"attempt submit",
|
|
"public case fields only",
|
|
)
|
|
TRAINING_ACCEPTANCE_VALIDATION_DOC_SNIPPETS = (
|
|
"M35 acceptance posture",
|
|
"Training Case acceptance manifest",
|
|
"focused milestone test suites",
|
|
"offline checks",
|
|
"public validation",
|
|
)
|
|
TRAINING_M41_RELEASE_INDEX_DOCS = (PROJECT_README, TRAINING_PRODUCT_DOC)
|
|
TRAINING_M41_RELEASE_INDEX_GUARD_TEST_INVENTORY = (
|
|
"test_training_readmes_expose_m1_m40_release_index_without_raw_test_modules",
|
|
)
|
|
TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY = {
|
|
"current_m1_m40_marker": (
|
|
CURRENT_TRAINING_DOC_MATURITY_MARKER,
|
|
CURRENT_TRAINING_DOC_MATURITY_SPAN,
|
|
CURRENT_TRAINING_DOC_MATURITY_PHRASE,
|
|
),
|
|
"acceptance_offline_validation": TRAINING_ACCEPTANCE_VALIDATION_DOC_SNIPPETS,
|
|
"public_api_privacy_contracts": (
|
|
"API contract coverage",
|
|
"error contract coverage",
|
|
"privacy contract coverage",
|
|
),
|
|
"package_smoke_offline_registry": (
|
|
"package smoke",
|
|
"training_case_package",
|
|
),
|
|
"dashboard_public_docs_privacy": (
|
|
"dashboard_js_syntax",
|
|
"dashboard and public docs privacy smoke coverage",
|
|
),
|
|
"single_set_docker_deployment": (
|
|
"single-set local Docker deployment",
|
|
),
|
|
"docker_smoke_guard_coverage": (
|
|
"Docker smoke guard coverage",
|
|
),
|
|
}
|
|
TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS = tuple(
|
|
snippet
|
|
for snippets in TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY.values()
|
|
for snippet in snippets
|
|
)
|
|
TRAINING_M41_STALE_RELEASE_INVENTORY_NAMES = ("m1_m28_maturity_wording",)
|
|
TRAINING_M41_STALE_RELEASE_WORDING = (
|
|
"M1-M39",
|
|
"through M39",
|
|
"through-M39",
|
|
)
|
|
TRAINING_RAW_TEST_MODULE_PATTERN = r"\btest_training_[a-z0-9_]+"
|
|
TRAINING_PUBLIC_PRIVACY_DOCS = (PROJECT_README, TRAINING_PRODUCT_DOC, TRAINING_PACKAGE_SMOKE_DOC)
|
|
TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS = (
|
|
"answer" + "_key",
|
|
"answer " + "key",
|
|
"private " + "answer",
|
|
"answer " + "detail",
|
|
"solution " + "key",
|
|
"solution_" + "key",
|
|
)
|
|
TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES = (
|
|
"private solution material",
|
|
"internal solution details",
|
|
)
|
|
LOCAL_DOCKER_POLICY_DOCS = (PROJECT_README, TRAINING_PRODUCT_DOC)
|
|
LOCAL_DOCKER_DOCUMENTATION_GUARD_TEST_INVENTORY = (
|
|
"test_readmes_describe_single_set_local_docker_deployment_policy",
|
|
"test_readmes_lock_single_set_docker_smoke_command_shape",
|
|
)
|
|
LOCAL_DOCKER_DOCUMENTATION_GUARD_SOURCE_CONSTANTS = (
|
|
"LOCAL_DOCKER_POLICY_DOCS",
|
|
"LOCAL_DOCKER_REBUILD_COMMANDS",
|
|
"LOCAL_DOCKER_SMOKE_COMMANDS",
|
|
"LOCAL_DOCKER_COMPOSE_PREFIX",
|
|
)
|
|
LOCAL_DOCKER_DEPLOY_DIR = "/root/deploy/ai-ict"
|
|
LOCAL_DOCKER_COMPOSE_PROJECT = "ai-ict"
|
|
LOCAL_DOCKER_CONTAINER = "ai-ict"
|
|
LOCAL_DOCKER_COMPOSE_FILE = f"{LOCAL_DOCKER_DEPLOY_DIR}/docker-compose.yml"
|
|
LOCAL_DOCKER_COMPOSE_PREFIX = (
|
|
f"docker compose -p {LOCAL_DOCKER_COMPOSE_PROJECT} -f {LOCAL_DOCKER_COMPOSE_FILE}"
|
|
)
|
|
LOCAL_DOCKER_REBUILD_COMMANDS = (
|
|
f"{LOCAL_DOCKER_COMPOSE_PREFIX} down --remove-orphans",
|
|
f"{LOCAL_DOCKER_COMPOSE_PREFIX} up -d --build",
|
|
)
|
|
LOCAL_DOCKER_SMOKE_COMMANDS = (
|
|
f"{LOCAL_DOCKER_COMPOSE_PREFIX} ps",
|
|
"test \"$(docker ps --filter label=com.docker.compose.project=ai-ict --format '{{.Names}}' | wc -l | tr -d ' ')\" = \"1\"",
|
|
"test \"$(docker ps --filter name=^/ai-ict$ --format '{{.Names}}' | wc -l | tr -d ' ')\" = \"1\"",
|
|
"test \"$(docker inspect -f '{{.State.Health.Status}}' ai-ict)\" = \"healthy\"",
|
|
)
|
|
LOCAL_DOCKER_POLICY_REQUIRED_SNIPPETS = (
|
|
"single-set local Docker deployment",
|
|
"compose project `ai-ict`",
|
|
"container `ai-ict`",
|
|
"deploy directory `/root/deploy/ai-ict`",
|
|
"exactly one active deployment set",
|
|
"one active `ai-ict` container",
|
|
"do not create a second local stack or a second `ai-ict` container",
|
|
*LOCAL_DOCKER_REBUILD_COMMANDS,
|
|
*LOCAL_DOCKER_SMOKE_COMMANDS,
|
|
)
|
|
LOCAL_DOCKER_POLICY_FORBIDDEN_ENCOURAGEMENT = (
|
|
"create another local stack",
|
|
"create a second active deployment set",
|
|
"run parallel local stacks",
|
|
"keep multiple active deployment sets",
|
|
"start an extra ai-ict container",
|
|
"duplicate ai-ict containers",
|
|
)
|
|
TRAINING_PUBLIC_PRIVACY_EXCLUSION_WORDS = (
|
|
"scrub",
|
|
"omit",
|
|
"reject",
|
|
"keep",
|
|
"out of",
|
|
"do not",
|
|
"does not",
|
|
"without",
|
|
"hides",
|
|
"not return",
|
|
)
|
|
TRAINING_DASHBOARD_FETCH_PATTERNS = (
|
|
(("GET", "/training/cases"), r"fetchJson\('/training/cases'\)"),
|
|
(("GET", "/training/progress"), r"fetchJson\('/training/progress'\)"),
|
|
(("POST", "/training/attempts"), r"fetchJson\('/training/attempts',\s*\{\s*method:\s*'POST'"),
|
|
(
|
|
("POST", "/training/attempts/{attempt_id}/annotations"),
|
|
r"fetchJson\(`/training/attempts/\$\{[^}]+\}/annotations`,\s*\{\s*method:\s*'POST'",
|
|
),
|
|
(
|
|
("POST", "/training/attempts/{attempt_id}/submit"),
|
|
r"fetchJson\(`/training/attempts/\$\{[^}]+\}/submit`,\s*\{\s*method:\s*'POST'",
|
|
),
|
|
)
|
|
|
|
|
|
def _training_api_surface_rows_from_product_doc() -> tuple[tuple[str, str, str], ...]:
|
|
text = _training_product_doc_section("API Surface")
|
|
rows = []
|
|
for method, path, capability in re.findall(
|
|
r"^\|\s*([A-Z]+)\s*\|\s*`([^`]+)`\s*\|\s*([^|\n]+?)\s*\|",
|
|
text,
|
|
re.MULTILINE,
|
|
):
|
|
if path.startswith("/training/"):
|
|
rows.append((method, path, " ".join(capability.split()).lower()))
|
|
return tuple(rows)
|
|
|
|
|
|
def _training_api_routes_from_product_doc() -> tuple[tuple[str, str], ...]:
|
|
return tuple((method, path) for method, path, _capability in _training_api_surface_rows_from_product_doc())
|
|
|
|
|
|
def _supported_training_routes_from_route_request() -> tuple[tuple[str, str], ...]:
|
|
source = _route_request_source()
|
|
routes = list(
|
|
re.findall(
|
|
r'if method == "(GET|POST)" and path == "(/training/[^"]+)":',
|
|
source,
|
|
)
|
|
)
|
|
|
|
for pattern, route in (
|
|
(
|
|
r'if method == "GET" and len\(path_parts\) == 3 and path_parts\[:2\] == \["training", "cases"\]:\s+case_id = _parse_path_int\(path_parts\[2\], "case_id"\)',
|
|
("GET", "/training/cases/{case_id}"),
|
|
),
|
|
(
|
|
r'if method == "POST" and len\(path_parts\) == 4 and path_parts\[:2\] == \["training", "attempts"\] and path_parts\[3\] == "annotations":\s+attempt_id = _parse_path_int\(path_parts\[2\], "attempt_id"\)',
|
|
("POST", "/training/attempts/{attempt_id}/annotations"),
|
|
),
|
|
(
|
|
r'if method == "POST" and len\(path_parts\) == 4 and path_parts\[:2\] == \["training", "attempts"\] and path_parts\[3\] == "submit":\s+attempt_id = _parse_path_int\(path_parts\[2\], "attempt_id"\)',
|
|
("POST", "/training/attempts/{attempt_id}/submit"),
|
|
),
|
|
):
|
|
if re.search(pattern, source):
|
|
routes.append(route)
|
|
return tuple(routes)
|
|
|
|
|
|
def _training_dashboard_routes_from_app() -> tuple[tuple[str, str], ...]:
|
|
source = TRAINING_DASHBOARD_APP.read_text(encoding="utf-8")
|
|
routes = []
|
|
for route, pattern in TRAINING_DASHBOARD_FETCH_PATTERNS:
|
|
if re.search(pattern, source, re.DOTALL):
|
|
routes.append(route)
|
|
return tuple(routes)
|
|
|
|
|
|
def _training_dashboard_fetch_paths_from_app() -> tuple[str, ...]:
|
|
source = TRAINING_DASHBOARD_APP.read_text(encoding="utf-8")
|
|
paths = []
|
|
for match in re.finditer(r"fetchJson\(\s*['`](?P<path>/training/[^'`]+)['`]", source):
|
|
path = re.sub(r"\$\{[^}]+\}", "{attempt_id}", match.group("path"))
|
|
paths.append(path.split("?", 1)[0])
|
|
return tuple(paths)
|
|
|
|
|
|
def _route_request_source() -> str:
|
|
source = API_SERVER.read_text(encoding="utf-8")
|
|
match = re.search(r"^def route_request\(.*?(?=^class APIRequestHandler)", source, re.DOTALL | re.MULTILINE)
|
|
if match is None:
|
|
raise AssertionError("src/api/server.py is missing route_request")
|
|
return match.group(0)
|
|
|
|
|
|
def _training_product_doc_section(heading: str) -> str:
|
|
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
|
|
match = re.search(
|
|
rf"^## {re.escape(heading)}\n(?P<section>.*?)(?:\n## |\Z)",
|
|
text,
|
|
re.DOTALL | re.MULTILINE,
|
|
)
|
|
if match is None:
|
|
raise AssertionError(f"Training product README is missing the {heading} section")
|
|
return match.group("section")
|
|
|
|
|
|
def _training_validation_section() -> str:
|
|
return _training_product_doc_section("Validation And Errors")
|
|
|
|
|
|
def _local_docker_deployment_section(path: Path) -> str:
|
|
text = path.read_text(encoding="utf-8")
|
|
match = re.search(
|
|
r"^## Local Docker [Dd]eployment\n(?P<section>.*?)(?:\n## |\Z)",
|
|
text,
|
|
re.DOTALL | re.MULTILINE,
|
|
)
|
|
if match is None:
|
|
raise AssertionError(f"{path} is missing the Local Docker deployment section")
|
|
return match.group("section")
|
|
|
|
|
|
def _local_docker_command_lines(text: str) -> tuple[str, ...]:
|
|
return tuple(
|
|
line.strip()
|
|
for line in text.splitlines()
|
|
if line.strip().startswith(("docker ", "test "))
|
|
)
|
|
|
|
|
|
def _training_public_privacy_lines(text: str) -> tuple[str, ...]:
|
|
lines = []
|
|
for line in text.splitlines():
|
|
lower_line = line.lower()
|
|
if any(phrase in lower_line for phrase in TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES):
|
|
lines.append(lower_line.strip())
|
|
return tuple(lines)
|
|
|
|
|
|
def _test_method_names_from_file(path: Path, class_name: str) -> tuple[str, ...]:
|
|
tree = ast.parse(path.read_text(encoding="utf-8"))
|
|
for node in tree.body:
|
|
if isinstance(node, ast.ClassDef) and node.name == class_name:
|
|
return tuple(
|
|
item.name
|
|
for item in node.body
|
|
if isinstance(item, ast.FunctionDef) and item.name.startswith("test_")
|
|
)
|
|
raise AssertionError(f"{path} is missing {class_name}")
|
|
|
|
|
|
class DocumentationConsistencyTests(unittest.TestCase):
|
|
def test_readme_lists_script_output_json_environment_variables(self) -> None:
|
|
script_env_vars = set()
|
|
for path in (ROOT / "scripts").glob("*.py"):
|
|
script_env_vars.update(re.findall(r'"([A-Z0-9_]+OUTPUT_JSON)"', path.read_text(encoding="utf-8")))
|
|
readme = (ROOT / "README.md").read_text(encoding="utf-8")
|
|
|
|
missing = sorted(name for name in script_env_vars if name not in readme)
|
|
|
|
self.assertEqual(missing, [])
|
|
|
|
def test_training_case_product_doc_lists_public_m1_m21_surface(self) -> None:
|
|
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
|
|
|
|
for snippet in (
|
|
"Training panel",
|
|
"/training/cases",
|
|
"/training/cases/{case_id}",
|
|
"/training/cases/export",
|
|
"/training/cases/import",
|
|
"/training/attempts",
|
|
"/training/attempts/{attempt_id}/annotations",
|
|
"/training/attempts/{attempt_id}/submit",
|
|
"/training/progress",
|
|
".venv/bin/python scripts/run_training_case_package.py",
|
|
".venv/bin/python scripts/run_offline_checks.py",
|
|
"private solution material",
|
|
"Data Isolation Guarantees",
|
|
):
|
|
with self.subTest(snippet=snippet):
|
|
self.assertIn(snippet, text)
|
|
|
|
def test_training_case_product_doc_reflects_current_m40_maturity(self) -> None:
|
|
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
|
|
|
|
for snippet in (
|
|
CURRENT_TRAINING_DOC_MATURITY_MARKER,
|
|
"API contract coverage",
|
|
"error contract coverage",
|
|
"fixture/schema contract",
|
|
"offline checks registry smoke",
|
|
"public response privacy contract coverage",
|
|
"dashboard and public docs privacy smoke coverage",
|
|
):
|
|
with self.subTest(snippet=snippet):
|
|
self.assertIn(snippet, text)
|
|
|
|
for stale_snippet in ("M1-M25", "through M25", *TRAINING_M41_STALE_RELEASE_WORDING):
|
|
with self.subTest(stale_snippet=stale_snippet):
|
|
self.assertNotIn(stale_snippet, text)
|
|
|
|
def test_training_readmes_expose_m1_m40_release_index_without_raw_test_modules(self) -> None:
|
|
for path in TRAINING_M41_RELEASE_INDEX_DOCS:
|
|
text = path.read_text(encoding="utf-8")
|
|
|
|
with self.subTest(path=path.name):
|
|
for snippet in TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS:
|
|
self.assertIn(snippet, text)
|
|
for stale_snippet in TRAINING_M41_STALE_RELEASE_WORDING:
|
|
self.assertNotIn(stale_snippet, text)
|
|
self.assertIsNone(re.search(TRAINING_RAW_TEST_MODULE_PATTERN, text))
|
|
|
|
def test_training_case_m24_m29_documentation_test_inventory_is_current(self) -> None:
|
|
test_names = set(
|
|
_test_method_names_from_file(Path(__file__), "DocumentationConsistencyTests")
|
|
)
|
|
package_docs_test_names = set(
|
|
_test_method_names_from_file(TRAINING_DOCS_CLI_TEST, "TrainingCaseDocsCliM10Tests")
|
|
)
|
|
inventory_names = tuple(
|
|
name
|
|
for names in TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY.values()
|
|
for name in names
|
|
)
|
|
unique_inventory_names = set(inventory_names)
|
|
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
|
|
|
|
self.assertEqual(len(inventory_names), 8)
|
|
self.assertEqual(len(unique_inventory_names), 7)
|
|
self.assertEqual(sorted(TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY), [
|
|
"dashboard_public_docs_privacy",
|
|
"m1_m40_maturity_wording",
|
|
"offline_registry_coverage",
|
|
"route_surface",
|
|
])
|
|
for surface, names in TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY.items():
|
|
with self.subTest(surface=surface):
|
|
self.assertTrue(names)
|
|
for name in names:
|
|
self.assertIn(name, test_names)
|
|
|
|
self.assertIn(
|
|
TRAINING_M24_M29_PACKAGE_DOCS_INVENTORY_TEST,
|
|
package_docs_test_names,
|
|
)
|
|
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_MARKER, text)
|
|
self.assertIn(CURRENT_TRAINING_DOC_MATURITY_SPAN, text)
|
|
|
|
def test_training_m41_release_index_guard_inventory_is_current(self) -> None:
|
|
module_path = Path(__file__)
|
|
module_source = module_path.read_text(encoding="utf-8")
|
|
test_names = set(
|
|
_test_method_names_from_file(module_path, "DocumentationConsistencyTests")
|
|
)
|
|
required_guard_test = (
|
|
"test_training_readmes_expose_m1_m40_release_index_without_raw_test_modules"
|
|
)
|
|
required_release_snippets = (
|
|
CURRENT_TRAINING_DOC_MATURITY_MARKER,
|
|
CURRENT_TRAINING_DOC_MATURITY_SPAN,
|
|
CURRENT_TRAINING_DOC_MATURITY_PHRASE,
|
|
*TRAINING_ACCEPTANCE_VALIDATION_DOC_SNIPPETS,
|
|
"API contract coverage",
|
|
"error contract coverage",
|
|
"privacy contract coverage",
|
|
"package smoke",
|
|
"training_case_package",
|
|
"dashboard_js_syntax",
|
|
"dashboard and public docs privacy smoke coverage",
|
|
"single-set local Docker deployment",
|
|
"Docker smoke guard coverage",
|
|
)
|
|
required_stale_inventory_names = ("m1_m28_maturity_wording",)
|
|
required_stale_wording = ("M1-M39", "through M39", "through-M39")
|
|
expected_release_snippets = tuple(
|
|
snippet
|
|
for snippets in TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY.values()
|
|
for snippet in snippets
|
|
)
|
|
active_release_inventory_values = "\n".join(
|
|
(
|
|
*TRAINING_M24_M29_DOCUMENTATION_TEST_INVENTORY,
|
|
*TRAINING_M41_RELEASE_INDEX_GUARD_TEST_INVENTORY,
|
|
*TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY,
|
|
*TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS,
|
|
CURRENT_TRAINING_DOC_MATURITY_MARKER,
|
|
CURRENT_TRAINING_DOC_MATURITY_SPAN,
|
|
CURRENT_TRAINING_DOC_MATURITY_PHRASE,
|
|
)
|
|
)
|
|
guard_match = re.search(
|
|
rf"^ def {TRAINING_M41_RELEASE_INDEX_GUARD_TEST_INVENTORY[0]}"
|
|
r"\(.*?(?=^ def |\Z)",
|
|
module_source,
|
|
re.DOTALL | re.MULTILINE,
|
|
)
|
|
|
|
self.assertEqual(
|
|
TRAINING_M41_RELEASE_INDEX_DOCS,
|
|
(PROJECT_README, TRAINING_PRODUCT_DOC),
|
|
)
|
|
self.assertEqual(TRAINING_M41_RELEASE_INDEX_GUARD_TEST_INVENTORY, (required_guard_test,))
|
|
self.assertEqual(
|
|
sorted(TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY),
|
|
[
|
|
"acceptance_offline_validation",
|
|
"current_m1_m40_marker",
|
|
"dashboard_public_docs_privacy",
|
|
"docker_smoke_guard_coverage",
|
|
"package_smoke_offline_registry",
|
|
"public_api_privacy_contracts",
|
|
"single_set_docker_deployment",
|
|
],
|
|
)
|
|
self.assertEqual(TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS, expected_release_snippets)
|
|
self.assertEqual(
|
|
TRAINING_M41_STALE_RELEASE_INVENTORY_NAMES,
|
|
required_stale_inventory_names,
|
|
)
|
|
self.assertEqual(TRAINING_M41_STALE_RELEASE_WORDING, required_stale_wording)
|
|
self.assertEqual(CURRENT_TRAINING_DOC_MATURITY_MARKER, "M1-M40")
|
|
self.assertEqual(CURRENT_TRAINING_DOC_MATURITY_SPAN, "through M40")
|
|
self.assertEqual(
|
|
CURRENT_TRAINING_DOC_MATURITY_PHRASE,
|
|
"current product maturity through M40",
|
|
)
|
|
|
|
self.assertIn(required_guard_test, test_names)
|
|
self.assertIsNotNone(guard_match)
|
|
guard_source = guard_match.group(0)
|
|
for constant_name in (
|
|
"TRAINING_M41_RELEASE_INDEX_DOCS",
|
|
"TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS",
|
|
"TRAINING_M41_STALE_RELEASE_WORDING",
|
|
"TRAINING_RAW_TEST_MODULE_PATTERN",
|
|
):
|
|
with self.subTest(guard_reference=constant_name):
|
|
self.assertIn(constant_name, guard_source)
|
|
|
|
for surface, snippets in TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY.items():
|
|
with self.subTest(surface=surface):
|
|
self.assertTrue(snippets)
|
|
|
|
for snippet in required_release_snippets:
|
|
with self.subTest(snippet=snippet):
|
|
self.assertIn(snippet, TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS)
|
|
|
|
for stale_value in (
|
|
*required_stale_inventory_names,
|
|
*required_stale_wording,
|
|
):
|
|
with self.subTest(stale_value=stale_value):
|
|
self.assertNotIn(stale_value, active_release_inventory_values)
|
|
|
|
def test_training_m41_release_index_inventory_shape_is_stable(self) -> None:
|
|
module_tree = ast.parse(Path(__file__).read_text(encoding="utf-8"))
|
|
inventory_assignments = [
|
|
node
|
|
for node in module_tree.body
|
|
if isinstance(node, ast.Assign)
|
|
for target in node.targets
|
|
if (
|
|
isinstance(target, ast.Name)
|
|
and target.id == "TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY"
|
|
)
|
|
]
|
|
expected_group_order = (
|
|
"current_m1_m40_marker",
|
|
"acceptance_offline_validation",
|
|
"public_api_privacy_contracts",
|
|
"package_smoke_offline_registry",
|
|
"dashboard_public_docs_privacy",
|
|
"single_set_docker_deployment",
|
|
"docker_smoke_guard_coverage",
|
|
)
|
|
grouped_inventory = TRAINING_M41_RELEASE_INDEX_REQUIRED_SNIPPET_INVENTORY
|
|
grouped_snippets = tuple(
|
|
snippet
|
|
for snippets in grouped_inventory.values()
|
|
for snippet in snippets
|
|
)
|
|
active_release_inventory = (
|
|
*TRAINING_M41_RELEASE_INDEX_GUARD_TEST_INVENTORY,
|
|
*grouped_inventory,
|
|
*grouped_snippets,
|
|
CURRENT_TRAINING_DOC_MATURITY_MARKER,
|
|
CURRENT_TRAINING_DOC_MATURITY_SPAN,
|
|
)
|
|
|
|
self.assertEqual(len(inventory_assignments), 1)
|
|
inventory_source = inventory_assignments[0].value
|
|
self.assertIsInstance(inventory_source, ast.Dict)
|
|
source_group_order = tuple(ast.literal_eval(key) for key in inventory_source.keys)
|
|
|
|
self.assertEqual(source_group_order, expected_group_order)
|
|
self.assertEqual(len(source_group_order), len(set(source_group_order)))
|
|
self.assertEqual(tuple(grouped_inventory), expected_group_order)
|
|
for group_name, snippets in grouped_inventory.items():
|
|
with self.subTest(group_name=group_name):
|
|
self.assertTrue(snippets)
|
|
|
|
self.assertEqual(TRAINING_M41_RELEASE_INDEX_DOC_SNIPPETS, grouped_snippets)
|
|
self.assertEqual(len(grouped_snippets), len(set(grouped_snippets)))
|
|
self.assertIn("M1-M40", active_release_inventory)
|
|
self.assertIn("through M40", active_release_inventory)
|
|
for stale_value in (
|
|
*TRAINING_M41_STALE_RELEASE_INVENTORY_NAMES,
|
|
"M1-M39",
|
|
"through M39",
|
|
):
|
|
with self.subTest(stale_value=stale_value):
|
|
self.assertNotIn(stale_value, active_release_inventory)
|
|
|
|
def test_local_docker_deployment_documentation_guard_inventory_is_current(self) -> None:
|
|
module_path = Path(__file__)
|
|
module_source = module_path.read_text(encoding="utf-8")
|
|
module_tree = ast.parse(module_source)
|
|
class_nodes = [
|
|
node
|
|
for node in module_tree.body
|
|
if isinstance(node, ast.ClassDef) and node.name == "DocumentationConsistencyTests"
|
|
]
|
|
top_level_names = {
|
|
target.id
|
|
for node in module_tree.body
|
|
if isinstance(node, ast.Assign)
|
|
for target in node.targets
|
|
if isinstance(target, ast.Name)
|
|
}
|
|
required_guard_tests = {
|
|
"test_readmes_describe_single_set_local_docker_deployment_policy",
|
|
"test_readmes_lock_single_set_docker_smoke_command_shape",
|
|
}
|
|
inventory_names = tuple(LOCAL_DOCKER_DOCUMENTATION_GUARD_TEST_INVENTORY)
|
|
|
|
self.assertEqual(len(class_nodes), 1)
|
|
self.assertEqual(len(inventory_names), len(set(inventory_names)))
|
|
self.assertLessEqual(required_guard_tests, set(inventory_names))
|
|
|
|
test_names = {
|
|
item.name
|
|
for item in class_nodes[0].body
|
|
if isinstance(item, ast.FunctionDef) and item.name.startswith("test_")
|
|
}
|
|
for name in inventory_names:
|
|
with self.subTest(name=name):
|
|
loader = unittest.TestLoader()
|
|
suite = loader.loadTestsFromName(
|
|
f"{type(self).__module__}.DocumentationConsistencyTests.{name}"
|
|
)
|
|
|
|
self.assertIn(name, test_names)
|
|
self.assertEqual(loader.errors, [])
|
|
self.assertGreater(suite.countTestCases(), 0)
|
|
|
|
for constant_name in LOCAL_DOCKER_DOCUMENTATION_GUARD_SOURCE_CONSTANTS:
|
|
with self.subTest(constant_name=constant_name):
|
|
self.assertIn(constant_name, top_level_names)
|
|
self.assertIn(f"{constant_name} =", module_source)
|
|
|
|
def test_training_case_acceptance_manifest_modules_are_discoverable(self) -> None:
|
|
manifest_modules = tuple(TRAINING_CASE_ACCEPTANCE_TEST_MANIFEST)
|
|
missing_modules = [
|
|
module
|
|
for module in TRAINING_CASE_REQUIRED_ACCEPTANCE_TEST_MODULES
|
|
if module not in manifest_modules
|
|
]
|
|
|
|
self.assertEqual(missing_modules, [])
|
|
self.assertEqual(len(manifest_modules), len(set(manifest_modules)))
|
|
|
|
for module in manifest_modules:
|
|
with self.subTest(module=module):
|
|
path = ROOT / "tests" / "unit" / f"{module}.py"
|
|
loader = unittest.TestLoader()
|
|
suite = loader.loadTestsFromName(f"tests.unit.{module}")
|
|
|
|
self.assertTrue(path.is_file(), path)
|
|
self.assertEqual(loader.errors, [])
|
|
self.assertGreater(suite.countTestCases(), 0)
|
|
|
|
def test_training_readmes_name_dashboard_privacy_and_offline_registry_coverage(self) -> None:
|
|
expected_doc_snippets = (
|
|
"dashboard and public docs privacy smoke coverage",
|
|
"dashboard_js_syntax",
|
|
"training_case_package",
|
|
)
|
|
for path in (PROJECT_README, TRAINING_PRODUCT_DOC):
|
|
text = path.read_text(encoding="utf-8")
|
|
with self.subTest(path=path.name):
|
|
for snippet in expected_doc_snippets:
|
|
self.assertIn(snippet, text)
|
|
|
|
registry_text = OFFLINE_CHECKS_SCRIPT.read_text(encoding="utf-8")
|
|
for check_name in ("dashboard_js_syntax", "training_case_package"):
|
|
with self.subTest(check_name=check_name):
|
|
self.assertEqual(registry_text.count(f'"{check_name}"'), 1)
|
|
|
|
def test_training_readmes_describe_acceptance_manifest_offline_validation_posture(self) -> None:
|
|
for path in (PROJECT_README, TRAINING_PRODUCT_DOC):
|
|
text = path.read_text(encoding="utf-8")
|
|
with self.subTest(path=path.name):
|
|
for snippet in TRAINING_ACCEPTANCE_VALIDATION_DOC_SNIPPETS:
|
|
self.assertIn(snippet, text)
|
|
|
|
for path in TRAINING_PUBLIC_PRIVACY_DOCS:
|
|
text = path.read_text(encoding="utf-8")
|
|
with self.subTest(path=path.name, surface="manifest_modules"):
|
|
for module in TRAINING_CASE_ACCEPTANCE_TEST_MANIFEST:
|
|
self.assertNotIn(module, text)
|
|
|
|
def test_readmes_describe_single_set_local_docker_deployment_policy(self) -> None:
|
|
for path in LOCAL_DOCKER_POLICY_DOCS:
|
|
text = path.read_text(encoding="utf-8")
|
|
lower_text = text.lower()
|
|
|
|
with self.subTest(path=path.name):
|
|
for snippet in LOCAL_DOCKER_POLICY_REQUIRED_SNIPPETS:
|
|
self.assertIn(snippet, text)
|
|
self.assertLess(
|
|
lower_text.index("down --remove-orphans"),
|
|
lower_text.index("up -d --build"),
|
|
)
|
|
for forbidden in LOCAL_DOCKER_POLICY_FORBIDDEN_ENCOURAGEMENT:
|
|
self.assertNotIn(forbidden, lower_text)
|
|
|
|
def test_readmes_lock_single_set_docker_smoke_command_shape(self) -> None:
|
|
command_lines_by_path = {
|
|
path: _local_docker_command_lines(_local_docker_deployment_section(path))
|
|
for path in LOCAL_DOCKER_POLICY_DOCS
|
|
}
|
|
expected_commands = LOCAL_DOCKER_REBUILD_COMMANDS + LOCAL_DOCKER_SMOKE_COMMANDS
|
|
|
|
self.assertEqual(
|
|
command_lines_by_path[PROJECT_README],
|
|
command_lines_by_path[TRAINING_PRODUCT_DOC],
|
|
)
|
|
for path, command_lines in command_lines_by_path.items():
|
|
with self.subTest(path=path.name):
|
|
for command in expected_commands:
|
|
self.assertIn(command, command_lines)
|
|
self.assertLess(
|
|
command_lines.index(LOCAL_DOCKER_REBUILD_COMMANDS[0]),
|
|
command_lines.index(LOCAL_DOCKER_REBUILD_COMMANDS[1]),
|
|
)
|
|
for command in command_lines:
|
|
if command.startswith("docker compose"):
|
|
self.assertTrue(command.startswith(LOCAL_DOCKER_COMPOSE_PREFIX), command)
|
|
self.assertNotIn("docker compose -p ai-ict down --remove-orphans", command_lines)
|
|
self.assertNotIn("docker compose -p ai-ict up -d --build", command_lines)
|
|
|
|
def test_training_case_product_doc_names_public_response_privacy_contracts(self) -> None:
|
|
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8")
|
|
|
|
for snippet in (
|
|
"case list/detail",
|
|
"package export/import",
|
|
"progress",
|
|
"submit",
|
|
"create-attempt",
|
|
"annotation lifecycle",
|
|
"privacy contract tests",
|
|
"scrub private solution material",
|
|
"internal solution details",
|
|
):
|
|
with self.subTest(snippet=snippet):
|
|
self.assertIn(snippet, text)
|
|
|
|
self.assertNotIn("private " + "answer", text.lower())
|
|
self.assertNotIn("answer " + "detail", text.lower())
|
|
|
|
def test_training_case_product_doc_api_routes_match_server_route_request(self) -> None:
|
|
documented_routes = _training_api_routes_from_product_doc()
|
|
server_routes = _supported_training_routes_from_route_request()
|
|
|
|
self.assertEqual(
|
|
len(documented_routes),
|
|
len(set(documented_routes)),
|
|
"Training API docs contain duplicate route entries",
|
|
)
|
|
self.assertEqual(
|
|
sorted(documented_routes),
|
|
sorted(server_routes),
|
|
)
|
|
|
|
def test_training_case_public_route_registry_is_documented_and_supported(self) -> None:
|
|
documented_routes = set(_training_api_routes_from_product_doc())
|
|
server_routes = set(_supported_training_routes_from_route_request())
|
|
|
|
missing_from_docs = [
|
|
route for route in TRAINING_PUBLIC_API_ROUTE_REGISTRY if route not in documented_routes
|
|
]
|
|
missing_from_server = [
|
|
route for route in TRAINING_PUBLIC_API_ROUTE_REGISTRY if route not in server_routes
|
|
]
|
|
|
|
self.assertEqual(missing_from_docs, [])
|
|
self.assertEqual(missing_from_server, [])
|
|
|
|
def test_training_case_product_doc_api_surface_wording_matches_public_routes(self) -> None:
|
|
capabilities = {
|
|
(method, path): capability
|
|
for method, path, capability in _training_api_surface_rows_from_product_doc()
|
|
}
|
|
server_routes = set(_supported_training_routes_from_route_request())
|
|
|
|
self.assertEqual(sorted(capabilities), sorted(TRAINING_PUBLIC_API_ROUTE_REGISTRY))
|
|
for route, snippets in TRAINING_ROUTE_SURFACE_WORDING.items():
|
|
with self.subTest(route=route):
|
|
self.assertIn(route, server_routes)
|
|
for snippet in snippets:
|
|
self.assertIn(snippet, capabilities[route])
|
|
|
|
def test_training_dashboard_routes_are_documented_public_api_routes(self) -> None:
|
|
dashboard_routes = _training_dashboard_routes_from_app()
|
|
dashboard_paths = _training_dashboard_fetch_paths_from_app()
|
|
documented_routes = set(_training_api_routes_from_product_doc())
|
|
server_routes = set(_supported_training_routes_from_route_request())
|
|
|
|
self.assertEqual(dashboard_routes, TRAINING_DASHBOARD_ROUTE_REGISTRY)
|
|
self.assertEqual(dashboard_paths, tuple(path for _method, path in TRAINING_DASHBOARD_ROUTE_REGISTRY))
|
|
for route in dashboard_routes:
|
|
with self.subTest(route=route):
|
|
self.assertIn(route, TRAINING_PUBLIC_API_ROUTE_REGISTRY)
|
|
self.assertIn(route, documented_routes)
|
|
self.assertIn(route, server_routes)
|
|
|
|
def test_training_dashboard_and_public_docs_keep_privacy_wording_exclusionary(self) -> None:
|
|
dashboard_section = _training_product_doc_section("Dashboard Entry").lower()
|
|
dashboard_source = "\n".join(
|
|
path.read_text(encoding="utf-8").lower()
|
|
for path in (TRAINING_DASHBOARD_HTML, TRAINING_DASHBOARD_APP)
|
|
)
|
|
|
|
for snippet in TRAINING_DASHBOARD_PUBLIC_WORDING:
|
|
with self.subTest(snippet=snippet):
|
|
self.assertIn(snippet, dashboard_section)
|
|
|
|
for term in TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS:
|
|
with self.subTest(surface="dashboard", term=term):
|
|
self.assertNotIn(term, dashboard_source)
|
|
|
|
for phrase in TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES:
|
|
with self.subTest(surface="dashboard", phrase=phrase):
|
|
self.assertNotIn(phrase, dashboard_source)
|
|
|
|
for path in TRAINING_PUBLIC_PRIVACY_DOCS:
|
|
text = path.read_text(encoding="utf-8")
|
|
lower_text = text.lower()
|
|
privacy_lines = _training_public_privacy_lines(text)
|
|
|
|
with self.subTest(path=path.name):
|
|
self.assertTrue(privacy_lines)
|
|
for term in TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS:
|
|
self.assertNotIn(term, lower_text)
|
|
for line in privacy_lines:
|
|
self.assertTrue(
|
|
any(word in line for word in TRAINING_PUBLIC_PRIVACY_EXCLUSION_WORDS),
|
|
line,
|
|
)
|
|
|
|
def test_training_case_product_doc_names_public_error_semantics(self) -> None:
|
|
section = _training_validation_section()
|
|
|
|
for snippet in (
|
|
"`400` with `invalid_json_body`",
|
|
"`400` with `invalid_path_param:<field>`",
|
|
"`400` with `invalid_limit_query_param:limit`",
|
|
"`404` with `not_found`",
|
|
"`500` with `internal_server_error`",
|
|
"do not echo raw internals",
|
|
):
|
|
with self.subTest(snippet=snippet):
|
|
self.assertIn(snippet, section)
|
|
|
|
self.assertNotIn("answer" + "_key", section)
|
|
|
|
def test_training_case_product_doc_avoids_private_literal_shapes(self) -> None:
|
|
text = TRAINING_PRODUCT_DOC.read_text(encoding="utf-8").lower()
|
|
|
|
for term in (
|
|
"data" + "base",
|
|
"data" + "base_url",
|
|
"post" + "gresql",
|
|
"psy" + "copg",
|
|
"pass" + "word",
|
|
"pass" + "wd",
|
|
"tok" + "en",
|
|
"sec" + "ret",
|
|
"api" + "_key",
|
|
"answer" + "_key",
|
|
):
|
|
with self.subTest(term=term):
|
|
self.assertNotIn(term, text)
|
|
|
|
self.assertIsNone(re.search(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", text))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|