From 6faec4a6401bd56855a3fae3db5fa599f154d33c Mon Sep 17 00:00:00 2001 From: yuqei Date: Thu, 30 Apr 2026 23:19:08 -0400 Subject: [PATCH] test: lock training dashboard docs privacy --- tests/unit/test_documentation_consistency.py | 131 +++++++++++++++++++ 1 file changed, 131 insertions(+) diff --git a/tests/unit/test_documentation_consistency.py b/tests/unit/test_documentation_consistency.py index 1ccdf64..3b37576 100644 --- a/tests/unit/test_documentation_consistency.py +++ b/tests/unit/test_documentation_consistency.py @@ -4,7 +4,11 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[2] +PROJECT_README = ROOT / "README.md" TRAINING_PRODUCT_DOC = ROOT / "docs" / "training_case_engine_m1_m6_product_readme.md" +TRAINING_PACKAGE_SMOKE_DOC = ROOT / "docs" / "examples" / "training_case_package_smoke_m6.md" +TRAINING_DASHBOARD_HTML = ROOT / "src" / "web" / "index.html" +TRAINING_DASHBOARD_APP = ROOT / "src" / "web" / "app.js" API_SERVER = ROOT / "src" / "api" / "server.py" TRAINING_PUBLIC_API_ROUTE_REGISTRY = ( ("GET", "/training/cases"), @@ -16,6 +20,13 @@ TRAINING_PUBLIC_API_ROUTE_REGISTRY = ( ("POST", "/training/attempts/{attempt_id}/submit"), ("GET", "/training/progress"), ) +TRAINING_DASHBOARD_ROUTE_REGISTRY = ( + ("GET", "/training/cases"), + ("GET", "/training/progress"), + ("POST", "/training/attempts"), + ("POST", "/training/attempts/{attempt_id}/annotations"), + ("POST", "/training/attempts/{attempt_id}/submit"), +) TRAINING_ROUTE_SURFACE_WORDING = { ("GET", "/training/cases"): ("list cases", "status", "skill tag", "instrument", "limit filters"), ("POST", "/training/cases/import"): ("import a versioned public package", "cases plus tasks"), @@ -26,6 +37,51 @@ TRAINING_ROUTE_SURFACE_WORDING = { ("POST", "/training/attempts/{attempt_id}/submit"): ("submit an attempt", "optional score"), ("GET", "/training/progress"): ("list progress", "user", "skill tag", "limit filters"), } +TRAINING_DASHBOARD_PUBLIC_WORDING = ( + "available cases", + "user progress", + "attempt start", + "annotation note save", + "attempt submit", + "public case fields only", +) +TRAINING_PUBLIC_PRIVACY_DOCS = (PROJECT_README, TRAINING_PRODUCT_DOC, TRAINING_PACKAGE_SMOKE_DOC) +TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS = ( + "answer" + "_key", + "private " + "answer", + "answer " + "detail", + "solution " + "key", + "solution_" + "key", +) +TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES = ( + "private solution material", + "internal solution details", +) +TRAINING_PUBLIC_PRIVACY_EXCLUSION_WORDS = ( + "scrub", + "omit", + "reject", + "keep", + "out of", + "do not", + "does not", + "without", + "hides", + "not return", +) +TRAINING_DASHBOARD_FETCH_PATTERNS = ( + (("GET", "/training/cases"), r"fetchJson\('/training/cases'\)"), + (("GET", "/training/progress"), r"fetchJson\('/training/progress'\)"), + (("POST", "/training/attempts"), r"fetchJson\('/training/attempts',\s*\{\s*method:\s*'POST'"), + ( + ("POST", "/training/attempts/{attempt_id}/annotations"), + r"fetchJson\(`/training/attempts/\$\{[^}]+\}/annotations`,\s*\{\s*method:\s*'POST'", + ), + ( + ("POST", "/training/attempts/{attempt_id}/submit"), + r"fetchJson\(`/training/attempts/\$\{[^}]+\}/submit`,\s*\{\s*method:\s*'POST'", + ), +) def _training_api_surface_rows_from_product_doc() -> tuple[tuple[str, str, str], ...]: @@ -73,6 +129,24 @@ def _supported_training_routes_from_route_request() -> tuple[tuple[str, str], .. return tuple(routes) +def _training_dashboard_routes_from_app() -> tuple[tuple[str, str], ...]: + source = TRAINING_DASHBOARD_APP.read_text(encoding="utf-8") + routes = [] + for route, pattern in TRAINING_DASHBOARD_FETCH_PATTERNS: + if re.search(pattern, source, re.DOTALL): + routes.append(route) + return tuple(routes) + + +def _training_dashboard_fetch_paths_from_app() -> tuple[str, ...]: + source = TRAINING_DASHBOARD_APP.read_text(encoding="utf-8") + paths = [] + for match in re.finditer(r"fetchJson\(\s*['`](?P/training/[^'`]+)['`]", source): + path = re.sub(r"\$\{[^}]+\}", "{attempt_id}", match.group("path")) + paths.append(path.split("?", 1)[0]) + return tuple(paths) + + def _route_request_source() -> str: source = API_SERVER.read_text(encoding="utf-8") match = re.search(r"^def route_request\(.*?(?=^class APIRequestHandler)", source, re.DOTALL | re.MULTILINE) @@ -97,6 +171,15 @@ def _training_validation_section() -> str: return _training_product_doc_section("Validation And Errors") +def _training_public_privacy_lines(text: str) -> tuple[str, ...]: + lines = [] + for line in text.splitlines(): + lower_line = line.lower() + if any(phrase in lower_line for phrase in TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES): + lines.append(lower_line.strip()) + return tuple(lines) + + class DocumentationConsistencyTests(unittest.TestCase): def test_readme_lists_script_output_json_environment_variables(self) -> None: script_env_vars = set() @@ -205,6 +288,54 @@ class DocumentationConsistencyTests(unittest.TestCase): for snippet in snippets: self.assertIn(snippet, capabilities[route]) + def test_training_dashboard_routes_are_documented_public_api_routes(self) -> None: + dashboard_routes = _training_dashboard_routes_from_app() + dashboard_paths = _training_dashboard_fetch_paths_from_app() + documented_routes = set(_training_api_routes_from_product_doc()) + server_routes = set(_supported_training_routes_from_route_request()) + + self.assertEqual(dashboard_routes, TRAINING_DASHBOARD_ROUTE_REGISTRY) + self.assertEqual(dashboard_paths, tuple(path for _method, path in TRAINING_DASHBOARD_ROUTE_REGISTRY)) + for route in dashboard_routes: + with self.subTest(route=route): + self.assertIn(route, TRAINING_PUBLIC_API_ROUTE_REGISTRY) + self.assertIn(route, documented_routes) + self.assertIn(route, server_routes) + + def test_training_dashboard_and_public_docs_keep_privacy_wording_exclusionary(self) -> None: + dashboard_section = _training_product_doc_section("Dashboard Entry").lower() + dashboard_source = "\n".join( + path.read_text(encoding="utf-8").lower() + for path in (TRAINING_DASHBOARD_HTML, TRAINING_DASHBOARD_APP) + ) + + for snippet in TRAINING_DASHBOARD_PUBLIC_WORDING: + with self.subTest(snippet=snippet): + self.assertIn(snippet, dashboard_section) + + for term in TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS: + with self.subTest(surface="dashboard", term=term): + self.assertNotIn(term, dashboard_source) + + for phrase in TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES: + with self.subTest(surface="dashboard", phrase=phrase): + self.assertNotIn(phrase, dashboard_source) + + for path in TRAINING_PUBLIC_PRIVACY_DOCS: + text = path.read_text(encoding="utf-8") + lower_text = text.lower() + privacy_lines = _training_public_privacy_lines(text) + + with self.subTest(path=path.name): + self.assertTrue(privacy_lines) + for term in TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS: + self.assertNotIn(term, lower_text) + for line in privacy_lines: + self.assertTrue( + any(word in line for word in TRAINING_PUBLIC_PRIVACY_EXCLUSION_WORDS), + line, + ) + def test_training_case_product_doc_names_public_error_semantics(self) -> None: section = _training_validation_section()