yuqei/ai-exchange
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: lock training dashboard docs privacy

Browse Source
This commit is contained in:
yuqei
2026-04-30 23:19:08 -04:00
parent 1caea87fe3
commit 6faec4a640
1 changed files with 131 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit/test_documentation_consistency.py
+131
View File
@@ -4,7 +4,11 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
PROJECT_README = ROOT / "README.md"
TRAINING_PRODUCT_DOC = ROOT / "docs" / "training_case_engine_m1_m6_product_readme.md"
TRAINING_PACKAGE_SMOKE_DOC = ROOT / "docs" / "examples" / "training_case_package_smoke_m6.md"
TRAINING_DASHBOARD_HTML = ROOT / "src" / "web" / "index.html"
TRAINING_DASHBOARD_APP = ROOT / "src" / "web" / "app.js"
API_SERVER = ROOT / "src" / "api" / "server.py"
TRAINING_PUBLIC_API_ROUTE_REGISTRY = (
("GET", "/training/cases"),
@@ -16,6 +20,13 @@ TRAINING_PUBLIC_API_ROUTE_REGISTRY = (
("POST", "/training/attempts/{attempt_id}/submit"),
("GET", "/training/progress"),
)
TRAINING_DASHBOARD_ROUTE_REGISTRY = (
("GET", "/training/cases"),
("GET", "/training/progress"),
("POST", "/training/attempts"),
("POST", "/training/attempts/{attempt_id}/annotations"),
("POST", "/training/attempts/{attempt_id}/submit"),
)
TRAINING_ROUTE_SURFACE_WORDING = {
("GET", "/training/cases"): ("list cases", "status", "skill tag", "instrument", "limit filters"),
("POST", "/training/cases/import"): ("import a versioned public package", "cases plus tasks"),
@@ -26,6 +37,51 @@ TRAINING_ROUTE_SURFACE_WORDING = {
("POST", "/training/attempts/{attempt_id}/submit"): ("submit an attempt", "optional score"),
("GET", "/training/progress"): ("list progress", "user", "skill tag", "limit filters"),
}
TRAINING_DASHBOARD_PUBLIC_WORDING = (
"available cases",
"user progress",
"attempt start",
"annotation note save",
"attempt submit",
"public case fields only",
)
TRAINING_PUBLIC_PRIVACY_DOCS = (PROJECT_README, TRAINING_PRODUCT_DOC, TRAINING_PACKAGE_SMOKE_DOC)
TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS = (
"answer" + "_key",
"private " + "answer",
"answer " + "detail",
"solution " + "key",
"solution_" + "key",
)
TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES = (
"private solution material",
"internal solution details",
)
TRAINING_PUBLIC_PRIVACY_EXCLUSION_WORDS = (
"scrub",
"omit",
"reject",
"keep",
"out of",
"do not",
"does not",
"without",
"hides",
"not return",
)
TRAINING_DASHBOARD_FETCH_PATTERNS = (
(("GET", "/training/cases"), r"fetchJson\('/training/cases'\)"),
(("GET", "/training/progress"), r"fetchJson\('/training/progress'\)"),
(("POST", "/training/attempts"), r"fetchJson\('/training/attempts',\s*\{\s*method:\s*'POST'"),
(
("POST", "/training/attempts/{attempt_id}/annotations"),
r"fetchJson\(`/training/attempts/\$\{[^}]+\}/annotations`,\s*\{\s*method:\s*'POST'",
),
(
("POST", "/training/attempts/{attempt_id}/submit"),
r"fetchJson\(`/training/attempts/\$\{[^}]+\}/submit`,\s*\{\s*method:\s*'POST'",
),
)
def _training_api_surface_rows_from_product_doc() -> tuple[tuple[str, str, str], ...]:
@@ -73,6 +129,24 @@ def _supported_training_routes_from_route_request() -> tuple[tuple[str, str], ..
return tuple(routes)
def _training_dashboard_routes_from_app() -> tuple[tuple[str, str], ...]:
source = TRAINING_DASHBOARD_APP.read_text(encoding="utf-8")
routes = []
for route, pattern in TRAINING_DASHBOARD_FETCH_PATTERNS:
if re.search(pattern, source, re.DOTALL):
routes.append(route)
return tuple(routes)
def _training_dashboard_fetch_paths_from_app() -> tuple[str, ...]:
source = TRAINING_DASHBOARD_APP.read_text(encoding="utf-8")
paths = []
for match in re.finditer(r"fetchJson\(\s*['`](?P<path>/training/[^'`]+)['`]", source):
path = re.sub(r"\$\{[^}]+\}", "{attempt_id}", match.group("path"))
paths.append(path.split("?", 1)[0])
return tuple(paths)
def _route_request_source() -> str:
source = API_SERVER.read_text(encoding="utf-8")
match = re.search(r"^def route_request\(.*?(?=^class APIRequestHandler)", source, re.DOTALL | re.MULTILINE)
@@ -97,6 +171,15 @@ def _training_validation_section() -> str:
return _training_product_doc_section("Validation And Errors")
def _training_public_privacy_lines(text: str) -> tuple[str, ...]:
lines = []
for line in text.splitlines():
lower_line = line.lower()
if any(phrase in lower_line for phrase in TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES):
lines.append(lower_line.strip())
return tuple(lines)
class DocumentationConsistencyTests(unittest.TestCase):
def test_readme_lists_script_output_json_environment_variables(self) -> None:
script_env_vars = set()
@@ -205,6 +288,54 @@ class DocumentationConsistencyTests(unittest.TestCase):
for snippet in snippets:
self.assertIn(snippet, capabilities[route])
def test_training_dashboard_routes_are_documented_public_api_routes(self) -> None:
dashboard_routes = _training_dashboard_routes_from_app()
dashboard_paths = _training_dashboard_fetch_paths_from_app()
documented_routes = set(_training_api_routes_from_product_doc())
server_routes = set(_supported_training_routes_from_route_request())
self.assertEqual(dashboard_routes, TRAINING_DASHBOARD_ROUTE_REGISTRY)
self.assertEqual(dashboard_paths, tuple(path for _method, path in TRAINING_DASHBOARD_ROUTE_REGISTRY))
for route in dashboard_routes:
with self.subTest(route=route):
self.assertIn(route, TRAINING_PUBLIC_API_ROUTE_REGISTRY)
self.assertIn(route, documented_routes)
self.assertIn(route, server_routes)
def test_training_dashboard_and_public_docs_keep_privacy_wording_exclusionary(self) -> None:
dashboard_section = _training_product_doc_section("Dashboard Entry").lower()
dashboard_source = "\n".join(
path.read_text(encoding="utf-8").lower()
for path in (TRAINING_DASHBOARD_HTML, TRAINING_DASHBOARD_APP)
)
for snippet in TRAINING_DASHBOARD_PUBLIC_WORDING:
with self.subTest(snippet=snippet):
self.assertIn(snippet, dashboard_section)
for term in TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS:
with self.subTest(surface="dashboard", term=term):
self.assertNotIn(term, dashboard_source)
for phrase in TRAINING_PUBLIC_PRIVACY_MATERIAL_PHRASES:
with self.subTest(surface="dashboard", phrase=phrase):
self.assertNotIn(phrase, dashboard_source)
for path in TRAINING_PUBLIC_PRIVACY_DOCS:
text = path.read_text(encoding="utf-8")
lower_text = text.lower()
privacy_lines = _training_public_privacy_lines(text)
with self.subTest(path=path.name):
self.assertTrue(privacy_lines)
for term in TRAINING_PUBLIC_PRIVACY_FORBIDDEN_TERMS:
self.assertNotIn(term, lower_text)
for line in privacy_lines:
self.assertTrue(
any(word in line for word in TRAINING_PUBLIC_PRIVACY_EXCLUSION_WORDS),
line,
)
def test_training_case_product_doc_names_public_error_semantics(self) -> None:
section = _training_validation_section()