test: lock training package fixture privacy

This commit is contained in:
2026-04-30 23:04:02 -04:00
parent 0024591b25
commit 1d287539b1
+59 -4
View File
@@ -36,6 +36,7 @@ PUBLIC_PACKAGE_REQUIRED_TEXT = (
"internal solution details",
)
PRIVATE_DOC_DRIFT_PHRASES = ("private " + "answer", "answer " + "detail")
PRIVATE_FIXTURE_DRIFT_PHRASES = PRIVATE_MATERIAL_PHRASES + PRIVATE_DOC_DRIFT_PHRASES
RESTRICTED_PUBLIC_TEXT = (
"data" + "base",
"data" + "base_url",
@@ -93,15 +94,29 @@ def _walk_public_text(value) -> list[str]:
class TrainingCaseDocsCliM10Tests(unittest.TestCase):
def assertNoPrivateFixtureMaterial(self, payload: dict) -> None:
serialized = json.dumps(payload, sort_keys=True)
lower_serialized = serialized.lower()
self.assertNotIn(PUBLIC_FIELD, serialized)
self.assertNotIn(PUBLIC_FIELD, lower_serialized)
self.assertIsNone(re.search(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", serialized))
for phrase in PRIVATE_FIXTURE_DRIFT_PHRASES:
self.assertNotIn(phrase, lower_serialized)
for text in _walk_public_text(payload):
lower_text = text.lower()
for fragment in RESTRICTED_PUBLIC_TEXT:
with self.subTest(text=text, fragment=fragment):
self.assertNotIn(fragment, lower_text)
def test_public_privacy_guard_rejects_solution_wording_drift(self) -> None:
polluted_payloads = [
{"items": [{"prompt": "Do not expose " + ("private " + "answer") + " wording."}]},
{"items": [{"prompt": "Do not expose internal solution details wording."}]},
]
for payload in polluted_payloads:
with self.subTest(payload=payload):
with self.assertRaises(AssertionError):
self.assertNoPrivateFixtureMaterial(payload)
def test_public_fixture_schema_contract_is_minimal_and_demo_safe(self) -> None:
payload = _fixture_payload()
@@ -130,8 +145,15 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
self.assertEqual(item["skill_tags"], ["sweep", "context_reading"])
self.assertEqual(item["difficulty"], "easy")
self.assertEqual(item["answer"], {})
self.assertEqual(len(item["rubric"]), 2)
self.assertEqual(
item["rubric"],
[
{"category": "context", "max_score": 3},
{"category": "risk_story", "max_score": 2},
],
)
self.assertEqual(len(item["tasks"]), 1)
self.assertEqual(item["metadata"], {"chapter_ref": "public_training_m6", "demo": True})
scenario = item["scenario"]
self.assertEqual(
@@ -152,7 +174,13 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
self.assertEqual(scenario["instrument_id"], 1)
self.assertEqual(scenario["timeframe"], "15m")
self.assertEqual(scenario["case_type"], "sweep")
self.assertEqual(scenario["start_ts"], "2026-04-20T13:00:00Z")
self.assertEqual(scenario["decision_ts"], "2026-04-20T14:00:00Z")
self.assertEqual(scenario["end_ts"], "2026-04-20T15:00:00Z")
self.assertEqual(scenario["source_event_type"], "sweep_event")
self.assertEqual(scenario["source_event_id"], 61)
self.assertEqual(scenario["source_event_refs"], [{"kind": "sweep_event", "id": 61}])
self.assertEqual(scenario["chapter_ref"], "public_training_m6")
task = item["tasks"][0]
self.assertEqual(
@@ -169,8 +197,11 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
)
self.assertEqual(task["task_type"], "short_answer")
self.assertEqual(task["skill_tag"], "sweep")
self.assertEqual(task["prompt"], "Describe what needs confirmation before marking this as a usable sweep.")
self.assertEqual(task["required_annotation_types"], ["text"])
self.assertEqual(task["max_score"], 5)
self.assertEqual(task["sort_order"], 0)
self.assertEqual(task["meta"], {"public_hint": "Focus on context and confirmation."})
self.assertNoPrivateFixtureMaterial(payload)
def test_fixture_is_compatible_with_public_service_and_cli_export(self) -> None:
@@ -193,14 +224,38 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
self.assertEqual(summary["imported"], 1)
self.assertEqual(summary["exported"], 1)
self.assertEqual(export_result, cli_export)
self.assertEqual(set(export_result), {"version", "metadata", "items"})
self.assertEqual(export_result["metadata"], {"item_count": 1})
exported_item = export_result["items"][0]
fixture_item = payload["items"][0]
self.assertEqual(
set(exported_item),
{
"id",
"title",
"scenario",
"skill_tags",
"difficulty",
"status",
"rubric",
"tasks",
"metadata",
},
)
self.assertEqual(exported_item["title"], fixture_item["title"])
self.assertEqual(set(exported_item["skill_tags"]), set(fixture_item["skill_tags"]))
self.assertEqual(exported_item["scenario"]["case_type"], fixture_item["scenario"]["case_type"])
self.assertEqual(exported_item["tasks"][0]["task_type"], fixture_item["tasks"][0]["task_type"])
self.assertEqual(
exported_item["metadata"],
{"chapter_ref": "public_training_m6", "demo": True, "external_id": fixture_item["external_id"]},
)
self.assertEqual(exported_item["rubric"], fixture_item["rubric"])
self.assertEqual(exported_item["scenario"], fixture_item["scenario"])
self.assertEqual(set(exported_item["tasks"][0]), set(fixture_item["tasks"][0]) | {"status"})
self.assertEqual(exported_item["tasks"][0]["status"], "active")
for key, value in fixture_item["tasks"][0].items():
with self.subTest(task_field=key):
self.assertEqual(exported_item["tasks"][0][key], value)
self.assertNotIn("answer", exported_item)
self.assertNoPrivateFixtureMaterial({"imported": import_result, "exported": export_result})