test: lock training package fixture privacy
This commit is contained in:
@@ -36,6 +36,7 @@ PUBLIC_PACKAGE_REQUIRED_TEXT = (
|
||||
"internal solution details",
|
||||
)
|
||||
PRIVATE_DOC_DRIFT_PHRASES = ("private " + "answer", "answer " + "detail")
|
||||
PRIVATE_FIXTURE_DRIFT_PHRASES = PRIVATE_MATERIAL_PHRASES + PRIVATE_DOC_DRIFT_PHRASES
|
||||
RESTRICTED_PUBLIC_TEXT = (
|
||||
"data" + "base",
|
||||
"data" + "base_url",
|
||||
@@ -93,15 +94,29 @@ def _walk_public_text(value) -> list[str]:
|
||||
class TrainingCaseDocsCliM10Tests(unittest.TestCase):
|
||||
def assertNoPrivateFixtureMaterial(self, payload: dict) -> None:
|
||||
serialized = json.dumps(payload, sort_keys=True)
|
||||
lower_serialized = serialized.lower()
|
||||
|
||||
self.assertNotIn(PUBLIC_FIELD, serialized)
|
||||
self.assertNotIn(PUBLIC_FIELD, lower_serialized)
|
||||
self.assertIsNone(re.search(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", serialized))
|
||||
for phrase in PRIVATE_FIXTURE_DRIFT_PHRASES:
|
||||
self.assertNotIn(phrase, lower_serialized)
|
||||
for text in _walk_public_text(payload):
|
||||
lower_text = text.lower()
|
||||
for fragment in RESTRICTED_PUBLIC_TEXT:
|
||||
with self.subTest(text=text, fragment=fragment):
|
||||
self.assertNotIn(fragment, lower_text)
|
||||
|
||||
def test_public_privacy_guard_rejects_solution_wording_drift(self) -> None:
|
||||
polluted_payloads = [
|
||||
{"items": [{"prompt": "Do not expose " + ("private " + "answer") + " wording."}]},
|
||||
{"items": [{"prompt": "Do not expose internal solution details wording."}]},
|
||||
]
|
||||
|
||||
for payload in polluted_payloads:
|
||||
with self.subTest(payload=payload):
|
||||
with self.assertRaises(AssertionError):
|
||||
self.assertNoPrivateFixtureMaterial(payload)
|
||||
|
||||
def test_public_fixture_schema_contract_is_minimal_and_demo_safe(self) -> None:
|
||||
payload = _fixture_payload()
|
||||
|
||||
@@ -130,8 +145,15 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
|
||||
self.assertEqual(item["skill_tags"], ["sweep", "context_reading"])
|
||||
self.assertEqual(item["difficulty"], "easy")
|
||||
self.assertEqual(item["answer"], {})
|
||||
self.assertEqual(len(item["rubric"]), 2)
|
||||
self.assertEqual(
|
||||
item["rubric"],
|
||||
[
|
||||
{"category": "context", "max_score": 3},
|
||||
{"category": "risk_story", "max_score": 2},
|
||||
],
|
||||
)
|
||||
self.assertEqual(len(item["tasks"]), 1)
|
||||
self.assertEqual(item["metadata"], {"chapter_ref": "public_training_m6", "demo": True})
|
||||
|
||||
scenario = item["scenario"]
|
||||
self.assertEqual(
|
||||
@@ -152,7 +174,13 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
|
||||
self.assertEqual(scenario["instrument_id"], 1)
|
||||
self.assertEqual(scenario["timeframe"], "15m")
|
||||
self.assertEqual(scenario["case_type"], "sweep")
|
||||
self.assertEqual(scenario["start_ts"], "2026-04-20T13:00:00Z")
|
||||
self.assertEqual(scenario["decision_ts"], "2026-04-20T14:00:00Z")
|
||||
self.assertEqual(scenario["end_ts"], "2026-04-20T15:00:00Z")
|
||||
self.assertEqual(scenario["source_event_type"], "sweep_event")
|
||||
self.assertEqual(scenario["source_event_id"], 61)
|
||||
self.assertEqual(scenario["source_event_refs"], [{"kind": "sweep_event", "id": 61}])
|
||||
self.assertEqual(scenario["chapter_ref"], "public_training_m6")
|
||||
|
||||
task = item["tasks"][0]
|
||||
self.assertEqual(
|
||||
@@ -169,8 +197,11 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(task["task_type"], "short_answer")
|
||||
self.assertEqual(task["skill_tag"], "sweep")
|
||||
self.assertEqual(task["prompt"], "Describe what needs confirmation before marking this as a usable sweep.")
|
||||
self.assertEqual(task["required_annotation_types"], ["text"])
|
||||
self.assertEqual(task["max_score"], 5)
|
||||
self.assertEqual(task["sort_order"], 0)
|
||||
self.assertEqual(task["meta"], {"public_hint": "Focus on context and confirmation."})
|
||||
self.assertNoPrivateFixtureMaterial(payload)
|
||||
|
||||
def test_fixture_is_compatible_with_public_service_and_cli_export(self) -> None:
|
||||
@@ -193,14 +224,38 @@ class TrainingCaseDocsCliM10Tests(unittest.TestCase):
|
||||
self.assertEqual(summary["imported"], 1)
|
||||
self.assertEqual(summary["exported"], 1)
|
||||
self.assertEqual(export_result, cli_export)
|
||||
self.assertEqual(set(export_result), {"version", "metadata", "items"})
|
||||
self.assertEqual(export_result["metadata"], {"item_count": 1})
|
||||
|
||||
exported_item = export_result["items"][0]
|
||||
fixture_item = payload["items"][0]
|
||||
self.assertEqual(
|
||||
set(exported_item),
|
||||
{
|
||||
"id",
|
||||
"title",
|
||||
"scenario",
|
||||
"skill_tags",
|
||||
"difficulty",
|
||||
"status",
|
||||
"rubric",
|
||||
"tasks",
|
||||
"metadata",
|
||||
},
|
||||
)
|
||||
self.assertEqual(exported_item["title"], fixture_item["title"])
|
||||
self.assertEqual(set(exported_item["skill_tags"]), set(fixture_item["skill_tags"]))
|
||||
self.assertEqual(exported_item["scenario"]["case_type"], fixture_item["scenario"]["case_type"])
|
||||
self.assertEqual(exported_item["tasks"][0]["task_type"], fixture_item["tasks"][0]["task_type"])
|
||||
self.assertEqual(
|
||||
exported_item["metadata"],
|
||||
{"chapter_ref": "public_training_m6", "demo": True, "external_id": fixture_item["external_id"]},
|
||||
)
|
||||
self.assertEqual(exported_item["rubric"], fixture_item["rubric"])
|
||||
self.assertEqual(exported_item["scenario"], fixture_item["scenario"])
|
||||
self.assertEqual(set(exported_item["tasks"][0]), set(fixture_item["tasks"][0]) | {"status"})
|
||||
self.assertEqual(exported_item["tasks"][0]["status"], "active")
|
||||
for key, value in fixture_item["tasks"][0].items():
|
||||
with self.subTest(task_field=key):
|
||||
self.assertEqual(exported_item["tasks"][0][key], value)
|
||||
self.assertNotIn("answer", exported_item)
|
||||
self.assertNoPrivateFixtureMaterial({"imported": import_result, "exported": export_result})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user