197 lines
7.9 KiB
Python
197 lines
7.9 KiB
Python
import io
|
|
import gzip
|
|
import json
|
|
import tarfile
|
|
import tempfile
|
|
import unittest
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
from backend.tools.legal_source_diff_report import build_diff_report
|
|
|
|
|
|
def write_tar_with_metadata(archive_path: Path, entries: list[tuple[str, dict]]) -> None:
|
|
with tarfile.open(archive_path, "w:gz") as tar:
|
|
for folder_name, payload in entries:
|
|
data = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
info = tarfile.TarInfo(name=f"cn-laws-download/output/{folder_name}/metadata.json")
|
|
info.size = len(data)
|
|
info.mtime = int(datetime.now().timestamp())
|
|
tar.addfile(info, io.BytesIO(data))
|
|
|
|
fulltext = b"sample fulltext"
|
|
fulltext_info = tarfile.TarInfo(name=f"cn-laws-download/output/{folder_name}/fulltext.txt")
|
|
fulltext_info.size = len(fulltext)
|
|
fulltext_info.mtime = int(datetime.now().timestamp())
|
|
tar.addfile(fulltext_info, io.BytesIO(fulltext))
|
|
|
|
|
|
class LegalSourceDiffReportTestCase(unittest.TestCase):
|
|
def test_reports_source_checksum_and_metadata_gaps(self):
|
|
with tempfile.TemporaryDirectory() as temp_dir:
|
|
temp_path = Path(temp_dir)
|
|
archive_path = temp_path / "cn-laws-download.tar.gz"
|
|
output_dir = temp_path / "out"
|
|
remote_json = temp_path / "remote.json"
|
|
prepare_json = temp_path / "prepare.json"
|
|
import_json = temp_path / "import.json"
|
|
jsonl_path = temp_path / "prepared.jsonl.gz"
|
|
|
|
entries = [
|
|
(
|
|
"src-a",
|
|
{
|
|
"bbbs": "SRC-A",
|
|
"row": {
|
|
"bbbs": "SRC-A",
|
|
"title": "Alpha Regulation",
|
|
"flxz": "Local Rule",
|
|
"zdjgName": "Authority A",
|
|
"sxrq": "2026-01-01",
|
|
},
|
|
},
|
|
),
|
|
(
|
|
"src-b",
|
|
{
|
|
"bbbs": "SRC-B",
|
|
"row": {
|
|
"bbbs": "SRC-B",
|
|
"title": "Beta Regulation",
|
|
"flxz": "Local Rule",
|
|
"zdjgName": "",
|
|
"sxrq": "",
|
|
},
|
|
},
|
|
),
|
|
(
|
|
"src-c",
|
|
{
|
|
"bbbs": "SRC-C",
|
|
"row": {
|
|
"bbbs": "SRC-C",
|
|
"title": "Gamma Regulation",
|
|
"flxz": "Local Rule",
|
|
"zdjgName": "Authority C",
|
|
"sxrq": "2026-03-01",
|
|
},
|
|
},
|
|
),
|
|
]
|
|
write_tar_with_metadata(archive_path, entries)
|
|
|
|
prepared_rows = [
|
|
{
|
|
"id": "SRC-A-1",
|
|
"tenant_id": "tenant-demo-001",
|
|
"source_id": "SRC-A",
|
|
"source_title": "Alpha Regulation",
|
|
"source_type": "Local Rule",
|
|
"authority": "Authority A",
|
|
"effective_date": "2026-01-01",
|
|
"region": "CN",
|
|
"tags_json": ["tag-a"],
|
|
"article_no": "1",
|
|
"article_text": "Alpha article 1",
|
|
"source_checksum": "ck-a",
|
|
"text_checksum": "text-a",
|
|
},
|
|
{
|
|
"id": "SRC-B-1",
|
|
"tenant_id": "tenant-demo-001",
|
|
"source_id": "SRC-B",
|
|
"source_title": "Beta Regulation",
|
|
"source_type": "Local Rule",
|
|
"authority": "",
|
|
"effective_date": "",
|
|
"region": "CN",
|
|
"tags_json": [],
|
|
"article_no": "1",
|
|
"article_text": "Beta article 1",
|
|
"source_checksum": "ck-b",
|
|
"text_checksum": "text-b",
|
|
},
|
|
{
|
|
"id": "SRC-C-1",
|
|
"tenant_id": "tenant-demo-001",
|
|
"source_id": "SRC-C",
|
|
"source_title": "Gamma Regulation",
|
|
"source_type": "Local Rule",
|
|
"authority": "Authority C",
|
|
"effective_date": "2026-03-01",
|
|
"region": "CN",
|
|
"tags_json": ["tag-c"],
|
|
"article_no": "1",
|
|
"article_text": "Gamma article 1",
|
|
"source_checksum": "ck-c",
|
|
"text_checksum": "text-c",
|
|
},
|
|
]
|
|
with gzip.open(jsonl_path, "wt", encoding="utf-8") as file_obj:
|
|
for row in prepared_rows:
|
|
file_obj.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
|
|
remote_json.write_text(
|
|
json.dumps(
|
|
{
|
|
"tenant_id": "tenant-demo-001",
|
|
"legal_article_count": 3,
|
|
"remote_source_count": 2,
|
|
"remote_title_count": 2,
|
|
"sources": [
|
|
{
|
|
"source_id": "FLK-SRC-A",
|
|
"source_title": "Alpha Regulation",
|
|
"source_checksum": "ck-a",
|
|
"source_checksum_count": 1,
|
|
"article_count": 10,
|
|
},
|
|
{
|
|
"source_id": "FLK-SRC-Z",
|
|
"source_title": "Gamma Regulation",
|
|
"source_checksum": "ck-c-remote",
|
|
"source_checksum_count": 1,
|
|
"article_count": 5,
|
|
},
|
|
],
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
prepare_json.write_text(
|
|
json.dumps({"rows": 3, "jsonl_path": str(jsonl_path)}, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
import_json.write_text(
|
|
json.dumps({"inserted": 3, "skipped_existing_duplicates": 0}, ensure_ascii=False),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
summary = build_diff_report(
|
|
laws_backup_dir=archive_path,
|
|
remote_source_json=remote_json,
|
|
prepare_summary_json=prepare_json,
|
|
import_summary_json=import_json,
|
|
output_dir=output_dir,
|
|
tenant_id="tenant-demo-001",
|
|
)
|
|
|
|
self.assertEqual(summary["status"], "WARN")
|
|
self.assertEqual(summary["counts"]["backup_sources"], 3)
|
|
self.assertEqual(summary["counts"]["missing_sources_in_remote"], 1)
|
|
self.assertEqual(summary["counts"]["covered_by_remote_same_title"], 1)
|
|
self.assertEqual(summary["counts"]["needs_incremental_review"], 1)
|
|
self.assertEqual(summary["counts"]["imported_sources"], 2)
|
|
self.assertEqual(summary["counts"]["checksum_mismatch_sources"], 1)
|
|
self.assertEqual(summary["counts"]["metadata_gap_sources"], 1)
|
|
self.assertTrue((output_dir / "legal-source-diff-report.json").exists())
|
|
self.assertTrue((output_dir / "imported-sources.csv").exists())
|
|
self.assertTrue((output_dir / "missing-sources.csv").exists())
|
|
self.assertTrue((output_dir / "checksum-mismatch-sources.csv").exists())
|
|
self.assertTrue((output_dir / "metadata-gap-sources.csv").exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|