607 lines
27 KiB
Python
607 lines
27 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
import gzip
|
|
import json
|
|
import tarfile
|
|
from collections import Counter, defaultdict
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
def read_json(path: Path | None) -> dict[str, Any]:
|
|
if not path or not path.exists():
|
|
return {}
|
|
return json.loads(path.read_text(encoding="utf-8-sig"))
|
|
|
|
|
|
def latest_file(directory: Path, pattern: str) -> Path | None:
|
|
files = sorted(directory.glob(pattern), key=lambda item: item.stat().st_mtime, reverse=True)
|
|
return files[0] if files else None
|
|
|
|
|
|
def find_laws_archive(laws_backup_dir: Path) -> Path:
|
|
if laws_backup_dir.is_file() and laws_backup_dir.name.endswith(".tar.gz"):
|
|
return laws_backup_dir
|
|
archive = latest_file(laws_backup_dir, "*.tar.gz")
|
|
if archive is None:
|
|
raise FileNotFoundError(f"cn-laws-download archive not found under {laws_backup_dir}")
|
|
return archive
|
|
|
|
|
|
def normalize_text(value: Any) -> str:
|
|
return str(value or "").strip()
|
|
|
|
|
|
def normalize_source_id(value: Any) -> str:
|
|
source_id = normalize_text(value)
|
|
if source_id.upper().startswith("FLK-"):
|
|
return source_id[4:]
|
|
return source_id
|
|
|
|
|
|
def metadata_parent(member_name: str) -> str:
|
|
return str(Path(member_name).parent).replace("\\", "/")
|
|
|
|
|
|
def pick_source_id(payload: dict[str, Any], member_name: str) -> str:
|
|
row = payload.get("row") or {}
|
|
detail_data = ((payload.get("detail") or {}).get("data") or {})
|
|
source_id = normalize_text(payload.get("bbbs") or row.get("bbbs") or detail_data.get("bbbs"))
|
|
if source_id:
|
|
return source_id
|
|
parts = Path(member_name).parts
|
|
if len(parts) >= 3 and parts[-1] == "metadata.json":
|
|
return parts[-2]
|
|
return ""
|
|
|
|
|
|
def scan_backup_sources(laws_backup_dir: Path) -> dict[str, Any]:
|
|
archive = find_laws_archive(laws_backup_dir)
|
|
sources_by_id: dict[str, dict[str, Any]] = {}
|
|
duplicate_ids: list[dict[str, Any]] = []
|
|
fulltext_parents: set[str] = set()
|
|
|
|
with tarfile.open(archive, "r:gz") as tar:
|
|
for member in tar:
|
|
if member.isfile() and member.name.endswith("/fulltext.txt"):
|
|
fulltext_parents.add(metadata_parent(member.name))
|
|
continue
|
|
if not (member.isfile() and member.name.endswith("/metadata.json")):
|
|
continue
|
|
file_obj = tar.extractfile(member)
|
|
if file_obj is None:
|
|
continue
|
|
try:
|
|
payload = json.loads(file_obj.read().decode("utf-8"))
|
|
except Exception as exc: # noqa: BLE001
|
|
duplicate_ids.append({"source_id": "", "metadata_path": member.name, "error": str(exc)})
|
|
continue
|
|
|
|
row = payload.get("row") or {}
|
|
detail_data = ((payload.get("detail") or {}).get("data") or {})
|
|
source_id = pick_source_id(payload, member.name)
|
|
title = normalize_text(row.get("title") or detail_data.get("title"))
|
|
source = {
|
|
"source_id": source_id,
|
|
"canonical_source_id": normalize_source_id(source_id),
|
|
"source_title": title,
|
|
"source_type": normalize_text(row.get("flxz") or detail_data.get("flxz") or "未知"),
|
|
"authority": normalize_text(row.get("zdjgName") or detail_data.get("zdjgName")),
|
|
"effective_date": normalize_text(row.get("sxrq") or detail_data.get("sxrq")),
|
|
"publish_date": normalize_text(row.get("gbrq") or detail_data.get("gbrq")),
|
|
"validity_status": normalize_text(row.get("sxx") or detail_data.get("sxx")),
|
|
"source_page": payload.get("source_page"),
|
|
"metadata_path": member.name,
|
|
"source_directory": metadata_parent(member.name),
|
|
"fulltext_length": int(payload.get("fulltext_length") or 0),
|
|
"has_reader_file": bool(payload.get("reader_file")),
|
|
"has_ofd_path": bool(payload.get("ofd_path")),
|
|
"saved_at": normalize_text(payload.get("saved_at")),
|
|
}
|
|
if not source_id:
|
|
duplicate_ids.append(source | {"error": "missing_source_id"})
|
|
continue
|
|
if source_id in sources_by_id:
|
|
duplicate_ids.append(source | {"error": "duplicate_source_id"})
|
|
continue
|
|
sources_by_id[source_id] = source
|
|
|
|
for source in sources_by_id.values():
|
|
source["has_fulltext"] = source["source_directory"] in fulltext_parents
|
|
|
|
return {
|
|
"archive": str(archive),
|
|
"sources": sources_by_id,
|
|
"duplicate_or_invalid_metadata": duplicate_ids,
|
|
"fulltext_documents": len(fulltext_parents),
|
|
}
|
|
|
|
|
|
def scan_prepared_source_inventory(prepare_summary_json: Path | None) -> dict[str, Any]:
|
|
summary = read_json(prepare_summary_json)
|
|
jsonl_path = Path(summary.get("jsonl_path") or "")
|
|
result: dict[str, Any] = {
|
|
"jsonl_path": str(jsonl_path) if jsonl_path else "",
|
|
"sources": {},
|
|
"error": "",
|
|
}
|
|
if not jsonl_path or not jsonl_path.exists():
|
|
result["error"] = "prepared_jsonl_not_found"
|
|
return result
|
|
|
|
sources_by_id: dict[str, dict[str, Any]] = {}
|
|
with gzip.open(jsonl_path, "rt", encoding="utf-8") as file_obj:
|
|
for line in file_obj:
|
|
if not line.strip():
|
|
continue
|
|
row = json.loads(line)
|
|
source_id = normalize_text(row.get("source_id"))
|
|
canonical_source_id = normalize_source_id(source_id)
|
|
if not canonical_source_id:
|
|
continue
|
|
source = sources_by_id.setdefault(
|
|
canonical_source_id,
|
|
{
|
|
"source_id": source_id,
|
|
"canonical_source_id": canonical_source_id,
|
|
"source_title": normalize_text(row.get("source_title")),
|
|
"source_type": normalize_text(row.get("source_type") or "未知"),
|
|
"authority": normalize_text(row.get("authority")),
|
|
"effective_date": normalize_text(row.get("effective_date")),
|
|
"region": normalize_text(row.get("region") or "CN"),
|
|
"tags_json": row.get("tags_json") or [],
|
|
"article_count": 0,
|
|
"source_checksum_values": set(),
|
|
"text_checksum_values": set(),
|
|
"missing_authority_rows": 0,
|
|
"missing_effective_date_rows": 0,
|
|
"missing_region_rows": 0,
|
|
"missing_tags_rows": 0,
|
|
},
|
|
)
|
|
source["article_count"] += 1
|
|
if not source["source_title"]:
|
|
source["source_title"] = normalize_text(row.get("source_title"))
|
|
if not source["authority"]:
|
|
source["authority"] = normalize_text(row.get("authority"))
|
|
if not source["effective_date"]:
|
|
source["effective_date"] = normalize_text(row.get("effective_date"))
|
|
if not source["region"]:
|
|
source["region"] = normalize_text(row.get("region") or "CN")
|
|
tags_json = row.get("tags_json") or []
|
|
if tags_json:
|
|
source["tags_json"] = tags_json
|
|
if not normalize_text(row.get("authority")):
|
|
source["missing_authority_rows"] += 1
|
|
if not normalize_text(row.get("effective_date")):
|
|
source["missing_effective_date_rows"] += 1
|
|
if not normalize_text(row.get("region")):
|
|
source["missing_region_rows"] += 1
|
|
if not tags_json:
|
|
source["missing_tags_rows"] += 1
|
|
checksum = normalize_text(row.get("source_checksum"))
|
|
if checksum:
|
|
source["source_checksum_values"].add(checksum)
|
|
text_checksum = normalize_text(row.get("text_checksum"))
|
|
if text_checksum:
|
|
source["text_checksum_values"].add(text_checksum)
|
|
|
|
for source in sources_by_id.values():
|
|
source_checksum_values = sorted(source.pop("source_checksum_values"))
|
|
text_checksum_values = sorted(source.pop("text_checksum_values"))
|
|
source["source_checksum"] = source_checksum_values[0] if len(source_checksum_values) == 1 else ";".join(source_checksum_values)
|
|
source["source_checksum_count"] = len(source_checksum_values)
|
|
source["text_checksum_count"] = len(text_checksum_values)
|
|
source["missing_fields"] = [
|
|
field
|
|
for field, missing_rows in [
|
|
("authority", source["missing_authority_rows"]),
|
|
("effective_date", source["missing_effective_date_rows"]),
|
|
("region", source["missing_region_rows"]),
|
|
("tags", source["missing_tags_rows"]),
|
|
]
|
|
if missing_rows > 0
|
|
]
|
|
result["sources"] = sources_by_id
|
|
return result
|
|
|
|
|
|
def build_title_index(sources: list[dict[str, Any]]) -> dict[str, list[dict[str, Any]]]:
|
|
title_index: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
|
for source in sources:
|
|
title = normalize_text(source.get("source_title"))
|
|
if title:
|
|
title_index[title].append(source)
|
|
return title_index
|
|
|
|
|
|
def build_source_comparison(
|
|
source: dict[str, Any],
|
|
remote_sources_by_id: dict[str, dict[str, Any]],
|
|
remote_title_index: dict[str, list[dict[str, Any]]],
|
|
) -> dict[str, Any]:
|
|
canonical_source_id = normalize_source_id(source.get("source_id"))
|
|
title = normalize_text(source.get("source_title"))
|
|
remote_exact = remote_sources_by_id.get(canonical_source_id)
|
|
title_matches = remote_title_index.get(title, [])
|
|
if remote_exact is not None:
|
|
classification = "imported_exact_source_id"
|
|
remote_matches = [remote_exact]
|
|
action_suggestion = "已入库；source_id 与远程运行库对齐。"
|
|
elif title_matches:
|
|
classification = "covered_by_remote_same_title"
|
|
remote_matches = title_matches
|
|
action_suggestion = "暂不重复导入；同标题内容已在运行库中覆盖，后续只需抽样核验。"
|
|
else:
|
|
classification = "needs_incremental_review"
|
|
remote_matches = []
|
|
action_suggestion = "需要进入增量补抓/补导入复核队列。"
|
|
|
|
remote_source_ids_same_title = [normalize_text(item.get("source_id")) for item in remote_matches if normalize_text(item.get("source_id"))]
|
|
remote_source_checksums = [normalize_text(item.get("source_checksum")) for item in remote_matches if normalize_text(item.get("source_checksum"))]
|
|
remote_same_title_article_count = sum(int(item.get("article_count") or 0) for item in remote_matches)
|
|
remote_same_title_count = len(remote_matches)
|
|
remote_source_checksum = normalize_text(remote_exact.get("source_checksum")) if remote_exact else ""
|
|
remote_source_checksum_count = int(remote_exact.get("source_checksum_count") or 0) if remote_exact else 0
|
|
|
|
checksum_mismatch = False
|
|
checksum_reason = ""
|
|
source_checksum = normalize_text(source.get("source_checksum"))
|
|
if remote_exact is not None:
|
|
if remote_source_checksum_count > 1:
|
|
checksum_mismatch = True
|
|
checksum_reason = "remote_source_checksum_inconsistent"
|
|
elif source_checksum and remote_source_checksum and source_checksum != remote_source_checksum:
|
|
checksum_mismatch = True
|
|
checksum_reason = "source_checksum_differs"
|
|
elif source_checksum and not remote_source_checksum:
|
|
checksum_mismatch = True
|
|
checksum_reason = "remote_source_checksum_missing"
|
|
elif remote_source_checksums:
|
|
remote_unique_checksum = remote_source_checksums[0] if len(set(remote_source_checksums)) == 1 else ""
|
|
if source_checksum and remote_unique_checksum and source_checksum != remote_unique_checksum:
|
|
checksum_mismatch = True
|
|
checksum_reason = "same_title_checksum_differs"
|
|
|
|
metadata_gaps = []
|
|
for field in ("authority", "effective_date", "region"):
|
|
if not normalize_text(source.get(field)):
|
|
metadata_gaps.append(field)
|
|
tags = source.get("tags_json") or []
|
|
if not tags:
|
|
metadata_gaps.append("tags")
|
|
|
|
return {
|
|
**source,
|
|
"canonical_source_id": canonical_source_id,
|
|
"classification": classification,
|
|
"remote_source_ids_same_title": ";".join(remote_source_ids_same_title),
|
|
"remote_same_title_count": remote_same_title_count,
|
|
"remote_same_title_article_count": remote_same_title_article_count,
|
|
"remote_source_checksum": remote_source_checksum,
|
|
"remote_source_checksum_count": remote_source_checksum_count,
|
|
"checksum_mismatch": checksum_mismatch,
|
|
"checksum_mismatch_reason": checksum_reason,
|
|
"metadata_gaps": ";".join(metadata_gaps),
|
|
"metadata_gap_fields": metadata_gaps,
|
|
"action_suggestion": action_suggestion,
|
|
}
|
|
|
|
|
|
def classify_missing_source(source: dict[str, Any], remote_title_index: dict[str, list[dict[str, Any]]]) -> dict[str, Any]:
|
|
title = normalize_text(source.get("source_title"))
|
|
title_matches = remote_title_index.get(title, [])
|
|
if title_matches:
|
|
remote_article_count = sum(int(item.get("article_count") or 0) for item in title_matches)
|
|
remote_source_ids = [item.get("source_id") for item in title_matches]
|
|
return {
|
|
**source,
|
|
"classification": "covered_by_remote_same_title",
|
|
"remote_source_ids_same_title": ";".join(remote_source_ids),
|
|
"remote_same_title_count": len(title_matches),
|
|
"remote_same_title_article_count": remote_article_count,
|
|
"action_suggestion": "暂不重复导入；同标题内容已在运行库中覆盖，后续只需抽样核验。",
|
|
}
|
|
return {
|
|
**source,
|
|
"classification": "needs_incremental_review",
|
|
"remote_source_ids_same_title": "",
|
|
"remote_same_title_count": 0,
|
|
"remote_same_title_article_count": 0,
|
|
"action_suggestion": "需要进入增量补抓/补导入复核队列。",
|
|
}
|
|
|
|
|
|
def write_csv(path: Path, rows: list[dict[str, Any]], fieldnames: list[str]) -> None:
|
|
with path.open("w", encoding="utf-8-sig", newline="") as file_obj:
|
|
writer = csv.DictWriter(file_obj, fieldnames=fieldnames, extrasaction="ignore")
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def markdown_table(rows: list[list[Any]]) -> str:
|
|
if not rows:
|
|
return ""
|
|
lines = [
|
|
"| " + " | ".join(str(item) for item in rows[0]) + " |",
|
|
"| " + " | ".join("---" for _ in rows[0]) + " |",
|
|
]
|
|
for row in rows[1:]:
|
|
lines.append("| " + " | ".join(str(item).replace("\n", " ") for item in row) + " |")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def build_diff_report(
|
|
*,
|
|
laws_backup_dir: Path,
|
|
remote_source_json: Path,
|
|
prepare_summary_json: Path | None,
|
|
import_summary_json: Path | None,
|
|
output_dir: Path,
|
|
tenant_id: str,
|
|
) -> dict[str, Any]:
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
backup_scan = scan_backup_sources(laws_backup_dir)
|
|
remote_payload = read_json(remote_source_json)
|
|
prepare_summary = read_json(prepare_summary_json)
|
|
import_summary = read_json(import_summary_json)
|
|
prepared_inventory = scan_prepared_source_inventory(prepare_summary_json)
|
|
|
|
backup_archive_sources_by_id = backup_scan["sources"]
|
|
prepared_sources_by_id = prepared_inventory["sources"]
|
|
remote_sources = remote_payload.get("sources") or []
|
|
remote_sources_by_id = {
|
|
normalize_source_id(item.get("source_id")): item
|
|
for item in remote_sources
|
|
if normalize_source_id(item.get("source_id"))
|
|
}
|
|
backup_sources_by_canonical_id = {}
|
|
for source_id, source in backup_archive_sources_by_id.items():
|
|
canonical_source_id = normalize_source_id(source_id)
|
|
if not canonical_source_id:
|
|
continue
|
|
merged_source = dict(source)
|
|
prepared_source = prepared_sources_by_id.get(canonical_source_id) or {}
|
|
merged_source.update(prepared_source)
|
|
merged_source["source_id"] = source_id
|
|
merged_source["canonical_source_id"] = canonical_source_id
|
|
backup_sources_by_canonical_id[canonical_source_id] = merged_source
|
|
remote_title_index = build_title_index(remote_sources)
|
|
|
|
backup_ids = set(backup_sources_by_canonical_id)
|
|
remote_ids = set(remote_sources_by_id)
|
|
comparison_rows = [
|
|
build_source_comparison(backup_sources_by_canonical_id[source_id], remote_sources_by_id, remote_title_index)
|
|
for source_id in sorted(backup_ids)
|
|
]
|
|
imported_sources = [
|
|
item for item in comparison_rows if item["classification"] in {"imported_exact_source_id", "covered_by_remote_same_title"}
|
|
]
|
|
missing_sources = [item for item in comparison_rows if item["classification"] == "needs_incremental_review"]
|
|
checksum_mismatch_sources = [item for item in comparison_rows if item["checksum_mismatch"]]
|
|
metadata_gap_sources = [item for item in comparison_rows if item["metadata_gaps"]]
|
|
extra_remote_sources = [
|
|
remote_sources_by_id[source_id]
|
|
for source_id in sorted(remote_ids - backup_ids)
|
|
]
|
|
|
|
classification_counts = Counter(item["classification"] for item in comparison_rows)
|
|
source_type_counts = Counter(normalize_text(item.get("source_type") or "未知") for item in missing_sources)
|
|
remote_duplicate_titles = [
|
|
{
|
|
"source_title": title,
|
|
"remote_source_count": len(items),
|
|
"remote_source_ids": ";".join(str(item.get("source_id")) for item in items),
|
|
"remote_article_count": sum(int(item.get("article_count") or 0) for item in items),
|
|
}
|
|
for title, items in remote_title_index.items()
|
|
if len(items) > 1
|
|
]
|
|
backup_title_index = build_title_index(list(backup_sources_by_canonical_id.values()))
|
|
backup_duplicate_titles = [
|
|
{
|
|
"source_title": title,
|
|
"backup_source_count": len(items),
|
|
"backup_source_ids": ";".join(str(item.get("source_id")) for item in items),
|
|
}
|
|
for title, items in backup_title_index.items()
|
|
if len(items) > 1
|
|
]
|
|
|
|
needs_incremental_review = missing_sources
|
|
covered_by_title = [item for item in imported_sources if item["classification"] == "covered_by_remote_same_title"]
|
|
imported_exact = [item for item in imported_sources if item["classification"] == "imported_exact_source_id"]
|
|
|
|
status = "PASS" if not needs_incremental_review else "WARN"
|
|
summary = {
|
|
"status": status,
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"tenant_id": tenant_id,
|
|
"laws_backup_dir": str(laws_backup_dir),
|
|
"archive": backup_scan["archive"],
|
|
"remote_source_json": str(remote_source_json),
|
|
"prepare_summary_json": str(prepare_summary_json) if prepare_summary_json else None,
|
|
"import_summary_json": str(import_summary_json) if import_summary_json else None,
|
|
"counts": {
|
|
"backup_sources": len(backup_sources_by_canonical_id),
|
|
"backup_fulltext_documents": backup_scan["fulltext_documents"],
|
|
"remote_sources": len(remote_sources_by_id),
|
|
"remote_legal_articles": int(remote_payload.get("legal_article_count") or 0),
|
|
"imported_sources": len(imported_sources),
|
|
"missing_sources_in_remote": len(missing_sources),
|
|
"covered_by_remote_same_title": len(covered_by_title),
|
|
"needs_incremental_review": len(needs_incremental_review),
|
|
"checksum_mismatch_sources": len(checksum_mismatch_sources),
|
|
"metadata_gap_sources": len(metadata_gap_sources),
|
|
"extra_remote_sources": len(extra_remote_sources),
|
|
"backup_duplicate_title_groups": len(backup_duplicate_titles),
|
|
"remote_duplicate_title_groups": len(remote_duplicate_titles),
|
|
"invalid_or_duplicate_metadata": len(backup_scan["duplicate_or_invalid_metadata"]),
|
|
"prepared_article_rows": int(prepare_summary.get("rows") or 0),
|
|
"import_inserted": int(import_summary.get("inserted") or 0),
|
|
"import_skipped_existing_duplicates": int(import_summary.get("skipped_existing_duplicates") or 0),
|
|
},
|
|
"classification_counts": dict(classification_counts),
|
|
"missing_source_type_counts": dict(source_type_counts.most_common()),
|
|
"imported_sources": imported_sources,
|
|
"missing_sources": missing_sources,
|
|
"checksum_mismatch_sources": checksum_mismatch_sources,
|
|
"metadata_gap_sources": metadata_gap_sources,
|
|
"extra_remote_sources": extra_remote_sources,
|
|
"backup_duplicate_titles": backup_duplicate_titles[:200],
|
|
"remote_duplicate_titles": remote_duplicate_titles[:200],
|
|
"invalid_or_duplicate_metadata": backup_scan["duplicate_or_invalid_metadata"][:200],
|
|
"conclusion": (
|
|
"本轮没有发现必须立即补导入的来源；缺失 source_id 均可被远程同标题来源覆盖。"
|
|
if not needs_incremental_review
|
|
else "存在远程同标题无法覆盖的来源，需要进入增量补抓/补导入复核队列。"
|
|
),
|
|
}
|
|
|
|
json_path = output_dir / "legal-source-diff-report.json"
|
|
md_path = output_dir / "legal-source-diff-report.md"
|
|
missing_csv = output_dir / "missing-sources.csv"
|
|
extra_csv = output_dir / "extra-remote-sources.csv"
|
|
covered_csv = output_dir / "covered-by-remote-title.csv"
|
|
|
|
json_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
|
common_fields = [
|
|
"source_id",
|
|
"canonical_source_id",
|
|
"source_title",
|
|
"source_type",
|
|
"authority",
|
|
"effective_date",
|
|
"region",
|
|
"tags_json",
|
|
"source_checksum",
|
|
"remote_source_checksum",
|
|
"remote_source_checksum_count",
|
|
"checksum_mismatch",
|
|
"checksum_mismatch_reason",
|
|
"metadata_gaps",
|
|
"publish_date",
|
|
"classification",
|
|
"remote_source_ids_same_title",
|
|
"remote_same_title_count",
|
|
"remote_same_title_article_count",
|
|
"action_suggestion",
|
|
"metadata_path",
|
|
]
|
|
imported_csv = output_dir / "imported-sources.csv"
|
|
write_csv(missing_csv, missing_sources, common_fields)
|
|
write_csv(imported_csv, imported_sources, common_fields)
|
|
write_csv(covered_csv, covered_by_title, common_fields)
|
|
write_csv(output_dir / "checksum-mismatch-sources.csv", checksum_mismatch_sources, common_fields)
|
|
write_csv(output_dir / "metadata-gap-sources.csv", metadata_gap_sources, common_fields)
|
|
write_csv(
|
|
extra_csv,
|
|
extra_remote_sources,
|
|
["source_id", "source_title", "source_type", "authority", "effective_date", "article_count"],
|
|
)
|
|
write_markdown_report(md_path, summary, comparison_rows)
|
|
return summary
|
|
|
|
|
|
def write_markdown_report(path: Path, summary: dict[str, Any], comparison_rows: list[dict[str, Any]]) -> None:
|
|
counts = summary["counts"]
|
|
sample_rows = [["source_id", "标题", "分类", "建议"]]
|
|
for item in comparison_rows[:20]:
|
|
sample_rows.append(
|
|
[
|
|
item.get("source_id"),
|
|
item.get("source_title"),
|
|
item.get("classification"),
|
|
item.get("action_suggestion"),
|
|
]
|
|
)
|
|
|
|
lines = [
|
|
"# 法律库来源差异复核报告",
|
|
"",
|
|
f"- 状态：`{summary['status']}`",
|
|
f"- 生成时间：{summary['generated_at']}",
|
|
f"- 租户：`{summary['tenant_id']}`",
|
|
f"- 备份包：`{summary['archive']}`",
|
|
"- 说明：对比时已规范化远程 `source_id`，自动剥离 `FLK-` 前缀后再做差异判断。",
|
|
"",
|
|
"## 结论",
|
|
"",
|
|
f"- {summary['conclusion']}",
|
|
f"- 备份包来源数：`{counts['backup_sources']}`；远程运行库来源数：`{counts['remote_sources']}`。",
|
|
f"- 远程 `legal_articles` 条文数：`{counts['remote_legal_articles']}`。",
|
|
f"- 已入库来源：`{counts['imported_sources']}`。",
|
|
f"- 备份包有但远程 source_id 没有的来源：`{counts['missing_sources_in_remote']}`。",
|
|
f"- 其中同标题已覆盖：`{counts['covered_by_remote_same_title']}`；需要增量复核：`{counts['needs_incremental_review']}`。",
|
|
f"- checksum 不一致：`{counts['checksum_mismatch_sources']}`。",
|
|
f"- 元数据缺口：`{counts['metadata_gap_sources']}`。",
|
|
f"- 远程额外来源：`{counts['extra_remote_sources']}`。",
|
|
"",
|
|
"## 关键统计",
|
|
"",
|
|
markdown_table(
|
|
[
|
|
["指标", "数量"],
|
|
["备份来源", counts["backup_sources"]],
|
|
["备份 fulltext 文档", counts["backup_fulltext_documents"]],
|
|
["远程来源", counts["remote_sources"]],
|
|
["远程 legal_articles", counts["remote_legal_articles"]],
|
|
["已入库来源", counts["imported_sources"]],
|
|
["缺失 source_id", counts["missing_sources_in_remote"]],
|
|
["同标题覆盖", counts["covered_by_remote_same_title"]],
|
|
["需增量复核", counts["needs_incremental_review"]],
|
|
["checksum 不一致", counts["checksum_mismatch_sources"]],
|
|
["元数据缺口", counts["metadata_gap_sources"]],
|
|
["远程额外来源", counts["extra_remote_sources"]],
|
|
["备份同名标题组", counts["backup_duplicate_title_groups"]],
|
|
["远程同名标题组", counts["remote_duplicate_title_groups"]],
|
|
]
|
|
),
|
|
"",
|
|
"## 来源样例",
|
|
"",
|
|
markdown_table(sample_rows) if len(sample_rows) > 1 else "无。",
|
|
"",
|
|
"## 输出文件",
|
|
"",
|
|
"- `legal-source-diff-report.json`",
|
|
"- `imported-sources.csv`",
|
|
"- `missing-sources.csv`",
|
|
"- `checksum-mismatch-sources.csv`",
|
|
"- `metadata-gap-sources.csv`",
|
|
"- `covered-by-remote-title.csv`",
|
|
"- `extra-remote-sources.csv`",
|
|
"",
|
|
]
|
|
path.write_text("\n".join(lines), encoding="utf-8")
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description="Compare cn-laws-download source metadata with remote legal_articles sources.")
|
|
parser.add_argument("--laws-backup-dir", type=Path, required=True)
|
|
parser.add_argument("--remote-source-json", type=Path, required=True)
|
|
parser.add_argument("--prepare-summary-json", type=Path)
|
|
parser.add_argument("--import-summary-json", type=Path)
|
|
parser.add_argument("--output-dir", type=Path, required=True)
|
|
parser.add_argument("--tenant-id", default="tenant-demo-001")
|
|
args = parser.parse_args()
|
|
|
|
summary = build_diff_report(
|
|
laws_backup_dir=args.laws_backup_dir,
|
|
remote_source_json=args.remote_source_json,
|
|
prepare_summary_json=args.prepare_summary_json,
|
|
import_summary_json=args.import_summary_json,
|
|
output_dir=args.output_dir,
|
|
tenant_id=args.tenant_id,
|
|
)
|
|
marker = "PASS" if summary["status"] == "PASS" else "WARN"
|
|
print(f"LEGAL_SOURCE_DIFF_REPORT_{marker} {args.output_dir}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|