chore: add asset inventory gap report
This commit is contained in:
@@ -0,0 +1,466 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
EXPECTED_RUNTIME_TEMPLATES = [
|
||||
{"template_code": "TPL-PURCHASE-GOODS-001", "name": "货物采购合同模板", "document_type_code": "procurement_goods"},
|
||||
{"template_code": "TPL-SALES-GOODS-001", "name": "销售合同模板", "document_type_code": "sales_goods"},
|
||||
{"template_code": "TPL-SERVICE-GENERAL-001", "name": "通用服务合同模板", "document_type_code": "service_general"},
|
||||
{"template_code": "TPL-TECH-DEVELOPMENT-001", "name": "软件技术开发合同模板", "document_type_code": "software_development"},
|
||||
{"template_code": "TPL-SOFTWARE-SAAS-001", "name": "软件即服务 SaaS 合同模板", "document_type_code": "software_saas"},
|
||||
{"template_code": "TPL-NDA-COMMERCIAL-001", "name": "商业保密协议模板", "document_type_code": "nda_commercial"},
|
||||
{"template_code": "TPL-LABOR-CONTRACT-001", "name": "劳动合同模板", "document_type_code": "labor_contract"},
|
||||
{"template_code": "TPL-LEASE-HOUSE-001", "name": "房屋租赁合同模板", "document_type_code": "lease_house"},
|
||||
{"template_code": "TPL-COMMISSION-SERVICE-001", "name": "委托服务合同模板", "document_type_code": "commission_service"},
|
||||
{"template_code": "TPL-COOPERATION-FRAMEWORK-001", "name": "合作框架协议模板", "document_type_code": "cooperation_framework"},
|
||||
]
|
||||
|
||||
REQUIRED_PROMPT_TYPES = [
|
||||
"contract_generation",
|
||||
"contract_review",
|
||||
"legal_chat",
|
||||
"clause_recommendation",
|
||||
"wording_rewrite",
|
||||
"regulation_impact",
|
||||
]
|
||||
|
||||
INVENTORY_FILES = {
|
||||
"legal_sources": "inventories/legal-source-inventory.csv",
|
||||
"knowledge_cards": "inventories/legal-knowledge-card-inventory.csv",
|
||||
"contract_templates": "inventories/contract-template-inventory.csv",
|
||||
"clause_library": "inventories/clause-library-inventory.csv",
|
||||
"review_rules": "inventories/review-risk-rule-inventory.csv",
|
||||
"generation_golden_samples": "inventories/generation-golden-sample-inventory.csv",
|
||||
"review_golden_samples": "inventories/review-golden-sample-inventory.csv",
|
||||
"kb_qa_golden_samples": "inventories/kb-qa-golden-sample-inventory.csv",
|
||||
}
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict[str, Any]:
|
||||
if not path or not path.exists():
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8-sig"))
|
||||
|
||||
|
||||
def read_csv_rows(path: Path) -> list[dict[str, str]]:
|
||||
if not path.exists():
|
||||
return []
|
||||
with path.open("r", encoding="utf-8-sig", newline="") as file_obj:
|
||||
return list(csv.DictReader(file_obj))
|
||||
|
||||
|
||||
def latest_file(directory: Path, pattern: str) -> Path | None:
|
||||
files = sorted(directory.glob(pattern), key=lambda item: item.stat().st_mtime, reverse=True)
|
||||
return files[0] if files else None
|
||||
|
||||
|
||||
def find_laws_archive(laws_backup_dir: Path) -> Path | None:
|
||||
if laws_backup_dir.is_file() and laws_backup_dir.name.endswith(".tar.gz"):
|
||||
return laws_backup_dir
|
||||
return latest_file(laws_backup_dir, "*.tar.gz")
|
||||
|
||||
|
||||
def scan_laws_backup(laws_backup_dir: Path, max_metadata_scan: int = 20000) -> dict[str, Any]:
|
||||
archive = find_laws_archive(laws_backup_dir)
|
||||
manifest_path = latest_file(laws_backup_dir, "*.manifest.json") if laws_backup_dir.is_dir() else None
|
||||
listing_summary_path = latest_file(laws_backup_dir, "*.tar_listing_summary.json") if laws_backup_dir.is_dir() else None
|
||||
result: dict[str, Any] = {
|
||||
"laws_backup_dir": str(laws_backup_dir),
|
||||
"archive": str(archive) if archive else None,
|
||||
"manifest": read_json(manifest_path) if manifest_path else {},
|
||||
"tar_listing_summary": read_json(listing_summary_path) if listing_summary_path else {},
|
||||
"metadata_documents": 0,
|
||||
"fulltext_documents": 0,
|
||||
"unique_bbbs": 0,
|
||||
"missing_title": 0,
|
||||
"missing_authority": 0,
|
||||
"missing_effective_date": 0,
|
||||
"source_type_counts": {},
|
||||
"top_authorities": [],
|
||||
"sample_sources": [],
|
||||
}
|
||||
if not archive or not archive.exists():
|
||||
result["error"] = "archive_not_found"
|
||||
return result
|
||||
|
||||
source_types: Counter[str] = Counter()
|
||||
authorities: Counter[str] = Counter()
|
||||
bbbs_values: set[str] = set()
|
||||
samples: list[dict[str, Any]] = []
|
||||
|
||||
with tarfile.open(archive, "r:gz") as tar:
|
||||
for member in tar:
|
||||
if member.isfile() and member.name.endswith("/fulltext.txt"):
|
||||
result["fulltext_documents"] += 1
|
||||
continue
|
||||
if not (member.isfile() and member.name.endswith("/metadata.json")):
|
||||
continue
|
||||
result["metadata_documents"] += 1
|
||||
if result["metadata_documents"] > max_metadata_scan:
|
||||
continue
|
||||
file_obj = tar.extractfile(member)
|
||||
if not file_obj:
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(file_obj.read().decode("utf-8"))
|
||||
except Exception:
|
||||
continue
|
||||
row = payload.get("row") or {}
|
||||
detail_data = ((payload.get("detail") or {}).get("data") or {})
|
||||
bbbs = str(payload.get("bbbs") or row.get("bbbs") or detail_data.get("bbbs") or "").strip()
|
||||
title = str(row.get("title") or detail_data.get("title") or "").strip()
|
||||
source_type = str(row.get("flxz") or detail_data.get("flxz") or "未知").strip() or "未知"
|
||||
authority = str(row.get("zdjgName") or detail_data.get("zdjgName") or "").strip()
|
||||
effective_date = str(row.get("sxrq") or detail_data.get("sxrq") or "").strip()
|
||||
if bbbs:
|
||||
bbbs_values.add(bbbs)
|
||||
if not title:
|
||||
result["missing_title"] += 1
|
||||
if not authority:
|
||||
result["missing_authority"] += 1
|
||||
if not effective_date:
|
||||
result["missing_effective_date"] += 1
|
||||
source_types[source_type] += 1
|
||||
if authority:
|
||||
authorities[authority] += 1
|
||||
if len(samples) < 10:
|
||||
samples.append(
|
||||
{
|
||||
"bbbs": bbbs,
|
||||
"title": title,
|
||||
"source_type": source_type,
|
||||
"authority": authority,
|
||||
"effective_date": effective_date,
|
||||
}
|
||||
)
|
||||
|
||||
result["unique_bbbs"] = len(bbbs_values)
|
||||
result["source_type_counts"] = dict(source_types.most_common())
|
||||
result["top_authorities"] = [{"authority": key, "count": value} for key, value in authorities.most_common(10)]
|
||||
result["sample_sources"] = samples
|
||||
return result
|
||||
|
||||
|
||||
def count_glob(root: Path, pattern: str) -> int:
|
||||
return len(list(root.glob(pattern)))
|
||||
|
||||
|
||||
def scan_prompt_suite(repo_root: Path) -> dict[str, Any]:
|
||||
manifest_path = repo_root / "project-docs/phase15-prompt-golden-samples/prompt-golden-suite-manifest.json"
|
||||
manifest = read_json(manifest_path)
|
||||
suites = []
|
||||
for suite in manifest.get("required_prompt_suites", []) + manifest.get("optional_prompt_suites", []):
|
||||
source = suite.get("source")
|
||||
sample_count = 0
|
||||
if source:
|
||||
source_path = repo_root / str(source)
|
||||
if "*" in str(source):
|
||||
sample_count = count_glob(repo_root, str(source))
|
||||
elif source_path.suffix.lower() == ".csv":
|
||||
sample_count = len(read_csv_rows(source_path))
|
||||
elif source_path.suffix.lower() == ".json":
|
||||
payload = read_json(source_path)
|
||||
sample_count = len(payload.get("samples") or [])
|
||||
elif source_path.exists():
|
||||
sample_count = 1
|
||||
suites.append(
|
||||
{
|
||||
"prompt_type": suite.get("prompt_type"),
|
||||
"sample_family": suite.get("sample_family"),
|
||||
"min_samples": suite.get("min_samples", 0),
|
||||
"source": source,
|
||||
"sample_count": sample_count,
|
||||
"meets_min_samples": sample_count >= int(suite.get("min_samples") or 0),
|
||||
}
|
||||
)
|
||||
return {"manifest_path": str(manifest_path), "suites": suites}
|
||||
|
||||
|
||||
def scan_local_assets(repo_root: Path, library_root: Path) -> dict[str, Any]:
|
||||
inventories = {}
|
||||
for key, relative in INVENTORY_FILES.items():
|
||||
rows = read_csv_rows(library_root / relative)
|
||||
inventories[key] = {
|
||||
"path": str(library_root / relative),
|
||||
"count": len(rows),
|
||||
"ids": [
|
||||
row.get("template_id")
|
||||
or row.get("source_id")
|
||||
or row.get("knowledge_card_id")
|
||||
or row.get("clause_id")
|
||||
or row.get("rule_id")
|
||||
or row.get("sample_id")
|
||||
for row in rows
|
||||
],
|
||||
}
|
||||
|
||||
return {
|
||||
"library_root": str(library_root),
|
||||
"inventories": inventories,
|
||||
"file_counts": {
|
||||
"contract_template_markdown": count_glob(library_root, "contract-templates/**/*.md"),
|
||||
"contract_template_variable_schema": count_glob(library_root, "contract-templates/**/*-variables.json"),
|
||||
"clause_markdown": count_glob(library_root, "clause-library/**/*.md"),
|
||||
"review_rule_markdown": count_glob(library_root, "review-rules/**/*.md"),
|
||||
"generation_sample_expected": count_glob(library_root, "qa-samples/generation/*-expected.md"),
|
||||
"review_sample_expected": count_glob(library_root, "qa-samples/review/*-expected.md"),
|
||||
},
|
||||
"prompt_suite": scan_prompt_suite(repo_root),
|
||||
}
|
||||
|
||||
|
||||
def build_gap_summary(
|
||||
laws_backup: dict[str, Any],
|
||||
local_assets: dict[str, Any],
|
||||
remote_db: dict[str, Any],
|
||||
prepare_summary: dict[str, Any],
|
||||
import_summary: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
local_template_ids = set(local_assets["inventories"]["contract_templates"]["ids"])
|
||||
remote_templates = remote_db.get("templates") or []
|
||||
remote_template_codes = {str(item.get("template_code") or "").strip() for item in remote_templates}
|
||||
expected_template_codes = {item["template_code"] for item in EXPECTED_RUNTIME_TEMPLATES}
|
||||
missing_local_to_remote = sorted(code for code in local_template_ids if code and code not in remote_template_codes)
|
||||
missing_expected_runtime = sorted(code for code in expected_template_codes if code not in remote_template_codes)
|
||||
|
||||
prompt_assets_by_type = {
|
||||
str(item.get("prompt_type")): int(item.get("count") or 0)
|
||||
for item in (remote_db.get("prompt_assets_by_type") or [])
|
||||
}
|
||||
missing_prompt_types = [prompt_type for prompt_type in REQUIRED_PROMPT_TYPES if prompt_assets_by_type.get(prompt_type, 0) == 0]
|
||||
|
||||
prepared_rows = int(prepare_summary.get("rows") or 0)
|
||||
inserted = int(import_summary.get("inserted") or 0)
|
||||
skipped_existing_duplicates = int(import_summary.get("skipped_existing_duplicates") or 0)
|
||||
remote_legal_articles = int(remote_db.get("legal_articles") or 0)
|
||||
legal_record_gap = max(prepared_rows - skipped_existing_duplicates - remote_legal_articles, 0)
|
||||
source_document_count = int(prepare_summary.get("source_documents") or laws_backup.get("metadata_documents") or 0)
|
||||
remote_legal_sources = int(remote_db.get("legal_sources") or 0)
|
||||
source_count_delta = max(source_document_count - remote_legal_sources, 0)
|
||||
|
||||
return {
|
||||
"legal_library": {
|
||||
"source_documents_in_backup": source_document_count,
|
||||
"prepared_article_rows": prepared_rows,
|
||||
"import_inserted": inserted,
|
||||
"import_skipped_existing_duplicates": skipped_existing_duplicates,
|
||||
"remote_legal_articles": remote_legal_articles,
|
||||
"remote_legal_sources": remote_legal_sources,
|
||||
"remote_indexed": int(remote_db.get("legal_articles_indexed") or 0),
|
||||
"remote_published": int(remote_db.get("legal_articles_published") or 0),
|
||||
"legal_record_gap_after_known_import": legal_record_gap,
|
||||
"source_count_delta_needs_review": source_count_delta,
|
||||
"vector_knowledge_documents": int(remote_db.get("knowledge_base_documents") or 0),
|
||||
"recommendation": (
|
||||
"同一备份包的 legal_articles 运行表导入已基本闭环；下一步不是重复全量导入，"
|
||||
"而是补可复跑增量抓取、source_count_delta 复核、以及分批向量化任务。"
|
||||
),
|
||||
},
|
||||
"templates": {
|
||||
"local_inventory_count": local_assets["inventories"]["contract_templates"]["count"],
|
||||
"remote_template_count": int(remote_db.get("contract_templates") or 0),
|
||||
"remote_template_versions": int(remote_db.get("contract_template_versions") or 0),
|
||||
"remote_templates_published": int(remote_db.get("templates_published") or 0),
|
||||
"missing_local_templates_in_remote": missing_local_to_remote,
|
||||
"missing_expected_runtime_templates": missing_expected_runtime,
|
||||
"remote_templates": remote_templates,
|
||||
},
|
||||
"prompts": {
|
||||
"required_prompt_types": REQUIRED_PROMPT_TYPES,
|
||||
"remote_prompt_assets": int(remote_db.get("prompt_assets") or 0),
|
||||
"remote_prompt_versions": int(remote_db.get("prompt_versions") or 0),
|
||||
"remote_prompt_eval_runs": int(remote_db.get("prompt_eval_runs") or 0),
|
||||
"missing_required_prompt_types": missing_prompt_types,
|
||||
"golden_suites": local_assets["prompt_suite"]["suites"],
|
||||
},
|
||||
"rules_and_clauses": {
|
||||
"local_clause_inventory_count": local_assets["inventories"]["clause_library"]["count"],
|
||||
"local_review_rule_inventory_count": local_assets["inventories"]["review_rules"]["count"],
|
||||
"remote_review_rules": int(remote_db.get("review_rules") or 0),
|
||||
"remote_review_rule_versions": int(remote_db.get("review_rule_versions") or 0),
|
||||
"remote_review_rules_published": int(remote_db.get("review_rules_published") or 0),
|
||||
},
|
||||
"priority_order": [
|
||||
"先补 Prompt 资产入库和发布，因为远程 prompt_assets=0，运行链路目前没有真正吃到 Prompt 治理资产。",
|
||||
"再补 Phase13 模板、规则、标准条款正式吸收，因为远程模板只有少量，review_rules=0。",
|
||||
"然后做法律库增量抓取/复核，不重复导入已完成的 690125 条。",
|
||||
"最后做全量法规向量化批处理，按法律、行政法规、司法解释、地方法规分批跑。",
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def markdown_table(rows: list[list[Any]]) -> str:
|
||||
if not rows:
|
||||
return ""
|
||||
header = rows[0]
|
||||
lines = [
|
||||
"| " + " | ".join(str(item) for item in header) + " |",
|
||||
"| " + " | ".join("---" for _ in header) + " |",
|
||||
]
|
||||
for row in rows[1:]:
|
||||
lines.append("| " + " | ".join(str(item).replace("\n", " ") for item in row) + " |")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_markdown_report(path: Path, summary: dict[str, Any]) -> None:
|
||||
gap = summary["gap_summary"]
|
||||
legal = gap["legal_library"]
|
||||
templates = gap["templates"]
|
||||
prompts = gap["prompts"]
|
||||
rules = gap["rules_and_clauses"]
|
||||
|
||||
prompt_rows = [["Prompt 类型", "黄金样本数", "最低要求", "样本达标", "远程资产数"]]
|
||||
remote_by_type = {
|
||||
str(item.get("prompt_type")): int(item.get("count") or 0)
|
||||
for item in (summary.get("remote_db") or {}).get("prompt_assets_by_type", [])
|
||||
}
|
||||
for suite in prompts["golden_suites"]:
|
||||
prompt_type = suite["prompt_type"]
|
||||
prompt_rows.append(
|
||||
[
|
||||
prompt_type,
|
||||
suite["sample_count"],
|
||||
suite["min_samples"],
|
||||
"是" if suite["meets_min_samples"] else "否",
|
||||
remote_by_type.get(prompt_type, 0),
|
||||
]
|
||||
)
|
||||
|
||||
template_rows = [["模板编号", "名称", "当前状态"]]
|
||||
remote_template_codes = {str(item.get("template_code") or "").strip() for item in templates["remote_templates"]}
|
||||
for item in EXPECTED_RUNTIME_TEMPLATES:
|
||||
status = "远程已存在" if item["template_code"] in remote_template_codes else "远程缺失"
|
||||
template_rows.append([item["template_code"], item["name"], status])
|
||||
|
||||
lines = [
|
||||
"# 法律库、模板、Prompt 资产盘点与缺口报告",
|
||||
"",
|
||||
f"生成时间：{summary['generated_at']}",
|
||||
"",
|
||||
"## 一、结论",
|
||||
"",
|
||||
"- 法律库运行表已经吸收本地 `cn-laws-download` 备份包的主体数据：远程 `legal_articles` 为 "
|
||||
f"{legal['remote_legal_articles']} 条，已索引 {legal['remote_indexed']} 条，已发布 {legal['remote_published']} 条。",
|
||||
f"- 同一备份包准备记录 {legal['prepared_article_rows']} 条，导入时跳过重复 {legal['import_skipped_existing_duplicates']} 条，"
|
||||
f"按既有导入规则计算的记录缺口为 {legal['legal_record_gap_after_known_import']}。",
|
||||
f"- 远程模板只有 {templates['remote_template_count']} 个，其中正式发布模板 {templates['remote_templates_published']} 个；"
|
||||
f"本地 Phase13 模板清单有 {templates['local_inventory_count']} 个。",
|
||||
f"- 远程 Prompt 资产为 {prompts['remote_prompt_assets']}，Prompt 版本为 {prompts['remote_prompt_versions']}，"
|
||||
"这说明 Prompt 治理页面/表已经有了，但生产运行库还没有种入正式 Prompt。",
|
||||
f"- 远程审查规则为 {rules['remote_review_rules']}，本地 Phase13 规则清单有 {rules['local_review_rule_inventory_count']} 条。",
|
||||
"",
|
||||
"## 二、法律库盘点",
|
||||
"",
|
||||
markdown_table(
|
||||
[
|
||||
["项目", "数量"],
|
||||
["备份包 metadata.json", legal["source_documents_in_backup"]],
|
||||
["备份包生成待入库条文", legal["prepared_article_rows"]],
|
||||
["导入插入", legal["import_inserted"]],
|
||||
["导入重复跳过", legal["import_skipped_existing_duplicates"]],
|
||||
["远程 legal_articles", legal["remote_legal_articles"]],
|
||||
["远程不同法源", legal["remote_legal_sources"]],
|
||||
["source 文档数与远程法源数差异", legal["source_count_delta_needs_review"]],
|
||||
["知识库文档数", legal["vector_knowledge_documents"]],
|
||||
]
|
||||
),
|
||||
"",
|
||||
"判断：当前不应重复把这 17,235 份全文再导一次。应该先复核 source 文档数与远程法源数的差异，再做增量抓取和分批向量化。",
|
||||
"",
|
||||
"## 三、模板缺口",
|
||||
"",
|
||||
markdown_table(template_rows),
|
||||
"",
|
||||
"缺失的本地 Phase13 模板编号：",
|
||||
"",
|
||||
", ".join(templates["missing_local_templates_in_remote"]) or "无",
|
||||
"",
|
||||
"## 四、Prompt 缺口",
|
||||
"",
|
||||
markdown_table(prompt_rows),
|
||||
"",
|
||||
"缺失的必备 Prompt 类型：",
|
||||
"",
|
||||
", ".join(prompts["missing_required_prompt_types"]) or "无",
|
||||
"",
|
||||
"## 五、规则与标准条款缺口",
|
||||
"",
|
||||
markdown_table(
|
||||
[
|
||||
["项目", "本地清单", "远程运行表"],
|
||||
["标准条款", rules["local_clause_inventory_count"], "当前没有独立标准条款运行表，主要在文档/模板/推荐规则中使用"],
|
||||
["审查规则", rules["local_review_rule_inventory_count"], rules["remote_review_rules"]],
|
||||
["审查规则版本", "-", rules["remote_review_rule_versions"]],
|
||||
["已发布审查规则", "-", rules["remote_review_rules_published"]],
|
||||
]
|
||||
),
|
||||
"",
|
||||
"## 六、建议执行顺序",
|
||||
"",
|
||||
]
|
||||
for index, item in enumerate(gap["priority_order"], start=1):
|
||||
lines.append(f"{index}. {item}")
|
||||
lines.extend(
|
||||
[
|
||||
"",
|
||||
"## 七、下一刀建议",
|
||||
"",
|
||||
"先做“Prompt 资产正式种入 + 发布 + 黄金样本评测”闭环。原因是法律库主体已入库，而远程 Prompt 资产为 0，"
|
||||
"这会直接影响生成、审查、Chat、条款推荐、措辞优化和法规影响扫描的运行质量。",
|
||||
"",
|
||||
]
|
||||
)
|
||||
path.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Build legal/template/prompt asset inventory gap report.")
|
||||
parser.add_argument("--laws-backup-dir", type=Path, required=True)
|
||||
parser.add_argument("--repo-root", type=Path, default=Path.cwd())
|
||||
parser.add_argument("--library-root", type=Path)
|
||||
parser.add_argument("--remote-db-json", type=Path)
|
||||
parser.add_argument("--prepare-summary-json", type=Path)
|
||||
parser.add_argument("--import-summary-json", type=Path)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
repo_root = args.repo_root.resolve()
|
||||
library_root = (args.library_root or repo_root / "project-docs/phase13-legal-knowledge-template-library").resolve()
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
laws_backup = scan_laws_backup(args.laws_backup_dir)
|
||||
local_assets = scan_local_assets(repo_root, library_root)
|
||||
remote_db = read_json(args.remote_db_json) if args.remote_db_json else {}
|
||||
prepare_summary = read_json(args.prepare_summary_json) if args.prepare_summary_json else {}
|
||||
import_summary = read_json(args.import_summary_json) if args.import_summary_json else {}
|
||||
gap_summary = build_gap_summary(laws_backup, local_assets, remote_db, prepare_summary, import_summary)
|
||||
|
||||
summary = {
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"repo_root": str(repo_root),
|
||||
"laws_backup": laws_backup,
|
||||
"local_assets": local_assets,
|
||||
"remote_db": remote_db,
|
||||
"prepare_summary": prepare_summary,
|
||||
"import_summary": import_summary,
|
||||
"gap_summary": gap_summary,
|
||||
}
|
||||
json_path = args.output_dir / "asset-inventory-gap-report.json"
|
||||
md_path = args.output_dir / "asset-inventory-gap-report.md"
|
||||
json_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
|
||||
write_markdown_report(md_path, summary)
|
||||
print(f"ASSET_INVENTORY_GAP_REPORT_PASS {args.output_dir}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -4522,3 +4522,37 @@
|
||||
- 清理后烟测记录数：`remote_smoke_record_count_after_cleanup=0`。
|
||||
- 当前闭环结论：
|
||||
- 条款推荐行为漏斗已完成“本地开发验证 -> Git 提交推送 -> 3.90 部署 -> Alembic 真相源 -> 远程 HTTP -> 真实库动作更新烟测”的完整闭环。
|
||||
|
||||
## 2026-05-22 法律库、模板、Prompt 资产盘点与缺口报告
|
||||
|
||||
- 本轮目标：
|
||||
- 暂停继续抓取，先确认现有法律库、模板、Prompt、审查规则和知识库资产到底已经吸收到什么程度。
|
||||
- 输出可复跑盘点脚本和报告，作为后续“继续抓取/继续导入/补模板/补 Prompt”的决策依据。
|
||||
- 新增工具：
|
||||
- `backend/tools/asset_inventory_gap_report.py`
|
||||
- `scripts/run-asset-inventory-gap-report.ps1`
|
||||
- 脚本只读本地备份包、本地 Phase13/Phase15 清单和远程 PostgreSQL 真相源，不抓取、不删除、不导入。
|
||||
- 盘点输入：
|
||||
- 本地法律备份包：`E:\DB_Backups\192.168.3.90_20260519_114618\file_backups\cn-laws-download_20260519_155247`
|
||||
- 既有导入摘要：`project-docs/qa-artifacts/cn-laws-download-import/2026-05-20T09-53-00`
|
||||
- 远程真相源：`192.168.3.90` 上 `ai-contract-backend-restore` 容器连接的 PostgreSQL。
|
||||
- 报告产物：
|
||||
- `project-docs/qa-artifacts/asset-inventory-gap-report/2026-05-22T09-17-53/asset-inventory-gap-report.md`
|
||||
- `project-docs/qa-artifacts/asset-inventory-gap-report/2026-05-22T09-17-53/asset-inventory-gap-report.json`
|
||||
- `project-docs/qa-artifacts/asset-inventory-gap-report/2026-05-22T09-17-53/remote-db-snapshot.json`
|
||||
- 核心结论：
|
||||
- 法律库主体已经吸收进运行表：远程 `legal_articles=690125`，已索引 `690125`，已发布 `690125`。
|
||||
- 同一备份包准备入库条文 `691425`，导入时跳过重复 `1300`，按既有导入规则计算的法律条文记录缺口为 `0`。
|
||||
- 备份包 source 文档 `17235`，远程按 `legal_articles.source_id` 去重法源 `17219`，差异 `16`，需要后续做来源级复核，但不应重复全量导入。
|
||||
- 远程知识库文档 `10`，其中 `indexed/ready/completed` 口径为 `6`；这说明全文法律库虽然进了 `legal_articles`，但还不是全量向量知识库。
|
||||
- 本地 Phase13 模板清单 `10` 个，远程运行表只有 `4` 个，正式发布模板 `0` 个。
|
||||
- 缺失模板编号：`TPL-COMMISSION-SERVICE-001`、`TPL-COOPERATION-FRAMEWORK-001`、`TPL-LEASE-HOUSE-001`、`TPL-SALES-GOODS-001`、`TPL-SERVICE-GENERAL-001`、`TPL-SOFTWARE-SAAS-001`、`TPL-TECH-DEVELOPMENT-001`。
|
||||
- 远程 `prompt_assets=0`、`prompt_versions=0`、`prompt_eval_runs=0`；必备 Prompt 类型 `contract_generation`、`contract_review`、`legal_chat`、`clause_recommendation`、`wording_rewrite`、`regulation_impact` 都还没有正式种入运行库。
|
||||
- 远程 `review_rules=0`，本地 Phase13 审查规则清单有 `8` 条。
|
||||
- 当前判断：
|
||||
- 不建议马上对同一 `cn-laws-download` 包做重复全量导入。
|
||||
- 下一阶段真正应该补的是“Prompt 资产正式种入 + 发布 + 黄金样本评测”，然后补 Phase13 模板、审查规则、标准条款吸收，最后再做法律库增量抓取和分批向量化。
|
||||
- 验证：
|
||||
- `python -m compileall backend\tools\asset_inventory_gap_report.py`
|
||||
- `$env:SSHPASS='***'; powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\run-asset-inventory-gap-report.ps1 -UseSshpass`
|
||||
- 输出：`ASSET_INVENTORY_GAP_REPORT_PASS D:\ai-contract-platform\project-docs\qa-artifacts\asset-inventory-gap-report\2026-05-22T09-17-53`
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
param(
|
||||
[string]$LawsBackupDir = "E:\DB_Backups\192.168.3.90_20260519_114618\file_backups\cn-laws-download_20260519_155247",
|
||||
[string]$ArtifactRoot = "",
|
||||
[string]$RemoteHost = "192.168.3.90",
|
||||
[string]$RemoteUser = "root",
|
||||
[string]$BackendContainer = "ai-contract-backend-restore",
|
||||
[string]$SshKeyPath = "$HOME\.ssh\id_rsa",
|
||||
[switch]$UseSshpass,
|
||||
[string]$PrepareSummaryJson = "",
|
||||
[string]$ImportSummaryJson = ""
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Resolve-SshpassPath {
|
||||
$command = Get-Command sshpass.exe -ErrorAction SilentlyContinue
|
||||
if ($null -ne $command) {
|
||||
return $command.Source
|
||||
}
|
||||
|
||||
$candidates = @(
|
||||
(Join-Path $env:LOCALAPPDATA "Microsoft\WinGet\Packages\xhcoding.sshpass-win32_Microsoft.Winget.Source_8wekyb3d8bbwe\sshpass.exe")
|
||||
)
|
||||
foreach ($candidate in $candidates) {
|
||||
if (Test-Path $candidate) {
|
||||
return $candidate
|
||||
}
|
||||
}
|
||||
|
||||
return $null
|
||||
}
|
||||
|
||||
function Invoke-LoggedCommand {
|
||||
param(
|
||||
[string]$Name,
|
||||
[string]$FilePath,
|
||||
[string[]]$Arguments,
|
||||
[string]$LogDir
|
||||
)
|
||||
|
||||
$logPath = Join-Path $LogDir "$Name.log"
|
||||
$previousErrorActionPreference = $ErrorActionPreference
|
||||
$ErrorActionPreference = "Continue"
|
||||
try {
|
||||
$output = & $FilePath @Arguments 2>&1
|
||||
$exitCode = if ($LASTEXITCODE -is [int]) { $LASTEXITCODE } else { 0 }
|
||||
} finally {
|
||||
$ErrorActionPreference = $previousErrorActionPreference
|
||||
}
|
||||
|
||||
$outputText = ($output -join [Environment]::NewLine)
|
||||
$outputText | Set-Content -Encoding UTF8 -Path $logPath
|
||||
if ($exitCode -ne 0) {
|
||||
throw "$Name failed with exit code $exitCode. See $logPath"
|
||||
}
|
||||
return $outputText
|
||||
}
|
||||
|
||||
function Get-JsonFromOutput {
|
||||
param([string]$Text)
|
||||
|
||||
$start = $Text.IndexOf("{")
|
||||
if ($start -lt 0) {
|
||||
throw "Remote output did not contain a JSON object."
|
||||
}
|
||||
return $Text.Substring($start)
|
||||
}
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
||||
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
||||
throw "GSTACK_MISSING: gstack is required for AI-assisted work in this repo."
|
||||
}
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
||||
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\asset-inventory-gap-report"
|
||||
}
|
||||
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss"
|
||||
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
||||
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($PrepareSummaryJson)) {
|
||||
$PrepareSummaryJson = Join-Path $repoRoot "project-docs\qa-artifacts\cn-laws-download-import\2026-05-20T09-53-00\cn_laws_prepare_summary_20260520.json"
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($ImportSummaryJson)) {
|
||||
$ImportSummaryJson = Join-Path $repoRoot "project-docs\qa-artifacts\cn-laws-download-import\2026-05-20T09-53-00\cn_laws_import_summary_20260520.json"
|
||||
}
|
||||
|
||||
foreach ($requiredPath in @($LawsBackupDir, $PrepareSummaryJson, $ImportSummaryJson)) {
|
||||
if (-not (Test-Path $requiredPath)) {
|
||||
throw "Required input path does not exist: $requiredPath"
|
||||
}
|
||||
}
|
||||
|
||||
$authMode = ""
|
||||
$sshpassPath = $null
|
||||
if ($UseSshpass -or -not [string]::IsNullOrWhiteSpace($env:SSHPASS)) {
|
||||
$sshpassPath = Resolve-SshpassPath
|
||||
if ([string]::IsNullOrWhiteSpace($sshpassPath)) {
|
||||
throw "sshpass.exe not found. Install sshpass or use SSH key auth."
|
||||
}
|
||||
if ([string]::IsNullOrWhiteSpace($env:SSHPASS)) {
|
||||
throw "SSHPASS environment variable is empty."
|
||||
}
|
||||
$authMode = "sshpass"
|
||||
} elseif (Test-Path $SshKeyPath) {
|
||||
$authMode = "key"
|
||||
} else {
|
||||
throw "SSH key not found and SSHPASS is not configured."
|
||||
}
|
||||
|
||||
$remoteScript = @'
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from sqlalchemy import create_engine, inspect, text
|
||||
|
||||
|
||||
errors = []
|
||||
database_url = os.environ.get("DATABASE_URL")
|
||||
if not database_url:
|
||||
raise RuntimeError("DATABASE_URL is required in backend container")
|
||||
|
||||
engine = create_engine(database_url)
|
||||
inspector = inspect(engine)
|
||||
tables = set(inspector.get_table_names())
|
||||
columns_cache = {}
|
||||
|
||||
|
||||
def columns(table_name):
|
||||
if table_name not in tables:
|
||||
return set()
|
||||
if table_name not in columns_cache:
|
||||
columns_cache[table_name] = {item["name"] for item in inspector.get_columns(table_name)}
|
||||
return columns_cache[table_name]
|
||||
|
||||
|
||||
def has_column(table_name, column_name):
|
||||
return column_name in columns(table_name)
|
||||
|
||||
|
||||
def count_table(conn, table_name, where_clause=None):
|
||||
if table_name not in tables:
|
||||
return 0
|
||||
sql = f"select count(*) from {table_name}"
|
||||
if where_clause:
|
||||
sql += f" where {where_clause}"
|
||||
try:
|
||||
return int(conn.execute(text(sql)).scalar() or 0)
|
||||
except Exception as exc:
|
||||
errors.append({"table": table_name, "where": where_clause, "error": str(exc)})
|
||||
return 0
|
||||
|
||||
|
||||
def rows(conn, sql):
|
||||
try:
|
||||
return [dict(item._mapping) for item in conn.execute(text(sql)).fetchall()]
|
||||
except Exception as exc:
|
||||
errors.append({"sql": sql, "error": str(exc)})
|
||||
return []
|
||||
|
||||
|
||||
def published_count(conn, table_name):
|
||||
if table_name not in tables:
|
||||
return 0
|
||||
if has_column(table_name, "published_at"):
|
||||
return count_table(conn, table_name, "published_at is not null")
|
||||
if has_column(table_name, "status"):
|
||||
return count_table(conn, table_name, "status = 'published'")
|
||||
if has_column(table_name, "is_published"):
|
||||
return count_table(conn, table_name, "is_published = true")
|
||||
return 0
|
||||
|
||||
|
||||
with engine.connect() as conn:
|
||||
snapshot = {
|
||||
"tables_present": sorted(
|
||||
table for table in [
|
||||
"legal_articles",
|
||||
"legal_sources",
|
||||
"knowledge_bases",
|
||||
"knowledge_base_documents",
|
||||
"contract_templates",
|
||||
"contract_template_versions",
|
||||
"review_rules",
|
||||
"review_rule_versions",
|
||||
"prompt_assets",
|
||||
"prompt_versions",
|
||||
"prompt_eval_runs",
|
||||
]
|
||||
if table in tables
|
||||
),
|
||||
"legal_articles": count_table(conn, "legal_articles"),
|
||||
"legal_sources": count_table(conn, "legal_sources"),
|
||||
"knowledge_bases": count_table(conn, "knowledge_bases"),
|
||||
"knowledge_base_documents": count_table(conn, "knowledge_base_documents"),
|
||||
"knowledge_base_documents_ready": (
|
||||
count_table(conn, "knowledge_base_documents", "status in ('ready', 'indexed', 'completed')")
|
||||
if has_column("knowledge_base_documents", "status")
|
||||
else 0
|
||||
),
|
||||
"contract_templates": count_table(conn, "contract_templates"),
|
||||
"templates_enabled": 0,
|
||||
"templates_published": published_count(conn, "contract_templates"),
|
||||
"contract_template_versions": count_table(conn, "contract_template_versions"),
|
||||
"review_rules": count_table(conn, "review_rules"),
|
||||
"review_rules_active": (
|
||||
count_table(conn, "review_rules", "is_active = true")
|
||||
if has_column("review_rules", "is_active")
|
||||
else 0
|
||||
),
|
||||
"review_rules_published": published_count(conn, "review_rules"),
|
||||
"review_rule_versions": count_table(conn, "review_rule_versions"),
|
||||
"prompt_assets": count_table(conn, "prompt_assets"),
|
||||
"prompts_published": published_count(conn, "prompt_assets"),
|
||||
"prompt_versions": count_table(conn, "prompt_versions"),
|
||||
"prompt_eval_runs": count_table(conn, "prompt_eval_runs"),
|
||||
"errors": errors,
|
||||
}
|
||||
|
||||
if has_column("legal_articles", "index_status"):
|
||||
snapshot["legal_articles_indexed"] = count_table(
|
||||
conn,
|
||||
"legal_articles",
|
||||
"index_status in ('indexed', 'ready', 'completed')",
|
||||
)
|
||||
elif has_column("legal_articles", "indexed_at"):
|
||||
snapshot["legal_articles_indexed"] = count_table(conn, "legal_articles", "indexed_at is not null")
|
||||
else:
|
||||
snapshot["legal_articles_indexed"] = 0
|
||||
|
||||
if has_column("legal_articles", "published_status"):
|
||||
snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "published_status = 'published'")
|
||||
elif has_column("legal_articles", "status"):
|
||||
snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "status = 'published'")
|
||||
elif has_column("legal_articles", "published_at"):
|
||||
snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "published_at is not null")
|
||||
elif has_column("legal_articles", "is_published"):
|
||||
snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "is_published = true")
|
||||
else:
|
||||
snapshot["legal_articles_published"] = 0
|
||||
|
||||
if snapshot["legal_sources"] == 0 and has_column("legal_articles", "source_id"):
|
||||
source_count = rows(
|
||||
conn,
|
||||
"select count(distinct source_id) as count from legal_articles where source_id is not null and source_id <> ''",
|
||||
)
|
||||
snapshot["legal_sources"] = int(source_count[0]["count"] or 0) if source_count else 0
|
||||
|
||||
if "contract_templates" in tables:
|
||||
if has_column("contract_templates", "is_enabled"):
|
||||
snapshot["templates_enabled"] = count_table(conn, "contract_templates", "is_enabled = true")
|
||||
elif has_column("contract_templates", "enabled"):
|
||||
snapshot["templates_enabled"] = count_table(conn, "contract_templates", "enabled = true")
|
||||
else:
|
||||
snapshot["templates_enabled"] = count_table(conn, "contract_templates")
|
||||
|
||||
if "contract_templates" in tables:
|
||||
select_columns = [
|
||||
column
|
||||
for column in ["id", "template_code", "name", "document_type_code", "status", "published_at", "is_enabled"]
|
||||
if has_column("contract_templates", column)
|
||||
]
|
||||
snapshot["templates"] = rows(
|
||||
conn,
|
||||
f"select {', '.join(select_columns)} from contract_templates order by template_code nulls last, name limit 100",
|
||||
) if select_columns else []
|
||||
else:
|
||||
snapshot["templates"] = []
|
||||
|
||||
if has_column("prompt_assets", "prompt_type"):
|
||||
snapshot["prompt_assets_by_type"] = rows(
|
||||
conn,
|
||||
"select prompt_type, count(*) as count from prompt_assets group by prompt_type order by prompt_type",
|
||||
)
|
||||
else:
|
||||
snapshot["prompt_assets_by_type"] = []
|
||||
|
||||
if has_column("review_rules", "status"):
|
||||
snapshot["review_rules_by_status"] = rows(
|
||||
conn,
|
||||
"select status, count(*) as count from review_rules group by status order by status",
|
||||
)
|
||||
else:
|
||||
snapshot["review_rules_by_status"] = []
|
||||
|
||||
print(json.dumps(snapshot, ensure_ascii=False, default=str, indent=2))
|
||||
'@
|
||||
|
||||
$localRemoteScript = Join-Path $artifactDir "remote-db-snapshot.py"
|
||||
$remoteSnapshotPath = Join-Path $artifactDir "remote-db-snapshot.json"
|
||||
$remoteScript | Set-Content -Encoding UTF8 -Path $localRemoteScript
|
||||
|
||||
$target = "${RemoteUser}@${RemoteHost}"
|
||||
$remoteTempScript = "/tmp/asset_inventory_remote_snapshot.py"
|
||||
|
||||
if ($authMode -eq "sshpass") {
|
||||
Invoke-LoggedCommand -Name "scp-remote-db-snapshot-script" -FilePath $sshpassPath -Arguments @(
|
||||
"-e",
|
||||
"scp",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "ConnectTimeout=10",
|
||||
$localRemoteScript,
|
||||
"${target}:$remoteTempScript"
|
||||
) -LogDir $artifactDir | Out-Null
|
||||
$remoteOutput = Invoke-LoggedCommand -Name "remote-db-snapshot" -FilePath $sshpassPath -Arguments @(
|
||||
"-e",
|
||||
"ssh",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "ConnectTimeout=10",
|
||||
$target,
|
||||
"docker cp $remoteTempScript ${BackendContainer}:$remoteTempScript && docker exec $BackendContainer sh -lc 'cd /app && python $remoteTempScript'"
|
||||
) -LogDir $artifactDir
|
||||
} else {
|
||||
Invoke-LoggedCommand -Name "scp-remote-db-snapshot-script" -FilePath "scp" -Arguments @(
|
||||
"-i", $SshKeyPath,
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "ConnectTimeout=10",
|
||||
$localRemoteScript,
|
||||
"${target}:$remoteTempScript"
|
||||
) -LogDir $artifactDir | Out-Null
|
||||
$remoteOutput = Invoke-LoggedCommand -Name "remote-db-snapshot" -FilePath "ssh" -Arguments @(
|
||||
"-i", $SshKeyPath,
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "StrictHostKeyChecking=accept-new",
|
||||
"-o", "ConnectTimeout=10",
|
||||
$target,
|
||||
"docker cp $remoteTempScript ${BackendContainer}:$remoteTempScript && docker exec $BackendContainer sh -lc 'cd /app && python $remoteTempScript'"
|
||||
) -LogDir $artifactDir
|
||||
}
|
||||
|
||||
Get-JsonFromOutput $remoteOutput | Set-Content -Encoding UTF8 -Path $remoteSnapshotPath
|
||||
|
||||
$toolPath = Join-Path $repoRoot "backend\tools\asset_inventory_gap_report.py"
|
||||
$toolOutput = & python $toolPath `
|
||||
--laws-backup-dir $LawsBackupDir `
|
||||
--repo-root $repoRoot `
|
||||
--remote-db-json $remoteSnapshotPath `
|
||||
--prepare-summary-json $PrepareSummaryJson `
|
||||
--import-summary-json $ImportSummaryJson `
|
||||
--output-dir $artifactDir
|
||||
|
||||
$toolExitCode = if ($LASTEXITCODE -is [int]) { $LASTEXITCODE } else { 0 }
|
||||
($toolOutput -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path (Join-Path $artifactDir "asset-inventory-gap-report-tool.log")
|
||||
if ($toolExitCode -ne 0) {
|
||||
throw "asset_inventory_gap_report.py failed with exit code $toolExitCode."
|
||||
}
|
||||
|
||||
Write-Output "ASSET_INVENTORY_GAP_REPORT_PASS $artifactDir"
|
||||
Reference in New Issue
Block a user