diff --git a/backend/tools/asset_inventory_gap_report.py b/backend/tools/asset_inventory_gap_report.py new file mode 100644 index 00000000..a4c513bf --- /dev/null +++ b/backend/tools/asset_inventory_gap_report.py @@ -0,0 +1,466 @@ +from __future__ import annotations + +import argparse +import csv +import json +import os +import tarfile +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +EXPECTED_RUNTIME_TEMPLATES = [ + {"template_code": "TPL-PURCHASE-GOODS-001", "name": "货物采购合同模板", "document_type_code": "procurement_goods"}, + {"template_code": "TPL-SALES-GOODS-001", "name": "销售合同模板", "document_type_code": "sales_goods"}, + {"template_code": "TPL-SERVICE-GENERAL-001", "name": "通用服务合同模板", "document_type_code": "service_general"}, + {"template_code": "TPL-TECH-DEVELOPMENT-001", "name": "软件技术开发合同模板", "document_type_code": "software_development"}, + {"template_code": "TPL-SOFTWARE-SAAS-001", "name": "软件即服务 SaaS 合同模板", "document_type_code": "software_saas"}, + {"template_code": "TPL-NDA-COMMERCIAL-001", "name": "商业保密协议模板", "document_type_code": "nda_commercial"}, + {"template_code": "TPL-LABOR-CONTRACT-001", "name": "劳动合同模板", "document_type_code": "labor_contract"}, + {"template_code": "TPL-LEASE-HOUSE-001", "name": "房屋租赁合同模板", "document_type_code": "lease_house"}, + {"template_code": "TPL-COMMISSION-SERVICE-001", "name": "委托服务合同模板", "document_type_code": "commission_service"}, + {"template_code": "TPL-COOPERATION-FRAMEWORK-001", "name": "合作框架协议模板", "document_type_code": "cooperation_framework"}, +] + +REQUIRED_PROMPT_TYPES = [ + "contract_generation", + "contract_review", + "legal_chat", + "clause_recommendation", + "wording_rewrite", + "regulation_impact", +] + +INVENTORY_FILES = { + "legal_sources": "inventories/legal-source-inventory.csv", + "knowledge_cards": "inventories/legal-knowledge-card-inventory.csv", + "contract_templates": "inventories/contract-template-inventory.csv", + "clause_library": "inventories/clause-library-inventory.csv", + "review_rules": "inventories/review-risk-rule-inventory.csv", + "generation_golden_samples": "inventories/generation-golden-sample-inventory.csv", + "review_golden_samples": "inventories/review-golden-sample-inventory.csv", + "kb_qa_golden_samples": "inventories/kb-qa-golden-sample-inventory.csv", +} + + +def read_json(path: Path) -> dict[str, Any]: + if not path or not path.exists(): + return {} + return json.loads(path.read_text(encoding="utf-8-sig")) + + +def read_csv_rows(path: Path) -> list[dict[str, str]]: + if not path.exists(): + return [] + with path.open("r", encoding="utf-8-sig", newline="") as file_obj: + return list(csv.DictReader(file_obj)) + + +def latest_file(directory: Path, pattern: str) -> Path | None: + files = sorted(directory.glob(pattern), key=lambda item: item.stat().st_mtime, reverse=True) + return files[0] if files else None + + +def find_laws_archive(laws_backup_dir: Path) -> Path | None: + if laws_backup_dir.is_file() and laws_backup_dir.name.endswith(".tar.gz"): + return laws_backup_dir + return latest_file(laws_backup_dir, "*.tar.gz") + + +def scan_laws_backup(laws_backup_dir: Path, max_metadata_scan: int = 20000) -> dict[str, Any]: + archive = find_laws_archive(laws_backup_dir) + manifest_path = latest_file(laws_backup_dir, "*.manifest.json") if laws_backup_dir.is_dir() else None + listing_summary_path = latest_file(laws_backup_dir, "*.tar_listing_summary.json") if laws_backup_dir.is_dir() else None + result: dict[str, Any] = { + "laws_backup_dir": str(laws_backup_dir), + "archive": str(archive) if archive else None, + "manifest": read_json(manifest_path) if manifest_path else {}, + "tar_listing_summary": read_json(listing_summary_path) if listing_summary_path else {}, + "metadata_documents": 0, + "fulltext_documents": 0, + "unique_bbbs": 0, + "missing_title": 0, + "missing_authority": 0, + "missing_effective_date": 0, + "source_type_counts": {}, + "top_authorities": [], + "sample_sources": [], + } + if not archive or not archive.exists(): + result["error"] = "archive_not_found" + return result + + source_types: Counter[str] = Counter() + authorities: Counter[str] = Counter() + bbbs_values: set[str] = set() + samples: list[dict[str, Any]] = [] + + with tarfile.open(archive, "r:gz") as tar: + for member in tar: + if member.isfile() and member.name.endswith("/fulltext.txt"): + result["fulltext_documents"] += 1 + continue + if not (member.isfile() and member.name.endswith("/metadata.json")): + continue + result["metadata_documents"] += 1 + if result["metadata_documents"] > max_metadata_scan: + continue + file_obj = tar.extractfile(member) + if not file_obj: + continue + try: + payload = json.loads(file_obj.read().decode("utf-8")) + except Exception: + continue + row = payload.get("row") or {} + detail_data = ((payload.get("detail") or {}).get("data") or {}) + bbbs = str(payload.get("bbbs") or row.get("bbbs") or detail_data.get("bbbs") or "").strip() + title = str(row.get("title") or detail_data.get("title") or "").strip() + source_type = str(row.get("flxz") or detail_data.get("flxz") or "未知").strip() or "未知" + authority = str(row.get("zdjgName") or detail_data.get("zdjgName") or "").strip() + effective_date = str(row.get("sxrq") or detail_data.get("sxrq") or "").strip() + if bbbs: + bbbs_values.add(bbbs) + if not title: + result["missing_title"] += 1 + if not authority: + result["missing_authority"] += 1 + if not effective_date: + result["missing_effective_date"] += 1 + source_types[source_type] += 1 + if authority: + authorities[authority] += 1 + if len(samples) < 10: + samples.append( + { + "bbbs": bbbs, + "title": title, + "source_type": source_type, + "authority": authority, + "effective_date": effective_date, + } + ) + + result["unique_bbbs"] = len(bbbs_values) + result["source_type_counts"] = dict(source_types.most_common()) + result["top_authorities"] = [{"authority": key, "count": value} for key, value in authorities.most_common(10)] + result["sample_sources"] = samples + return result + + +def count_glob(root: Path, pattern: str) -> int: + return len(list(root.glob(pattern))) + + +def scan_prompt_suite(repo_root: Path) -> dict[str, Any]: + manifest_path = repo_root / "project-docs/phase15-prompt-golden-samples/prompt-golden-suite-manifest.json" + manifest = read_json(manifest_path) + suites = [] + for suite in manifest.get("required_prompt_suites", []) + manifest.get("optional_prompt_suites", []): + source = suite.get("source") + sample_count = 0 + if source: + source_path = repo_root / str(source) + if "*" in str(source): + sample_count = count_glob(repo_root, str(source)) + elif source_path.suffix.lower() == ".csv": + sample_count = len(read_csv_rows(source_path)) + elif source_path.suffix.lower() == ".json": + payload = read_json(source_path) + sample_count = len(payload.get("samples") or []) + elif source_path.exists(): + sample_count = 1 + suites.append( + { + "prompt_type": suite.get("prompt_type"), + "sample_family": suite.get("sample_family"), + "min_samples": suite.get("min_samples", 0), + "source": source, + "sample_count": sample_count, + "meets_min_samples": sample_count >= int(suite.get("min_samples") or 0), + } + ) + return {"manifest_path": str(manifest_path), "suites": suites} + + +def scan_local_assets(repo_root: Path, library_root: Path) -> dict[str, Any]: + inventories = {} + for key, relative in INVENTORY_FILES.items(): + rows = read_csv_rows(library_root / relative) + inventories[key] = { + "path": str(library_root / relative), + "count": len(rows), + "ids": [ + row.get("template_id") + or row.get("source_id") + or row.get("knowledge_card_id") + or row.get("clause_id") + or row.get("rule_id") + or row.get("sample_id") + for row in rows + ], + } + + return { + "library_root": str(library_root), + "inventories": inventories, + "file_counts": { + "contract_template_markdown": count_glob(library_root, "contract-templates/**/*.md"), + "contract_template_variable_schema": count_glob(library_root, "contract-templates/**/*-variables.json"), + "clause_markdown": count_glob(library_root, "clause-library/**/*.md"), + "review_rule_markdown": count_glob(library_root, "review-rules/**/*.md"), + "generation_sample_expected": count_glob(library_root, "qa-samples/generation/*-expected.md"), + "review_sample_expected": count_glob(library_root, "qa-samples/review/*-expected.md"), + }, + "prompt_suite": scan_prompt_suite(repo_root), + } + + +def build_gap_summary( + laws_backup: dict[str, Any], + local_assets: dict[str, Any], + remote_db: dict[str, Any], + prepare_summary: dict[str, Any], + import_summary: dict[str, Any], +) -> dict[str, Any]: + local_template_ids = set(local_assets["inventories"]["contract_templates"]["ids"]) + remote_templates = remote_db.get("templates") or [] + remote_template_codes = {str(item.get("template_code") or "").strip() for item in remote_templates} + expected_template_codes = {item["template_code"] for item in EXPECTED_RUNTIME_TEMPLATES} + missing_local_to_remote = sorted(code for code in local_template_ids if code and code not in remote_template_codes) + missing_expected_runtime = sorted(code for code in expected_template_codes if code not in remote_template_codes) + + prompt_assets_by_type = { + str(item.get("prompt_type")): int(item.get("count") or 0) + for item in (remote_db.get("prompt_assets_by_type") or []) + } + missing_prompt_types = [prompt_type for prompt_type in REQUIRED_PROMPT_TYPES if prompt_assets_by_type.get(prompt_type, 0) == 0] + + prepared_rows = int(prepare_summary.get("rows") or 0) + inserted = int(import_summary.get("inserted") or 0) + skipped_existing_duplicates = int(import_summary.get("skipped_existing_duplicates") or 0) + remote_legal_articles = int(remote_db.get("legal_articles") or 0) + legal_record_gap = max(prepared_rows - skipped_existing_duplicates - remote_legal_articles, 0) + source_document_count = int(prepare_summary.get("source_documents") or laws_backup.get("metadata_documents") or 0) + remote_legal_sources = int(remote_db.get("legal_sources") or 0) + source_count_delta = max(source_document_count - remote_legal_sources, 0) + + return { + "legal_library": { + "source_documents_in_backup": source_document_count, + "prepared_article_rows": prepared_rows, + "import_inserted": inserted, + "import_skipped_existing_duplicates": skipped_existing_duplicates, + "remote_legal_articles": remote_legal_articles, + "remote_legal_sources": remote_legal_sources, + "remote_indexed": int(remote_db.get("legal_articles_indexed") or 0), + "remote_published": int(remote_db.get("legal_articles_published") or 0), + "legal_record_gap_after_known_import": legal_record_gap, + "source_count_delta_needs_review": source_count_delta, + "vector_knowledge_documents": int(remote_db.get("knowledge_base_documents") or 0), + "recommendation": ( + "同一备份包的 legal_articles 运行表导入已基本闭环;下一步不是重复全量导入," + "而是补可复跑增量抓取、source_count_delta 复核、以及分批向量化任务。" + ), + }, + "templates": { + "local_inventory_count": local_assets["inventories"]["contract_templates"]["count"], + "remote_template_count": int(remote_db.get("contract_templates") or 0), + "remote_template_versions": int(remote_db.get("contract_template_versions") or 0), + "remote_templates_published": int(remote_db.get("templates_published") or 0), + "missing_local_templates_in_remote": missing_local_to_remote, + "missing_expected_runtime_templates": missing_expected_runtime, + "remote_templates": remote_templates, + }, + "prompts": { + "required_prompt_types": REQUIRED_PROMPT_TYPES, + "remote_prompt_assets": int(remote_db.get("prompt_assets") or 0), + "remote_prompt_versions": int(remote_db.get("prompt_versions") or 0), + "remote_prompt_eval_runs": int(remote_db.get("prompt_eval_runs") or 0), + "missing_required_prompt_types": missing_prompt_types, + "golden_suites": local_assets["prompt_suite"]["suites"], + }, + "rules_and_clauses": { + "local_clause_inventory_count": local_assets["inventories"]["clause_library"]["count"], + "local_review_rule_inventory_count": local_assets["inventories"]["review_rules"]["count"], + "remote_review_rules": int(remote_db.get("review_rules") or 0), + "remote_review_rule_versions": int(remote_db.get("review_rule_versions") or 0), + "remote_review_rules_published": int(remote_db.get("review_rules_published") or 0), + }, + "priority_order": [ + "先补 Prompt 资产入库和发布,因为远程 prompt_assets=0,运行链路目前没有真正吃到 Prompt 治理资产。", + "再补 Phase13 模板、规则、标准条款正式吸收,因为远程模板只有少量,review_rules=0。", + "然后做法律库增量抓取/复核,不重复导入已完成的 690125 条。", + "最后做全量法规向量化批处理,按法律、行政法规、司法解释、地方法规分批跑。", + ], + } + + +def markdown_table(rows: list[list[Any]]) -> str: + if not rows: + return "" + header = rows[0] + lines = [ + "| " + " | ".join(str(item) for item in header) + " |", + "| " + " | ".join("---" for _ in header) + " |", + ] + for row in rows[1:]: + lines.append("| " + " | ".join(str(item).replace("\n", " ") for item in row) + " |") + return "\n".join(lines) + + +def write_markdown_report(path: Path, summary: dict[str, Any]) -> None: + gap = summary["gap_summary"] + legal = gap["legal_library"] + templates = gap["templates"] + prompts = gap["prompts"] + rules = gap["rules_and_clauses"] + + prompt_rows = [["Prompt 类型", "黄金样本数", "最低要求", "样本达标", "远程资产数"]] + remote_by_type = { + str(item.get("prompt_type")): int(item.get("count") or 0) + for item in (summary.get("remote_db") or {}).get("prompt_assets_by_type", []) + } + for suite in prompts["golden_suites"]: + prompt_type = suite["prompt_type"] + prompt_rows.append( + [ + prompt_type, + suite["sample_count"], + suite["min_samples"], + "是" if suite["meets_min_samples"] else "否", + remote_by_type.get(prompt_type, 0), + ] + ) + + template_rows = [["模板编号", "名称", "当前状态"]] + remote_template_codes = {str(item.get("template_code") or "").strip() for item in templates["remote_templates"]} + for item in EXPECTED_RUNTIME_TEMPLATES: + status = "远程已存在" if item["template_code"] in remote_template_codes else "远程缺失" + template_rows.append([item["template_code"], item["name"], status]) + + lines = [ + "# 法律库、模板、Prompt 资产盘点与缺口报告", + "", + f"生成时间:{summary['generated_at']}", + "", + "## 一、结论", + "", + "- 法律库运行表已经吸收本地 `cn-laws-download` 备份包的主体数据:远程 `legal_articles` 为 " + f"{legal['remote_legal_articles']} 条,已索引 {legal['remote_indexed']} 条,已发布 {legal['remote_published']} 条。", + f"- 同一备份包准备记录 {legal['prepared_article_rows']} 条,导入时跳过重复 {legal['import_skipped_existing_duplicates']} 条," + f"按既有导入规则计算的记录缺口为 {legal['legal_record_gap_after_known_import']}。", + f"- 远程模板只有 {templates['remote_template_count']} 个,其中正式发布模板 {templates['remote_templates_published']} 个;" + f"本地 Phase13 模板清单有 {templates['local_inventory_count']} 个。", + f"- 远程 Prompt 资产为 {prompts['remote_prompt_assets']},Prompt 版本为 {prompts['remote_prompt_versions']}," + "这说明 Prompt 治理页面/表已经有了,但生产运行库还没有种入正式 Prompt。", + f"- 远程审查规则为 {rules['remote_review_rules']},本地 Phase13 规则清单有 {rules['local_review_rule_inventory_count']} 条。", + "", + "## 二、法律库盘点", + "", + markdown_table( + [ + ["项目", "数量"], + ["备份包 metadata.json", legal["source_documents_in_backup"]], + ["备份包生成待入库条文", legal["prepared_article_rows"]], + ["导入插入", legal["import_inserted"]], + ["导入重复跳过", legal["import_skipped_existing_duplicates"]], + ["远程 legal_articles", legal["remote_legal_articles"]], + ["远程不同法源", legal["remote_legal_sources"]], + ["source 文档数与远程法源数差异", legal["source_count_delta_needs_review"]], + ["知识库文档数", legal["vector_knowledge_documents"]], + ] + ), + "", + "判断:当前不应重复把这 17,235 份全文再导一次。应该先复核 source 文档数与远程法源数的差异,再做增量抓取和分批向量化。", + "", + "## 三、模板缺口", + "", + markdown_table(template_rows), + "", + "缺失的本地 Phase13 模板编号:", + "", + ", ".join(templates["missing_local_templates_in_remote"]) or "无", + "", + "## 四、Prompt 缺口", + "", + markdown_table(prompt_rows), + "", + "缺失的必备 Prompt 类型:", + "", + ", ".join(prompts["missing_required_prompt_types"]) or "无", + "", + "## 五、规则与标准条款缺口", + "", + markdown_table( + [ + ["项目", "本地清单", "远程运行表"], + ["标准条款", rules["local_clause_inventory_count"], "当前没有独立标准条款运行表,主要在文档/模板/推荐规则中使用"], + ["审查规则", rules["local_review_rule_inventory_count"], rules["remote_review_rules"]], + ["审查规则版本", "-", rules["remote_review_rule_versions"]], + ["已发布审查规则", "-", rules["remote_review_rules_published"]], + ] + ), + "", + "## 六、建议执行顺序", + "", + ] + for index, item in enumerate(gap["priority_order"], start=1): + lines.append(f"{index}. {item}") + lines.extend( + [ + "", + "## 七、下一刀建议", + "", + "先做“Prompt 资产正式种入 + 发布 + 黄金样本评测”闭环。原因是法律库主体已入库,而远程 Prompt 资产为 0," + "这会直接影响生成、审查、Chat、条款推荐、措辞优化和法规影响扫描的运行质量。", + "", + ] + ) + path.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser(description="Build legal/template/prompt asset inventory gap report.") + parser.add_argument("--laws-backup-dir", type=Path, required=True) + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--library-root", type=Path) + parser.add_argument("--remote-db-json", type=Path) + parser.add_argument("--prepare-summary-json", type=Path) + parser.add_argument("--import-summary-json", type=Path) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + library_root = (args.library_root or repo_root / "project-docs/phase13-legal-knowledge-template-library").resolve() + args.output_dir.mkdir(parents=True, exist_ok=True) + + laws_backup = scan_laws_backup(args.laws_backup_dir) + local_assets = scan_local_assets(repo_root, library_root) + remote_db = read_json(args.remote_db_json) if args.remote_db_json else {} + prepare_summary = read_json(args.prepare_summary_json) if args.prepare_summary_json else {} + import_summary = read_json(args.import_summary_json) if args.import_summary_json else {} + gap_summary = build_gap_summary(laws_backup, local_assets, remote_db, prepare_summary, import_summary) + + summary = { + "generated_at": datetime.now(timezone.utc).isoformat(), + "repo_root": str(repo_root), + "laws_backup": laws_backup, + "local_assets": local_assets, + "remote_db": remote_db, + "prepare_summary": prepare_summary, + "import_summary": import_summary, + "gap_summary": gap_summary, + } + json_path = args.output_dir / "asset-inventory-gap-report.json" + md_path = args.output_dir / "asset-inventory-gap-report.md" + json_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2, default=str), encoding="utf-8") + write_markdown_report(md_path, summary) + print(f"ASSET_INVENTORY_GAP_REPORT_PASS {args.output_dir}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md index 44fbef50..12eef36f 100644 --- a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md +++ b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md @@ -4522,3 +4522,37 @@ - 清理后烟测记录数:`remote_smoke_record_count_after_cleanup=0`。 - 当前闭环结论: - 条款推荐行为漏斗已完成“本地开发验证 -> Git 提交推送 -> 3.90 部署 -> Alembic 真相源 -> 远程 HTTP -> 真实库动作更新烟测”的完整闭环。 + +## 2026-05-22 法律库、模板、Prompt 资产盘点与缺口报告 + +- 本轮目标: + - 暂停继续抓取,先确认现有法律库、模板、Prompt、审查规则和知识库资产到底已经吸收到什么程度。 + - 输出可复跑盘点脚本和报告,作为后续“继续抓取/继续导入/补模板/补 Prompt”的决策依据。 +- 新增工具: + - `backend/tools/asset_inventory_gap_report.py` + - `scripts/run-asset-inventory-gap-report.ps1` + - 脚本只读本地备份包、本地 Phase13/Phase15 清单和远程 PostgreSQL 真相源,不抓取、不删除、不导入。 +- 盘点输入: + - 本地法律备份包:`E:\DB_Backups\192.168.3.90_20260519_114618\file_backups\cn-laws-download_20260519_155247` + - 既有导入摘要:`project-docs/qa-artifacts/cn-laws-download-import/2026-05-20T09-53-00` + - 远程真相源:`192.168.3.90` 上 `ai-contract-backend-restore` 容器连接的 PostgreSQL。 +- 报告产物: + - `project-docs/qa-artifacts/asset-inventory-gap-report/2026-05-22T09-17-53/asset-inventory-gap-report.md` + - `project-docs/qa-artifacts/asset-inventory-gap-report/2026-05-22T09-17-53/asset-inventory-gap-report.json` + - `project-docs/qa-artifacts/asset-inventory-gap-report/2026-05-22T09-17-53/remote-db-snapshot.json` +- 核心结论: + - 法律库主体已经吸收进运行表:远程 `legal_articles=690125`,已索引 `690125`,已发布 `690125`。 + - 同一备份包准备入库条文 `691425`,导入时跳过重复 `1300`,按既有导入规则计算的法律条文记录缺口为 `0`。 + - 备份包 source 文档 `17235`,远程按 `legal_articles.source_id` 去重法源 `17219`,差异 `16`,需要后续做来源级复核,但不应重复全量导入。 + - 远程知识库文档 `10`,其中 `indexed/ready/completed` 口径为 `6`;这说明全文法律库虽然进了 `legal_articles`,但还不是全量向量知识库。 + - 本地 Phase13 模板清单 `10` 个,远程运行表只有 `4` 个,正式发布模板 `0` 个。 + - 缺失模板编号:`TPL-COMMISSION-SERVICE-001`、`TPL-COOPERATION-FRAMEWORK-001`、`TPL-LEASE-HOUSE-001`、`TPL-SALES-GOODS-001`、`TPL-SERVICE-GENERAL-001`、`TPL-SOFTWARE-SAAS-001`、`TPL-TECH-DEVELOPMENT-001`。 + - 远程 `prompt_assets=0`、`prompt_versions=0`、`prompt_eval_runs=0`;必备 Prompt 类型 `contract_generation`、`contract_review`、`legal_chat`、`clause_recommendation`、`wording_rewrite`、`regulation_impact` 都还没有正式种入运行库。 + - 远程 `review_rules=0`,本地 Phase13 审查规则清单有 `8` 条。 +- 当前判断: + - 不建议马上对同一 `cn-laws-download` 包做重复全量导入。 + - 下一阶段真正应该补的是“Prompt 资产正式种入 + 发布 + 黄金样本评测”,然后补 Phase13 模板、审查规则、标准条款吸收,最后再做法律库增量抓取和分批向量化。 +- 验证: + - `python -m compileall backend\tools\asset_inventory_gap_report.py` + - `$env:SSHPASS='***'; powershell -NoProfile -ExecutionPolicy Bypass -File .\scripts\run-asset-inventory-gap-report.ps1 -UseSshpass` + - 输出:`ASSET_INVENTORY_GAP_REPORT_PASS D:\ai-contract-platform\project-docs\qa-artifacts\asset-inventory-gap-report\2026-05-22T09-17-53` diff --git a/scripts/run-asset-inventory-gap-report.ps1 b/scripts/run-asset-inventory-gap-report.ps1 new file mode 100644 index 00000000..5f303887 --- /dev/null +++ b/scripts/run-asset-inventory-gap-report.ps1 @@ -0,0 +1,350 @@ +param( + [string]$LawsBackupDir = "E:\DB_Backups\192.168.3.90_20260519_114618\file_backups\cn-laws-download_20260519_155247", + [string]$ArtifactRoot = "", + [string]$RemoteHost = "192.168.3.90", + [string]$RemoteUser = "root", + [string]$BackendContainer = "ai-contract-backend-restore", + [string]$SshKeyPath = "$HOME\.ssh\id_rsa", + [switch]$UseSshpass, + [string]$PrepareSummaryJson = "", + [string]$ImportSummaryJson = "" +) + +$ErrorActionPreference = "Stop" + +function Resolve-SshpassPath { + $command = Get-Command sshpass.exe -ErrorAction SilentlyContinue + if ($null -ne $command) { + return $command.Source + } + + $candidates = @( + (Join-Path $env:LOCALAPPDATA "Microsoft\WinGet\Packages\xhcoding.sshpass-win32_Microsoft.Winget.Source_8wekyb3d8bbwe\sshpass.exe") + ) + foreach ($candidate in $candidates) { + if (Test-Path $candidate) { + return $candidate + } + } + + return $null +} + +function Invoke-LoggedCommand { + param( + [string]$Name, + [string]$FilePath, + [string[]]$Arguments, + [string]$LogDir + ) + + $logPath = Join-Path $LogDir "$Name.log" + $previousErrorActionPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + try { + $output = & $FilePath @Arguments 2>&1 + $exitCode = if ($LASTEXITCODE -is [int]) { $LASTEXITCODE } else { 0 } + } finally { + $ErrorActionPreference = $previousErrorActionPreference + } + + $outputText = ($output -join [Environment]::NewLine) + $outputText | Set-Content -Encoding UTF8 -Path $logPath + if ($exitCode -ne 0) { + throw "$Name failed with exit code $exitCode. See $logPath" + } + return $outputText +} + +function Get-JsonFromOutput { + param([string]$Text) + + $start = $Text.IndexOf("{") + if ($start -lt 0) { + throw "Remote output did not contain a JSON object." + } + return $Text.Substring($start) +} + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) { + throw "GSTACK_MISSING: gstack is required for AI-assisted work in this repo." +} + +if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\asset-inventory-gap-report" +} +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss" +$artifactDir = Join-Path $ArtifactRoot $timestamp +New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null + +if ([string]::IsNullOrWhiteSpace($PrepareSummaryJson)) { + $PrepareSummaryJson = Join-Path $repoRoot "project-docs\qa-artifacts\cn-laws-download-import\2026-05-20T09-53-00\cn_laws_prepare_summary_20260520.json" +} +if ([string]::IsNullOrWhiteSpace($ImportSummaryJson)) { + $ImportSummaryJson = Join-Path $repoRoot "project-docs\qa-artifacts\cn-laws-download-import\2026-05-20T09-53-00\cn_laws_import_summary_20260520.json" +} + +foreach ($requiredPath in @($LawsBackupDir, $PrepareSummaryJson, $ImportSummaryJson)) { + if (-not (Test-Path $requiredPath)) { + throw "Required input path does not exist: $requiredPath" + } +} + +$authMode = "" +$sshpassPath = $null +if ($UseSshpass -or -not [string]::IsNullOrWhiteSpace($env:SSHPASS)) { + $sshpassPath = Resolve-SshpassPath + if ([string]::IsNullOrWhiteSpace($sshpassPath)) { + throw "sshpass.exe not found. Install sshpass or use SSH key auth." + } + if ([string]::IsNullOrWhiteSpace($env:SSHPASS)) { + throw "SSHPASS environment variable is empty." + } + $authMode = "sshpass" +} elseif (Test-Path $SshKeyPath) { + $authMode = "key" +} else { + throw "SSH key not found and SSHPASS is not configured." +} + +$remoteScript = @' +from __future__ import annotations + +import json +import os +from sqlalchemy import create_engine, inspect, text + + +errors = [] +database_url = os.environ.get("DATABASE_URL") +if not database_url: + raise RuntimeError("DATABASE_URL is required in backend container") + +engine = create_engine(database_url) +inspector = inspect(engine) +tables = set(inspector.get_table_names()) +columns_cache = {} + + +def columns(table_name): + if table_name not in tables: + return set() + if table_name not in columns_cache: + columns_cache[table_name] = {item["name"] for item in inspector.get_columns(table_name)} + return columns_cache[table_name] + + +def has_column(table_name, column_name): + return column_name in columns(table_name) + + +def count_table(conn, table_name, where_clause=None): + if table_name not in tables: + return 0 + sql = f"select count(*) from {table_name}" + if where_clause: + sql += f" where {where_clause}" + try: + return int(conn.execute(text(sql)).scalar() or 0) + except Exception as exc: + errors.append({"table": table_name, "where": where_clause, "error": str(exc)}) + return 0 + + +def rows(conn, sql): + try: + return [dict(item._mapping) for item in conn.execute(text(sql)).fetchall()] + except Exception as exc: + errors.append({"sql": sql, "error": str(exc)}) + return [] + + +def published_count(conn, table_name): + if table_name not in tables: + return 0 + if has_column(table_name, "published_at"): + return count_table(conn, table_name, "published_at is not null") + if has_column(table_name, "status"): + return count_table(conn, table_name, "status = 'published'") + if has_column(table_name, "is_published"): + return count_table(conn, table_name, "is_published = true") + return 0 + + +with engine.connect() as conn: + snapshot = { + "tables_present": sorted( + table for table in [ + "legal_articles", + "legal_sources", + "knowledge_bases", + "knowledge_base_documents", + "contract_templates", + "contract_template_versions", + "review_rules", + "review_rule_versions", + "prompt_assets", + "prompt_versions", + "prompt_eval_runs", + ] + if table in tables + ), + "legal_articles": count_table(conn, "legal_articles"), + "legal_sources": count_table(conn, "legal_sources"), + "knowledge_bases": count_table(conn, "knowledge_bases"), + "knowledge_base_documents": count_table(conn, "knowledge_base_documents"), + "knowledge_base_documents_ready": ( + count_table(conn, "knowledge_base_documents", "status in ('ready', 'indexed', 'completed')") + if has_column("knowledge_base_documents", "status") + else 0 + ), + "contract_templates": count_table(conn, "contract_templates"), + "templates_enabled": 0, + "templates_published": published_count(conn, "contract_templates"), + "contract_template_versions": count_table(conn, "contract_template_versions"), + "review_rules": count_table(conn, "review_rules"), + "review_rules_active": ( + count_table(conn, "review_rules", "is_active = true") + if has_column("review_rules", "is_active") + else 0 + ), + "review_rules_published": published_count(conn, "review_rules"), + "review_rule_versions": count_table(conn, "review_rule_versions"), + "prompt_assets": count_table(conn, "prompt_assets"), + "prompts_published": published_count(conn, "prompt_assets"), + "prompt_versions": count_table(conn, "prompt_versions"), + "prompt_eval_runs": count_table(conn, "prompt_eval_runs"), + "errors": errors, + } + + if has_column("legal_articles", "index_status"): + snapshot["legal_articles_indexed"] = count_table( + conn, + "legal_articles", + "index_status in ('indexed', 'ready', 'completed')", + ) + elif has_column("legal_articles", "indexed_at"): + snapshot["legal_articles_indexed"] = count_table(conn, "legal_articles", "indexed_at is not null") + else: + snapshot["legal_articles_indexed"] = 0 + + if has_column("legal_articles", "published_status"): + snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "published_status = 'published'") + elif has_column("legal_articles", "status"): + snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "status = 'published'") + elif has_column("legal_articles", "published_at"): + snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "published_at is not null") + elif has_column("legal_articles", "is_published"): + snapshot["legal_articles_published"] = count_table(conn, "legal_articles", "is_published = true") + else: + snapshot["legal_articles_published"] = 0 + + if snapshot["legal_sources"] == 0 and has_column("legal_articles", "source_id"): + source_count = rows( + conn, + "select count(distinct source_id) as count from legal_articles where source_id is not null and source_id <> ''", + ) + snapshot["legal_sources"] = int(source_count[0]["count"] or 0) if source_count else 0 + + if "contract_templates" in tables: + if has_column("contract_templates", "is_enabled"): + snapshot["templates_enabled"] = count_table(conn, "contract_templates", "is_enabled = true") + elif has_column("contract_templates", "enabled"): + snapshot["templates_enabled"] = count_table(conn, "contract_templates", "enabled = true") + else: + snapshot["templates_enabled"] = count_table(conn, "contract_templates") + + if "contract_templates" in tables: + select_columns = [ + column + for column in ["id", "template_code", "name", "document_type_code", "status", "published_at", "is_enabled"] + if has_column("contract_templates", column) + ] + snapshot["templates"] = rows( + conn, + f"select {', '.join(select_columns)} from contract_templates order by template_code nulls last, name limit 100", + ) if select_columns else [] + else: + snapshot["templates"] = [] + + if has_column("prompt_assets", "prompt_type"): + snapshot["prompt_assets_by_type"] = rows( + conn, + "select prompt_type, count(*) as count from prompt_assets group by prompt_type order by prompt_type", + ) + else: + snapshot["prompt_assets_by_type"] = [] + + if has_column("review_rules", "status"): + snapshot["review_rules_by_status"] = rows( + conn, + "select status, count(*) as count from review_rules group by status order by status", + ) + else: + snapshot["review_rules_by_status"] = [] + +print(json.dumps(snapshot, ensure_ascii=False, default=str, indent=2)) +'@ + +$localRemoteScript = Join-Path $artifactDir "remote-db-snapshot.py" +$remoteSnapshotPath = Join-Path $artifactDir "remote-db-snapshot.json" +$remoteScript | Set-Content -Encoding UTF8 -Path $localRemoteScript + +$target = "${RemoteUser}@${RemoteHost}" +$remoteTempScript = "/tmp/asset_inventory_remote_snapshot.py" + +if ($authMode -eq "sshpass") { + Invoke-LoggedCommand -Name "scp-remote-db-snapshot-script" -FilePath $sshpassPath -Arguments @( + "-e", + "scp", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + $localRemoteScript, + "${target}:$remoteTempScript" + ) -LogDir $artifactDir | Out-Null + $remoteOutput = Invoke-LoggedCommand -Name "remote-db-snapshot" -FilePath $sshpassPath -Arguments @( + "-e", + "ssh", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + $target, + "docker cp $remoteTempScript ${BackendContainer}:$remoteTempScript && docker exec $BackendContainer sh -lc 'cd /app && python $remoteTempScript'" + ) -LogDir $artifactDir +} else { + Invoke-LoggedCommand -Name "scp-remote-db-snapshot-script" -FilePath "scp" -Arguments @( + "-i", $SshKeyPath, + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + $localRemoteScript, + "${target}:$remoteTempScript" + ) -LogDir $artifactDir | Out-Null + $remoteOutput = Invoke-LoggedCommand -Name "remote-db-snapshot" -FilePath "ssh" -Arguments @( + "-i", $SshKeyPath, + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "ConnectTimeout=10", + $target, + "docker cp $remoteTempScript ${BackendContainer}:$remoteTempScript && docker exec $BackendContainer sh -lc 'cd /app && python $remoteTempScript'" + ) -LogDir $artifactDir +} + +Get-JsonFromOutput $remoteOutput | Set-Content -Encoding UTF8 -Path $remoteSnapshotPath + +$toolPath = Join-Path $repoRoot "backend\tools\asset_inventory_gap_report.py" +$toolOutput = & python $toolPath ` + --laws-backup-dir $LawsBackupDir ` + --repo-root $repoRoot ` + --remote-db-json $remoteSnapshotPath ` + --prepare-summary-json $PrepareSummaryJson ` + --import-summary-json $ImportSummaryJson ` + --output-dir $artifactDir + +$toolExitCode = if ($LASTEXITCODE -is [int]) { $LASTEXITCODE } else { 0 } +($toolOutput -join [Environment]::NewLine) | Set-Content -Encoding UTF8 -Path (Join-Path $artifactDir "asset-inventory-gap-report-tool.log") +if ($toolExitCode -ne 0) { + throw "asset_inventory_gap_report.py failed with exit code $toolExitCode." +} + +Write-Output "ASSET_INVENTORY_GAP_REPORT_PASS $artifactDir"