From f71b830683dba4e9c35e2b31058e1fd25444deef Mon Sep 17 00:00:00 2001 From: yuqei Date: Wed, 13 May 2026 23:54:28 +0800 Subject: [PATCH] feat: add knowledge source metadata fields --- backend/app/clients/shared_ai_platform.py | 45 +++++++++- backend/app/models/knowledge_base_document.py | 7 +- ..._add_knowledge_document_source_metadata.py | 73 ++++++++++++++++ backend/tests/test_knowledge_bases_api.py | 12 +++ .../tests/test_shared_ai_platform_client.py | 83 +++++++++++++++++++ frontend/src/app/knowledge-bases/page.tsx | 66 +++++++++++++-- frontend/src/types/api.ts | 4 + ...草与审查助手-当前工作进度保存-2026-05-11.md | 49 +++++++++++ .../governance-productization-backlog.csv | 2 +- 9 files changed, 330 insertions(+), 11 deletions(-) create mode 100644 backend/migrations/versions/20260513_0020_add_knowledge_document_source_metadata.py diff --git a/backend/app/clients/shared_ai_platform.py b/backend/app/clients/shared_ai_platform.py index 3e160963..ca2a0c75 100644 --- a/backend/app/clients/shared_ai_platform.py +++ b/backend/app/clients/shared_ai_platform.py @@ -135,7 +135,8 @@ class RemoteSharedAIPlatformClient: return payload def list_documents(self, current_user: Dict[str, Any], knowledge_base_id: str, page: int, page_size: int) -> dict: - return self.sdk.list_documents(self._context(current_user), knowledge_base_id, page=page, page_size=page_size) + payload = self.sdk.list_documents(self._context(current_user), knowledge_base_id, page=page, page_size=page_size) + return self._normalize_remote_knowledge_documents_payload(payload) def upload_document( self, @@ -145,13 +146,14 @@ class RemoteSharedAIPlatformClient: mime_type: Optional[str], content: bytes, ) -> dict: - return self.sdk.upload_document( + payload = self.sdk.upload_document( self._context(current_user), knowledge_base_id, file_name=file_name, mime_type=mime_type, content=content, ) + return self._normalize_remote_knowledge_document_item(payload) def retry_document( self, @@ -162,7 +164,7 @@ class RemoteSharedAIPlatformClient: mime_type: Optional[str] = None, content: Optional[bytes] = None, ) -> dict: - return self.sdk.retry_document( + payload = self.sdk.retry_document( self._context(current_user), knowledge_base_id, document_id, @@ -170,6 +172,7 @@ class RemoteSharedAIPlatformClient: mime_type=mime_type, content=content, ) + return self._normalize_remote_knowledge_document_item(payload) def _context(self, current_user: Dict[str, Any]) -> SharedAIRequestContext: return SharedAIRequestContext( @@ -191,6 +194,42 @@ class RemoteSharedAIPlatformClient: item["retrieval_mode"] = "hybrid" return payload + def _normalize_remote_knowledge_documents_payload(self, payload: dict) -> dict: + items = payload.get("items") + if not isinstance(items, list): + return payload + + payload["items"] = [self._normalize_remote_knowledge_document_item(item) for item in items] + return payload + + def _normalize_remote_knowledge_document_item(self, payload: dict) -> dict: + metadata = payload.get("metadata_json") + if not isinstance(metadata, Mapping): + metadata = payload.get("metadata") + metadata = metadata if isinstance(metadata, Mapping) else {} + + normalized = dict(payload) + source_type = normalized.get("source_type") or metadata.get("source_type") + authority = normalized.get("authority") or metadata.get("authority") or metadata.get("issuer") + effective_date = normalized.get("effective_date") or metadata.get("effective_date") + region = normalized.get("region") or metadata.get("region") or metadata.get("jurisdiction") + tags = normalized.get("tags") + if tags is None: + tags = normalized.get("tags_json") + if tags is None: + tags = metadata.get("tags") + + normalized["source_type"] = source_type + normalized["authority"] = authority + normalized["effective_date"] = effective_date + normalized["region"] = region + normalized["tags"] = list(tags) if isinstance(tags, list) else None + if normalized["tags"] is not None and normalized.get("tags_json") is None: + normalized["tags_json"] = normalized["tags"] + if metadata and normalized.get("metadata_json") is None: + normalized["metadata_json"] = dict(metadata) + return normalized + def _normalize_review_context(self, review_context: Any) -> Optional[Dict[str, Any]]: if review_context is None: return None diff --git a/backend/app/models/knowledge_base_document.py b/backend/app/models/knowledge_base_document.py index f91c8ce9..e613260d 100644 --- a/backend/app/models/knowledge_base_document.py +++ b/backend/app/models/knowledge_base_document.py @@ -1,7 +1,7 @@ from datetime import datetime, timezone from typing import Optional -from sqlalchemy import DateTime, Integer, String, Text +from sqlalchemy import JSON, DateTime, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column from app.db import Base @@ -17,6 +17,11 @@ class KnowledgeBaseDocument(Base): mime_type: Mapped[Optional[str]] = mapped_column(String(120), nullable=True) file_size: Mapped[int] = mapped_column(Integer, default=0) status: Mapped[str] = mapped_column(String(32), default="indexed") + source_type: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) + authority: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + effective_date: Mapped[Optional[str]] = mapped_column(String(32), nullable=True) + region: Mapped[Optional[str]] = mapped_column(String(120), nullable=True, index=True) + tags_json: Mapped[Optional[list[str]]] = mapped_column(JSON, nullable=True) content_excerpt: Mapped[Optional[str]] = mapped_column(Text, nullable=True) error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True) diff --git a/backend/migrations/versions/20260513_0020_add_knowledge_document_source_metadata.py b/backend/migrations/versions/20260513_0020_add_knowledge_document_source_metadata.py new file mode 100644 index 00000000..415bd1ba --- /dev/null +++ b/backend/migrations/versions/20260513_0020_add_knowledge_document_source_metadata.py @@ -0,0 +1,73 @@ +"""add knowledge document source metadata + +Revision ID: 20260513_0020 +Revises: 20260513_0019 +Create Date: 2026-05-13 23:55:00 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = "20260513_0020" +down_revision = "20260513_0019" +branch_labels = None +depends_on = None + + +def _table_names(inspector: sa.Inspector) -> set[str]: + return set(inspector.get_table_names()) + + +def _column_names(inspector: sa.Inspector, table_name: str) -> set[str]: + return {column["name"] for column in inspector.get_columns(table_name)} + + +def _index_names(inspector: sa.Inspector, table_name: str) -> set[str]: + return {index["name"] for index in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "knowledge_base_documents" not in _table_names(inspector): + return + + columns = _column_names(inspector, "knowledge_base_documents") + if "source_type" not in columns: + op.add_column("knowledge_base_documents", sa.Column("source_type", sa.String(length=64), nullable=True)) + if "authority" not in columns: + op.add_column("knowledge_base_documents", sa.Column("authority", sa.String(length=255), nullable=True)) + if "effective_date" not in columns: + op.add_column("knowledge_base_documents", sa.Column("effective_date", sa.String(length=32), nullable=True)) + if "region" not in columns: + op.add_column("knowledge_base_documents", sa.Column("region", sa.String(length=120), nullable=True)) + if "tags_json" not in columns: + op.add_column("knowledge_base_documents", sa.Column("tags_json", sa.JSON(), nullable=True)) + + inspector = sa.inspect(bind) + indexes = _index_names(inspector, "knowledge_base_documents") + if "ix_knowledge_base_documents_source_type" not in indexes: + op.create_index("ix_knowledge_base_documents_source_type", "knowledge_base_documents", ["source_type"], unique=False) + if "ix_knowledge_base_documents_region" not in indexes: + op.create_index("ix_knowledge_base_documents_region", "knowledge_base_documents", ["region"], unique=False) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "knowledge_base_documents" not in _table_names(inspector): + return + + indexes = _index_names(inspector, "knowledge_base_documents") + if "ix_knowledge_base_documents_region" in indexes: + op.drop_index("ix_knowledge_base_documents_region", table_name="knowledge_base_documents") + if "ix_knowledge_base_documents_source_type" in indexes: + op.drop_index("ix_knowledge_base_documents_source_type", table_name="knowledge_base_documents") + + columns = _column_names(inspector, "knowledge_base_documents") + for column_name in ["tags_json", "region", "effective_date", "authority", "source_type"]: + if column_name in columns: + op.drop_column("knowledge_base_documents", column_name) diff --git a/backend/tests/test_knowledge_bases_api.py b/backend/tests/test_knowledge_bases_api.py index b37b6349..0919f430 100644 --- a/backend/tests/test_knowledge_bases_api.py +++ b/backend/tests/test_knowledge_bases_api.py @@ -64,6 +64,11 @@ class StubKnowledgeBaseService: "mime_type": "text/plain", "file_size": 12, "status": "indexed", + "source_type": "statute", + "authority": "全国人民代表大会", + "effective_date": "2021-01-01", + "region": "CN", + "tags": ["合同法", "通用"], "content_excerpt": "合同解除规则", "extracted_text_length": 6, "preview_truncated": False, @@ -84,6 +89,11 @@ class StubKnowledgeBaseService: "mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "file_size": 18, "status": "processing", + "source_type": "policy", + "authority": "企业法务部", + "effective_date": "2026-05-01", + "region": "CN-SH", + "tags": ["制度"], "content_excerpt": None, "extracted_text_length": 0, "preview_truncated": False, @@ -193,6 +203,8 @@ class KnowledgeBasesApiTestCase(unittest.TestCase): self.assertEqual(blocked_item["index_status"], "blocked") self.assertEqual(blocked_item["failure_reason"], "文本提取尚未完成，索引阶段暂时阻塞。") self.assertIn("继续进入索引阶段", blocked_item["next_action"]) + self.assertEqual(items[0]["authority"], "全国人民代表大会") + self.assertEqual(items[0]["region"], "CN") def test_admin_can_create_knowledge_base(self): app.dependency_overrides[require_current_user] = lambda: { diff --git a/backend/tests/test_shared_ai_platform_client.py b/backend/tests/test_shared_ai_platform_client.py index 97108b0f..a479f3db 100644 --- a/backend/tests/test_shared_ai_platform_client.py +++ b/backend/tests/test_shared_ai_platform_client.py @@ -96,6 +96,89 @@ class RemoteSharedAiPlatformClientStreamTestCase(unittest.TestCase): ], ) + def test_remote_document_payloads_are_normalized_for_source_metadata(self): + class StubSdk: + def list_documents(self, context, knowledge_base_id, *, page=1, page_size=20): + return { + "items": [ + { + "id": "doc-1", + "knowledge_base_id": knowledge_base_id, + "file_name": "civil-code.md", + "mime_type": "text/markdown", + "file_size": 128, + "status": "indexed", + "metadata": { + "source_type": "statute", + "issuer": "全国人民代表大会", + "effective_date": "2021-01-01", + "jurisdiction": "CN", + "tags": ["民法典", "合同"], + }, + } + ], + "page": page, + "page_size": page_size, + "total": 1, + } + + def upload_document(self, context, knowledge_base_id, *, file_name, mime_type, content): + return { + "id": "doc-2", + "knowledge_base_id": knowledge_base_id, + "file_name": file_name, + "mime_type": mime_type, + "file_size": len(content), + "status": "indexed", + "tags_json": ["制度"], + "metadata_json": { + "source_type": "policy", + "authority": "企业法务部", + "effective_date": "2026-05-01", + "region": "CN-SH", + }, + } + + def retry_document(self, context, knowledge_base_id, document_id, *, file_name=None, mime_type=None, content=None): + return { + "id": document_id, + "knowledge_base_id": knowledge_base_id, + "file_name": file_name or "retry.txt", + "mime_type": mime_type or "text/plain", + "file_size": len(content or b""), + "status": "indexed", + "metadata": { + "source_type": "judicial-interpretation", + "authority": "最高人民法院", + "effective_date": "2024-02-01", + "region": "CN", + }, + } + + client = RemoteSharedAIPlatformClient( + base_url="http://testserver", + app_id="test-app", + api_key="test-key", + timeout_seconds=30.0, + ) + client.sdk = StubSdk() + current_user = {"id": "user-1", "tenant_id": "tenant-1", "role": "admin"} + + listed = client.list_documents(current_user, "kb-1", 1, 20) + uploaded = client.upload_document(current_user, "kb-1", "policy.md", "text/markdown", b"hello") + retried = client.retry_document(current_user, "kb-1", "doc-3", file_name="retry.md", mime_type="text/markdown", content=b"world") + + self.assertEqual(listed["items"][0]["source_type"], "statute") + self.assertEqual(listed["items"][0]["authority"], "全国人民代表大会") + self.assertEqual(listed["items"][0]["effective_date"], "2021-01-01") + self.assertEqual(listed["items"][0]["region"], "CN") + self.assertEqual(listed["items"][0]["tags"], ["民法典", "合同"]) + self.assertEqual(uploaded["authority"], "企业法务部") + self.assertEqual(uploaded["region"], "CN-SH") + self.assertEqual(uploaded["tags"], ["制度"]) + self.assertEqual(retried["source_type"], "judicial-interpretation") + self.assertEqual(retried["authority"], "最高人民法院") + if __name__ == "__main__": unittest.main() diff --git a/frontend/src/app/knowledge-bases/page.tsx b/frontend/src/app/knowledge-bases/page.tsx index c2943c7b..8515b94b 100644 --- a/frontend/src/app/knowledge-bases/page.tsx +++ b/frontend/src/app/knowledge-bases/page.tsx @@ -2,7 +2,7 @@ import Link from "next/link"; import { InboxOutlined, ReloadOutlined } from "@ant-design/icons"; -import { Alert, Button, Card, Empty, Form, Input, List, Space, Table, Tag, Typography, Upload, message } from "antd"; +import { Alert, Button, Card, Empty, Form, Input, List, Select, Space, Table, Tag, Typography, Upload, message } from "antd"; import type { UploadFile } from "antd/es/upload/interface"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -175,6 +175,8 @@ export default function KnowledgeBasesPage() { const [items, setItems] = useState([]); const [selectedId, setSelectedId] = useState(null); const [documents, setDocuments] = useState([]); + const [sourceTypeFilter, setSourceTypeFilter] = useState("all"); + const [regionFilter, setRegionFilter] = useState("all"); const [documentsLoading, setDocumentsLoading] = useState(false); const [fileList, setFileList] = useState([]); @@ -183,8 +185,16 @@ export default function KnowledgeBasesPage() { [items, selectedId], ); + const filteredDocuments = useMemo(() => { + return documents.filter((item) => { + const sourceTypeMatch = sourceTypeFilter === "all" || (item.source_type || "unknown") === sourceTypeFilter; + const regionMatch = regionFilter === "all" || (item.region || "unknown") === regionFilter; + return sourceTypeMatch && regionMatch; + }); + }, [documents, regionFilter, sourceTypeFilter]); + const orderedDocuments = useMemo(() => { - return [...documents].sort((left, right) => { + return [...filteredDocuments].sort((left, right) => { const leftPriority = isDocumentFailedLike(left) ? 0 : left.status === "processing" || left.status === "pending" ? 1 : 2; const rightPriority = isDocumentFailedLike(right) ? 0 : right.status === "processing" || right.status === "pending" ? 1 : 2; if (leftPriority !== rightPriority) { @@ -192,13 +202,23 @@ export default function KnowledgeBasesPage() { } return new Date(right.created_at).getTime() - new Date(left.created_at).getTime(); }); - }, [documents]); + }, [filteredDocuments]); const failedDocuments = useMemo( () => orderedDocuments.filter((document) => isDocumentFailedLike(document)), [orderedDocuments], ); + const sourceTypeOptions = useMemo(() => { + const values = Array.from(new Set(documents.map((item) => item.source_type || "unknown"))); + return [{ value: "all", label: "全部来源类型" }, ...values.map((value) => ({ value, label: value }))]; + }, [documents]); + + const regionOptions = useMemo(() => { + const values = Array.from(new Set(documents.map((item) => item.region || "unknown"))); + return [{ value: "all", label: "全部适用区域" }, ...values.map((value) => ({ value, label: value }))]; + }, [documents]); + const loadKnowledgeBases = useCallback(async () => { if (currentUser?.role !== "admin") { setLoading(false); @@ -566,9 +586,23 @@ export default function KnowledgeBasesPage() { className="review-surface-card" title="文档索引状态" extra={ - + + + + } > {documentsError ? ( @@ -625,6 +659,26 @@ export default function KnowledgeBasesPage() { width: 120, render: (value: number) => `${value} B`, }, + { + title: "法源元数据", + key: "source_meta", + width: 260, + render: (_value: unknown, record: KnowledgeBaseDocumentItem) => ( + + {record.source_type || "-"} + {record.authority || "-"} + {record.effective_date || "-"} + {record.region || "-"} + {record.tags?.length ? ( + + {record.tags.map((tag) => ( + {tag} + ))} + + ) : null} + + ), + }, { title: "摘要 / 生命周期说明", dataIndex: "content_excerpt", diff --git a/frontend/src/types/api.ts b/frontend/src/types/api.ts index 17aaa7fa..8f58a07c 100644 --- a/frontend/src/types/api.ts +++ b/frontend/src/types/api.ts @@ -243,6 +243,10 @@ export type KnowledgeBaseDocumentItem = { mime_type: string | null; file_size: number; status: string; + authority?: string | null; + effective_date?: string | null; + region?: string | null; + tags?: string[] | null; content_excerpt: string | null; error_message: string | null; extracted_text_length: number; diff --git a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md index 0e43b750..6e1073b9 100644 --- a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md +++ b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md @@ -1117,3 +1117,52 @@ - 当前用户体系还没有独立的 `ops` / `production_approver` 角色 - 运营复核和最终批准目前仍主要由 admin 侧承接 - 正式生产放行仍继续阻塞在真人复核、证据 READY、最终批准签署，以及知识库发布链的 shared-ai 远端能力缺口 + +## 2026-05-13 GOV-DB-007 知识库法源元数据第一层收口 + +### 本轮完成项 +- 已为本地知识库文档表补齐法源元数据字段： + - `backend/app/models/knowledge_base_document.py` + - 新增 `source_type / authority / effective_date / region / tags_json` +- 已新增数据库迁移： + - `backend/migrations/versions/20260513_0020_add_knowledge_document_source_metadata.py` + - 为 `knowledge_base_documents` 增加上述字段，并补 `source_type`、`region` 索引 +- 已补齐知识库页面的法源治理与过滤能力： + - `frontend/src/app/knowledge-bases/page.tsx` + - 文档列表新增“法源元数据”列，展示 `source_type / authority / effective_date / region / tags` + - 页面新增 `source_type`、`region` 两个过滤器，可按法源维度筛选知识库文档 +- 已补齐前端类型定义： + - `frontend/src/types/api.ts` + - `KnowledgeBaseDocumentItem` 新增 `authority / effective_date / region / tags` +- 已补强 shared-ai 返回文档元数据的兼容层： + - `backend/app/clients/shared_ai_platform.py` + - 若 shared-ai 先把法源元数据放在 `metadata` 或 `metadata_json` 中，`ai-contract-platform` 现在会自动平铺为 `source_type / authority / effective_date / region / tags` + - `tags_json` 也会自动映射为前端可直接消费的 `tags` +- 已补齐测试桩与兼容性验证： + - `backend/tests/test_knowledge_bases_api.py` + - `backend/tests/test_shared_ai_platform_client.py` + +### 本轮验证 +- 后端测试： + - `python -m unittest backend.tests.test_knowledge_bases_api backend.tests.test_shared_ai_platform_client` + - 结果：通过 +- 后端编译： + - `python -m py_compile backend/app/models/knowledge_base_document.py backend/app/clients/shared_ai_platform.py` + - 结果：通过 +- 前端检查： + - `npm --prefix frontend run lint` + - 结果：通过 + - `npm --prefix frontend run build` + - 结果：通过，知识库页面构建正常 + +### 当前状态更新 +- `GOV-DB-007` 已从 `PLANNED` 推进到 `IN_PROGRESS` +- `ai-contract-platform` 本仓库侧已经形成第一层闭环： + - 本地表结构已具备法源元数据字段 + - shared-ai 返回的知识库文档数据已可在本仓库侧平铺、展示和过滤 + - 管理员已可在知识库页面按法源维度治理和筛查文档 + +### 当前阻塞项 +- `GOV-DB-007` 仍未宣告 `COMPLETED`，原因不是本仓库缺页面或字段，而是 shared-ai 远端当前还没有正式生产化的知识库法源元数据写入链： + - 远端知识库文档真实数据是否持续提供 `source_type / authority / effective_date / region / tags`，仍依赖 shared-ai 知识库导入/同步链配合 + - 当 shared-ai 远端稳定产出这些字段后，本项即可自然收口为 `COMPLETED` diff --git a/project-docs/formal-launch-ops/governance-productization-backlog.csv b/project-docs/formal-launch-ops/governance-productization-backlog.csv index fec5708d..91f46105 100644 --- a/project-docs/formal-launch-ops/governance-productization-backlog.csv +++ b/project-docs/formal-launch-ops/governance-productization-backlog.csv @@ -5,7 +5,7 @@ batch,priority,area,task_id,title,current_gap,target_deliverable,depends_on,acce 5A,P0,database,GOV-DB-004,新增证据登记表,当前证据路径和证据状态只存在文档与脚本规则中,新增 governance_evidences 表支持 file_id url status checksum reviewer metadata,GOV-DB-001,系统可校验证据是否存在且不是模板占位,COMPLETED 5A,P0,database,GOV-DB-005,新增发布批次与门禁结果表,当前上线批准状态只存在 PowerShell 产物中,新增 governance_release_batches 和 governance_gate_runs 表,GOV-DB-001;GOV-DB-003;GOV-DB-004,系统可追踪一次发布批次的门禁快照和结果,COMPLETED 5A,P1,database,GOV-DB-006,模板表补治理字段,contract_templates 只有内容和启停没有版本与发布元数据,为 contract_templates 增加 template_code version published_at published_by latest_governance_asset_id,GOV-DB-001,模板可区分草稿版和生产发布版,COMPLETED -5A,P1,database,GOV-DB-007,知识库文档表补法源元数据,知识库文档缺少效力级别生效日期发文机关和法源类型,为知识库相关表增加 source_type authority effective_date region tags,none,知识库可按法源维度治理和过滤,PLANNED +5A,P1,database,GOV-DB-007,知识库文档表补法源元数据,知识库文档缺少效力级别生效日期发文机关和法源类型,为知识库相关表增加 source_type authority effective_date region tags,none,知识库可按法源维度治理和过滤,IN_PROGRESS 5B,P0,backend_api,GOV-API-001,新增治理资产列表接口,当前没有统一治理视图接口,新增 /governance/assets 列表筛选汇总接口,GOV-DB-001,前端可按资产类型状态角色查看治理资产,COMPLETED 5B,P0,backend_api,GOV-API-002,新增复核分配接口,当前无法在系统内分配真实复核人,新增 /governance/assignments 创建更新撤销接口,GOV-DB-002,管理员可在系统内完成复核任务分配,COMPLETED 5B,P0,backend_api,GOV-API-003,新增复核回填接口,当前复核结论只能通过CSV和脚本回填,新增 /governance/decisions 提交接口和批量导入接口,GOV-DB-003,真实复核人或管理员可提交复核结果,COMPLETED