feat: add knowledge source metadata fields

This commit is contained in:
2026-05-13 23:54:28 +08:00
parent 23e6967eec
commit f71b830683
9 changed files with 330 additions and 11 deletions
+42 -3
View File
@@ -135,7 +135,8 @@ class RemoteSharedAIPlatformClient:
return payload
def list_documents(self, current_user: Dict[str, Any], knowledge_base_id: str, page: int, page_size: int) -> dict:
return self.sdk.list_documents(self._context(current_user), knowledge_base_id, page=page, page_size=page_size)
payload = self.sdk.list_documents(self._context(current_user), knowledge_base_id, page=page, page_size=page_size)
return self._normalize_remote_knowledge_documents_payload(payload)
def upload_document(
self,
@@ -145,13 +146,14 @@ class RemoteSharedAIPlatformClient:
mime_type: Optional[str],
content: bytes,
) -> dict:
return self.sdk.upload_document(
payload = self.sdk.upload_document(
self._context(current_user),
knowledge_base_id,
file_name=file_name,
mime_type=mime_type,
content=content,
)
return self._normalize_remote_knowledge_document_item(payload)
def retry_document(
self,
@@ -162,7 +164,7 @@ class RemoteSharedAIPlatformClient:
mime_type: Optional[str] = None,
content: Optional[bytes] = None,
) -> dict:
return self.sdk.retry_document(
payload = self.sdk.retry_document(
self._context(current_user),
knowledge_base_id,
document_id,
@@ -170,6 +172,7 @@ class RemoteSharedAIPlatformClient:
mime_type=mime_type,
content=content,
)
return self._normalize_remote_knowledge_document_item(payload)
def _context(self, current_user: Dict[str, Any]) -> SharedAIRequestContext:
return SharedAIRequestContext(
@@ -191,6 +194,42 @@ class RemoteSharedAIPlatformClient:
item["retrieval_mode"] = "hybrid"
return payload
def _normalize_remote_knowledge_documents_payload(self, payload: dict) -> dict:
items = payload.get("items")
if not isinstance(items, list):
return payload
payload["items"] = [self._normalize_remote_knowledge_document_item(item) for item in items]
return payload
def _normalize_remote_knowledge_document_item(self, payload: dict) -> dict:
metadata = payload.get("metadata_json")
if not isinstance(metadata, Mapping):
metadata = payload.get("metadata")
metadata = metadata if isinstance(metadata, Mapping) else {}
normalized = dict(payload)
source_type = normalized.get("source_type") or metadata.get("source_type")
authority = normalized.get("authority") or metadata.get("authority") or metadata.get("issuer")
effective_date = normalized.get("effective_date") or metadata.get("effective_date")
region = normalized.get("region") or metadata.get("region") or metadata.get("jurisdiction")
tags = normalized.get("tags")
if tags is None:
tags = normalized.get("tags_json")
if tags is None:
tags = metadata.get("tags")
normalized["source_type"] = source_type
normalized["authority"] = authority
normalized["effective_date"] = effective_date
normalized["region"] = region
normalized["tags"] = list(tags) if isinstance(tags, list) else None
if normalized["tags"] is not None and normalized.get("tags_json") is None:
normalized["tags_json"] = normalized["tags"]
if metadata and normalized.get("metadata_json") is None:
normalized["metadata_json"] = dict(metadata)
return normalized
def _normalize_review_context(self, review_context: Any) -> Optional[Dict[str, Any]]:
if review_context is None:
return None
@@ -1,7 +1,7 @@
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy import DateTime, Integer, String, Text
from sqlalchemy import JSON, DateTime, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from app.db import Base
@@ -17,6 +17,11 @@ class KnowledgeBaseDocument(Base):
mime_type: Mapped[Optional[str]] = mapped_column(String(120), nullable=True)
file_size: Mapped[int] = mapped_column(Integer, default=0)
status: Mapped[str] = mapped_column(String(32), default="indexed")
source_type: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True)
authority: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
effective_date: Mapped[Optional[str]] = mapped_column(String(32), nullable=True)
region: Mapped[Optional[str]] = mapped_column(String(120), nullable=True, index=True)
tags_json: Mapped[Optional[list[str]]] = mapped_column(JSON, nullable=True)
content_excerpt: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
error_message: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True)
@@ -0,0 +1,73 @@
"""add knowledge document source metadata
Revision ID: 20260513_0020
Revises: 20260513_0019
Create Date: 2026-05-13 23:55:00
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260513_0020"
down_revision = "20260513_0019"
branch_labels = None
depends_on = None
def _table_names(inspector: sa.Inspector) -> set[str]:
return set(inspector.get_table_names())
def _column_names(inspector: sa.Inspector, table_name: str) -> set[str]:
return {column["name"] for column in inspector.get_columns(table_name)}
def _index_names(inspector: sa.Inspector, table_name: str) -> set[str]:
return {index["name"] for index in inspector.get_indexes(table_name)}
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if "knowledge_base_documents" not in _table_names(inspector):
return
columns = _column_names(inspector, "knowledge_base_documents")
if "source_type" not in columns:
op.add_column("knowledge_base_documents", sa.Column("source_type", sa.String(length=64), nullable=True))
if "authority" not in columns:
op.add_column("knowledge_base_documents", sa.Column("authority", sa.String(length=255), nullable=True))
if "effective_date" not in columns:
op.add_column("knowledge_base_documents", sa.Column("effective_date", sa.String(length=32), nullable=True))
if "region" not in columns:
op.add_column("knowledge_base_documents", sa.Column("region", sa.String(length=120), nullable=True))
if "tags_json" not in columns:
op.add_column("knowledge_base_documents", sa.Column("tags_json", sa.JSON(), nullable=True))
inspector = sa.inspect(bind)
indexes = _index_names(inspector, "knowledge_base_documents")
if "ix_knowledge_base_documents_source_type" not in indexes:
op.create_index("ix_knowledge_base_documents_source_type", "knowledge_base_documents", ["source_type"], unique=False)
if "ix_knowledge_base_documents_region" not in indexes:
op.create_index("ix_knowledge_base_documents_region", "knowledge_base_documents", ["region"], unique=False)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if "knowledge_base_documents" not in _table_names(inspector):
return
indexes = _index_names(inspector, "knowledge_base_documents")
if "ix_knowledge_base_documents_region" in indexes:
op.drop_index("ix_knowledge_base_documents_region", table_name="knowledge_base_documents")
if "ix_knowledge_base_documents_source_type" in indexes:
op.drop_index("ix_knowledge_base_documents_source_type", table_name="knowledge_base_documents")
columns = _column_names(inspector, "knowledge_base_documents")
for column_name in ["tags_json", "region", "effective_date", "authority", "source_type"]:
if column_name in columns:
op.drop_column("knowledge_base_documents", column_name)
+12
View File
@@ -64,6 +64,11 @@ class StubKnowledgeBaseService:
"mime_type": "text/plain",
"file_size": 12,
"status": "indexed",
"source_type": "statute",
"authority": "全国人民代表大会",
"effective_date": "2021-01-01",
"region": "CN",
"tags": ["合同法", "通用"],
"content_excerpt": "合同解除规则",
"extracted_text_length": 6,
"preview_truncated": False,
@@ -84,6 +89,11 @@ class StubKnowledgeBaseService:
"mime_type": "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
"file_size": 18,
"status": "processing",
"source_type": "policy",
"authority": "企业法务部",
"effective_date": "2026-05-01",
"region": "CN-SH",
"tags": ["制度"],
"content_excerpt": None,
"extracted_text_length": 0,
"preview_truncated": False,
@@ -193,6 +203,8 @@ class KnowledgeBasesApiTestCase(unittest.TestCase):
self.assertEqual(blocked_item["index_status"], "blocked")
self.assertEqual(blocked_item["failure_reason"], "文本提取尚未完成,索引阶段暂时阻塞。")
self.assertIn("继续进入索引阶段", blocked_item["next_action"])
self.assertEqual(items[0]["authority"], "全国人民代表大会")
self.assertEqual(items[0]["region"], "CN")
def test_admin_can_create_knowledge_base(self):
app.dependency_overrides[require_current_user] = lambda: {
@@ -96,6 +96,89 @@ class RemoteSharedAiPlatformClientStreamTestCase(unittest.TestCase):
],
)
def test_remote_document_payloads_are_normalized_for_source_metadata(self):
class StubSdk:
def list_documents(self, context, knowledge_base_id, *, page=1, page_size=20):
return {
"items": [
{
"id": "doc-1",
"knowledge_base_id": knowledge_base_id,
"file_name": "civil-code.md",
"mime_type": "text/markdown",
"file_size": 128,
"status": "indexed",
"metadata": {
"source_type": "statute",
"issuer": "全国人民代表大会",
"effective_date": "2021-01-01",
"jurisdiction": "CN",
"tags": ["民法典", "合同"],
},
}
],
"page": page,
"page_size": page_size,
"total": 1,
}
def upload_document(self, context, knowledge_base_id, *, file_name, mime_type, content):
return {
"id": "doc-2",
"knowledge_base_id": knowledge_base_id,
"file_name": file_name,
"mime_type": mime_type,
"file_size": len(content),
"status": "indexed",
"tags_json": ["制度"],
"metadata_json": {
"source_type": "policy",
"authority": "企业法务部",
"effective_date": "2026-05-01",
"region": "CN-SH",
},
}
def retry_document(self, context, knowledge_base_id, document_id, *, file_name=None, mime_type=None, content=None):
return {
"id": document_id,
"knowledge_base_id": knowledge_base_id,
"file_name": file_name or "retry.txt",
"mime_type": mime_type or "text/plain",
"file_size": len(content or b""),
"status": "indexed",
"metadata": {
"source_type": "judicial-interpretation",
"authority": "最高人民法院",
"effective_date": "2024-02-01",
"region": "CN",
},
}
client = RemoteSharedAIPlatformClient(
base_url="http://testserver",
app_id="test-app",
api_key="test-key",
timeout_seconds=30.0,
)
client.sdk = StubSdk()
current_user = {"id": "user-1", "tenant_id": "tenant-1", "role": "admin"}
listed = client.list_documents(current_user, "kb-1", 1, 20)
uploaded = client.upload_document(current_user, "kb-1", "policy.md", "text/markdown", b"hello")
retried = client.retry_document(current_user, "kb-1", "doc-3", file_name="retry.md", mime_type="text/markdown", content=b"world")
self.assertEqual(listed["items"][0]["source_type"], "statute")
self.assertEqual(listed["items"][0]["authority"], "全国人民代表大会")
self.assertEqual(listed["items"][0]["effective_date"], "2021-01-01")
self.assertEqual(listed["items"][0]["region"], "CN")
self.assertEqual(listed["items"][0]["tags"], ["民法典", "合同"])
self.assertEqual(uploaded["authority"], "企业法务部")
self.assertEqual(uploaded["region"], "CN-SH")
self.assertEqual(uploaded["tags"], ["制度"])
self.assertEqual(retried["source_type"], "judicial-interpretation")
self.assertEqual(retried["authority"], "最高人民法院")
if __name__ == "__main__":
unittest.main()
+60 -6
View File
@@ -2,7 +2,7 @@
import Link from "next/link";
import { InboxOutlined, ReloadOutlined } from "@ant-design/icons";
import { Alert, Button, Card, Empty, Form, Input, List, Space, Table, Tag, Typography, Upload, message } from "antd";
import { Alert, Button, Card, Empty, Form, Input, List, Select, Space, Table, Tag, Typography, Upload, message } from "antd";
import type { UploadFile } from "antd/es/upload/interface";
import { useCallback, useEffect, useMemo, useState } from "react";
@@ -175,6 +175,8 @@ export default function KnowledgeBasesPage() {
const [items, setItems] = useState<KnowledgeBaseItem[]>([]);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [documents, setDocuments] = useState<KnowledgeBaseDocumentItem[]>([]);
const [sourceTypeFilter, setSourceTypeFilter] = useState<string>("all");
const [regionFilter, setRegionFilter] = useState<string>("all");
const [documentsLoading, setDocumentsLoading] = useState(false);
const [fileList, setFileList] = useState<UploadFile[]>([]);
@@ -183,8 +185,16 @@ export default function KnowledgeBasesPage() {
[items, selectedId],
);
const filteredDocuments = useMemo(() => {
return documents.filter((item) => {
const sourceTypeMatch = sourceTypeFilter === "all" || (item.source_type || "unknown") === sourceTypeFilter;
const regionMatch = regionFilter === "all" || (item.region || "unknown") === regionFilter;
return sourceTypeMatch && regionMatch;
});
}, [documents, regionFilter, sourceTypeFilter]);
const orderedDocuments = useMemo(() => {
return [...documents].sort((left, right) => {
return [...filteredDocuments].sort((left, right) => {
const leftPriority = isDocumentFailedLike(left) ? 0 : left.status === "processing" || left.status === "pending" ? 1 : 2;
const rightPriority = isDocumentFailedLike(right) ? 0 : right.status === "processing" || right.status === "pending" ? 1 : 2;
if (leftPriority !== rightPriority) {
@@ -192,13 +202,23 @@ export default function KnowledgeBasesPage() {
}
return new Date(right.created_at).getTime() - new Date(left.created_at).getTime();
});
}, [documents]);
}, [filteredDocuments]);
const failedDocuments = useMemo(
() => orderedDocuments.filter((document) => isDocumentFailedLike(document)),
[orderedDocuments],
);
const sourceTypeOptions = useMemo(() => {
const values = Array.from(new Set(documents.map((item) => item.source_type || "unknown")));
return [{ value: "all", label: "全部来源类型" }, ...values.map((value) => ({ value, label: value }))];
}, [documents]);
const regionOptions = useMemo(() => {
const values = Array.from(new Set(documents.map((item) => item.region || "unknown")));
return [{ value: "all", label: "全部适用区域" }, ...values.map((value) => ({ value, label: value }))];
}, [documents]);
const loadKnowledgeBases = useCallback(async () => {
if (currentUser?.role !== "admin") {
setLoading(false);
@@ -566,9 +586,23 @@ export default function KnowledgeBasesPage() {
className="review-surface-card"
title="文档索引状态"
extra={
<Button icon={<ReloadOutlined />} loading={refreshingDocuments} onClick={() => void handleRefreshDocuments()} disabled={!selectedId}>
刷新状态
</Button>
<Space wrap>
<Select
value={sourceTypeFilter}
style={{ width: 160 }}
options={sourceTypeOptions}
onChange={setSourceTypeFilter}
/>
<Select
value={regionFilter}
style={{ width: 160 }}
options={regionOptions}
onChange={setRegionFilter}
/>
<Button icon={<ReloadOutlined />} loading={refreshingDocuments} onClick={() => void handleRefreshDocuments()} disabled={!selectedId}>
刷新状态
</Button>
</Space>
}
>
{documentsError ? (
@@ -625,6 +659,26 @@ export default function KnowledgeBasesPage() {
width: 120,
render: (value: number) => `${value} B`,
},
{
title: "法源元数据",
key: "source_meta",
width: 260,
render: (_value: unknown, record: KnowledgeBaseDocumentItem) => (
<Space direction="vertical" size={4}>
<Text>{record.source_type || "-"}</Text>
<Text type="secondary">{record.authority || "-"}</Text>
<Text type="secondary">{record.effective_date || "-"}</Text>
<Text type="secondary">{record.region || "-"}</Text>
{record.tags?.length ? (
<Space wrap size={[4, 4]}>
{record.tags.map((tag) => (
<Tag key={tag}>{tag}</Tag>
))}
</Space>
) : null}
</Space>
),
},
{
title: "摘要 / 生命周期说明",
dataIndex: "content_excerpt",
+4
View File
@@ -243,6 +243,10 @@ export type KnowledgeBaseDocumentItem = {
mime_type: string | null;
file_size: number;
status: string;
authority?: string | null;
effective_date?: string | null;
region?: string | null;
tags?: string[] | null;
content_excerpt: string | null;
error_message: string | null;
extracted_text_length: number;
@@ -1117,3 +1117,52 @@
- 当前用户体系还没有独立的 `ops` / `production_approver` 角色
- 运营复核和最终批准目前仍主要由 admin 侧承接
- 正式生产放行仍继续阻塞在真人复核、证据 READY、最终批准签署,以及知识库发布链的 shared-ai 远端能力缺口
## 2026-05-13 GOV-DB-007 知识库法源元数据第一层收口
### 本轮完成项
- 已为本地知识库文档表补齐法源元数据字段:
- `backend/app/models/knowledge_base_document.py`
- 新增 `source_type / authority / effective_date / region / tags_json`
- 已新增数据库迁移:
- `backend/migrations/versions/20260513_0020_add_knowledge_document_source_metadata.py`
- 为 `knowledge_base_documents` 增加上述字段,并补 `source_type`、`region` 索引
- 已补齐知识库页面的法源治理与过滤能力:
- `frontend/src/app/knowledge-bases/page.tsx`
- 文档列表新增“法源元数据”列,展示 `source_type / authority / effective_date / region / tags`
- 页面新增 `source_type`、`region` 两个过滤器,可按法源维度筛选知识库文档
- 已补齐前端类型定义:
- `frontend/src/types/api.ts`
- `KnowledgeBaseDocumentItem` 新增 `authority / effective_date / region / tags`
- 已补强 shared-ai 返回文档元数据的兼容层:
- `backend/app/clients/shared_ai_platform.py`
- 若 shared-ai 先把法源元数据放在 `metadata` 或 `metadata_json` 中,`ai-contract-platform` 现在会自动平铺为 `source_type / authority / effective_date / region / tags`
- `tags_json` 也会自动映射为前端可直接消费的 `tags`
- 已补齐测试桩与兼容性验证:
- `backend/tests/test_knowledge_bases_api.py`
- `backend/tests/test_shared_ai_platform_client.py`
### 本轮验证
- 后端测试:
- `python -m unittest backend.tests.test_knowledge_bases_api backend.tests.test_shared_ai_platform_client`
- 结果:通过
- 后端编译:
- `python -m py_compile backend/app/models/knowledge_base_document.py backend/app/clients/shared_ai_platform.py`
- 结果:通过
- 前端检查:
- `npm --prefix frontend run lint`
- 结果:通过
- `npm --prefix frontend run build`
- 结果:通过,知识库页面构建正常
### 当前状态更新
- `GOV-DB-007` 已从 `PLANNED` 推进到 `IN_PROGRESS`
- `ai-contract-platform` 本仓库侧已经形成第一层闭环:
- 本地表结构已具备法源元数据字段
- shared-ai 返回的知识库文档数据已可在本仓库侧平铺、展示和过滤
- 管理员已可在知识库页面按法源维度治理和筛查文档
### 当前阻塞项
- `GOV-DB-007` 仍未宣告 `COMPLETED`,原因不是本仓库缺页面或字段,而是 shared-ai 远端当前还没有正式生产化的知识库法源元数据写入链:
- 远端知识库文档真实数据是否持续提供 `source_type / authority / effective_date / region / tags`,仍依赖 shared-ai 知识库导入/同步链配合
- 当 shared-ai 远端稳定产出这些字段后,本项即可自然收口为 `COMPLETED`
@@ -5,7 +5,7 @@ batch,priority,area,task_id,title,current_gap,target_deliverable,depends_on,acce
5A,P0,database,GOV-DB-004,新增证据登记表,当前证据路径和证据状态只存在文档与脚本规则中,新增 governance_evidences 表支持 file_id url status checksum reviewer metadata,GOV-DB-001,系统可校验证据是否存在且不是模板占位,COMPLETED
5A,P0,database,GOV-DB-005,新增发布批次与门禁结果表,当前上线批准状态只存在 PowerShell 产物中,新增 governance_release_batches 和 governance_gate_runs 表,GOV-DB-001;GOV-DB-003;GOV-DB-004,系统可追踪一次发布批次的门禁快照和结果,COMPLETED
5A,P1,database,GOV-DB-006,模板表补治理字段,contract_templates 只有内容和启停没有版本与发布元数据,为 contract_templates 增加 template_code version published_at published_by latest_governance_asset_id,GOV-DB-001,模板可区分草稿版和生产发布版,COMPLETED
5A,P1,database,GOV-DB-007,知识库文档表补法源元数据,知识库文档缺少效力级别生效日期发文机关和法源类型,为知识库相关表增加 source_type authority effective_date region tags,none,知识库可按法源维度治理和过滤,PLANNED
5A,P1,database,GOV-DB-007,知识库文档表补法源元数据,知识库文档缺少效力级别生效日期发文机关和法源类型,为知识库相关表增加 source_type authority effective_date region tags,none,知识库可按法源维度治理和过滤,IN_PROGRESS
5B,P0,backend_api,GOV-API-001,新增治理资产列表接口,当前没有统一治理视图接口,新增 /governance/assets 列表筛选汇总接口,GOV-DB-001,前端可按资产类型状态角色查看治理资产,COMPLETED
5B,P0,backend_api,GOV-API-002,新增复核分配接口,当前无法在系统内分配真实复核人,新增 /governance/assignments 创建更新撤销接口,GOV-DB-002,管理员可在系统内完成复核任务分配,COMPLETED
5B,P0,backend_api,GOV-API-003,新增复核回填接口,当前复核结论只能通过CSV和脚本回填,新增 /governance/decisions 提交接口和批量导入接口,GOV-DB-003,真实复核人或管理员可提交复核结果,COMPLETED
1 batch priority area task_id title current_gap target_deliverable depends_on acceptance status
5 5A P0 database GOV-DB-004 新增证据登记表 当前证据路径和证据状态只存在文档与脚本规则中 新增 governance_evidences 表支持 file_id url status checksum reviewer metadata GOV-DB-001 系统可校验证据是否存在且不是模板占位 COMPLETED
6 5A P0 database GOV-DB-005 新增发布批次与门禁结果表 当前上线批准状态只存在 PowerShell 产物中 新增 governance_release_batches 和 governance_gate_runs 表 GOV-DB-001;GOV-DB-003;GOV-DB-004 系统可追踪一次发布批次的门禁快照和结果 COMPLETED
7 5A P1 database GOV-DB-006 模板表补治理字段 contract_templates 只有内容和启停没有版本与发布元数据 为 contract_templates 增加 template_code version published_at published_by latest_governance_asset_id GOV-DB-001 模板可区分草稿版和生产发布版 COMPLETED
8 5A P1 database GOV-DB-007 知识库文档表补法源元数据 知识库文档缺少效力级别生效日期发文机关和法源类型 为知识库相关表增加 source_type authority effective_date region tags none 知识库可按法源维度治理和过滤 PLANNED IN_PROGRESS
9 5B P0 backend_api GOV-API-001 新增治理资产列表接口 当前没有统一治理视图接口 新增 /governance/assets 列表筛选汇总接口 GOV-DB-001 前端可按资产类型状态角色查看治理资产 COMPLETED
10 5B P0 backend_api GOV-API-002 新增复核分配接口 当前无法在系统内分配真实复核人 新增 /governance/assignments 创建更新撤销接口 GOV-DB-002 管理员可在系统内完成复核任务分配 COMPLETED
11 5B P0 backend_api GOV-API-003 新增复核回填接口 当前复核结论只能通过CSV和脚本回填 新增 /governance/decisions 提交接口和批量导入接口 GOV-DB-003 真实复核人或管理员可提交复核结果 COMPLETED