From 5fd02c6ba0c2b5737c5fa886bc4c523d98414320 Mon Sep 17 00:00:00 2001 From: yuqei Date: Thu, 14 May 2026 00:17:09 +0800 Subject: [PATCH] feat: complete knowledge source metadata loop --- backend/app/api/knowledge_bases.py | 187 +++++++++++++++++- .../knowledge_base_document_repository.py | 10 + backend/tests/test_knowledge_bases_api.py | 80 ++++++++ frontend/src/app/knowledge-bases/page.tsx | 34 +++- frontend/src/services/knowledge-base.ts | 46 ++++- ...草与审查助手-当前工作进度保存-2026-05-11.md | 34 ++-- .../governance-productization-backlog.csv | 2 +- scripts/import-phase13-library.ps1 | 68 ++++++- 8 files changed, 433 insertions(+), 28 deletions(-) diff --git a/backend/app/api/knowledge_bases.py b/backend/app/api/knowledge_bases.py index 930de60e..c8a945e6 100644 --- a/backend/app/api/knowledge_bases.py +++ b/backend/app/api/knowledge_bases.py @@ -1,12 +1,15 @@ -from typing import Optional +import json +from typing import Any, Optional -from fastapi import APIRouter, Depends, File, Form, Request, UploadFile +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile from sqlalchemy.orm import Session from app.api.reviews import require_current_user from app.clients.shared_ai_platform import SharedAIPlatformClient, get_shared_ai_platform_client from app.core.response import success_response from app.db import get_db_session +from app.models.knowledge_base_document import KnowledgeBaseDocument +from app.repositories.knowledge_base_document_repository import KnowledgeBaseDocumentRepository from app.services.audit_log_service import AuditLogService router = APIRouter(prefix="/knowledge-bases", tags=["knowledge-bases"]) @@ -24,6 +27,121 @@ def get_audit_log_service(db: Session = Depends(get_db_session)) -> AuditLogServ return AuditLogService(db) +def _parse_tags_json(tags_json: Optional[str]) -> Optional[list[str]]: + if tags_json is None or not str(tags_json).strip(): + return None + try: + payload = json.loads(tags_json) + except json.JSONDecodeError as exc: + raise HTTPException(status_code=400, detail="tags_json must be a valid JSON array") from exc + if not isinstance(payload, list) or not all(isinstance(item, str) for item in payload): + raise HTTPException(status_code=400, detail="tags_json must be a JSON array of strings") + normalized = [item.strip() for item in payload if item and item.strip()] + return normalized or None + + +def _build_metadata_override( + *, + source_type: Optional[str], + authority: Optional[str], + effective_date: Optional[str], + region: Optional[str], + tags: Optional[list[str]], +) -> dict[str, Any]: + payload: dict[str, Any] = {} + if source_type and source_type.strip(): + payload["source_type"] = source_type.strip() + if authority and authority.strip(): + payload["authority"] = authority.strip() + if effective_date and effective_date.strip(): + payload["effective_date"] = effective_date.strip() + if region and region.strip(): + payload["region"] = region.strip() + if tags is not None: + payload["tags"] = tags + return payload + + +def _merge_local_document_overlay(remote_item: dict[str, Any], local_item: KnowledgeBaseDocument) -> dict[str, Any]: + merged = dict(remote_item) + if local_item.source_type: + merged["source_type"] = local_item.source_type + if local_item.authority: + merged["authority"] = local_item.authority + if local_item.effective_date: + merged["effective_date"] = local_item.effective_date + if local_item.region: + merged["region"] = local_item.region + if local_item.tags_json is not None: + merged["tags"] = local_item.tags_json + merged["tags_json"] = local_item.tags_json + return merged + + +def _upsert_local_document_overlay( + db: Session, + *, + current_user: dict, + remote_item: dict[str, Any], + metadata_override: Optional[dict[str, Any]] = None, +) -> dict[str, Any]: + repository = KnowledgeBaseDocumentRepository(db) + tenant_id = str(current_user.get("tenant_id") or "") + knowledge_base_id = str(remote_item.get("knowledge_base_id") or "") + document_id = str(remote_item.get("id") or "") + if not tenant_id or not knowledge_base_id or not document_id: + return remote_item + + existing = repository.get_by_id(document_id, tenant_id=tenant_id, knowledge_base_id=knowledge_base_id) + overlay = metadata_override or {} + + source_type = overlay.get("source_type") or remote_item.get("source_type") or (existing.source_type if existing else None) + authority = overlay.get("authority") or remote_item.get("authority") or (existing.authority if existing else None) + effective_date = overlay.get("effective_date") or remote_item.get("effective_date") or (existing.effective_date if existing else None) + region = overlay.get("region") or remote_item.get("region") or (existing.region if existing else None) + tags = overlay.get("tags") + if tags is None: + tags = remote_item.get("tags") + if tags is None: + tags = remote_item.get("tags_json") + if tags is None and existing is not None: + tags = existing.tags_json + + if existing is None: + existing = KnowledgeBaseDocument( + id=document_id, + knowledge_base_id=knowledge_base_id, + tenant_id=tenant_id, + file_name=str(remote_item.get("file_name") or ""), + mime_type=remote_item.get("mime_type"), + file_size=int(remote_item.get("file_size") or 0), + status=str(remote_item.get("status") or "indexed"), + source_type=source_type, + authority=authority, + effective_date=effective_date, + region=region, + tags_json=tags, + content_excerpt=remote_item.get("content_excerpt"), + error_message=remote_item.get("error_message"), + ) + repository.create(existing) + else: + existing.file_name = str(remote_item.get("file_name") or existing.file_name) + existing.mime_type = remote_item.get("mime_type") or existing.mime_type + existing.file_size = int(remote_item.get("file_size") or existing.file_size or 0) + existing.status = str(remote_item.get("status") or existing.status or "indexed") + existing.source_type = source_type + existing.authority = authority + existing.effective_date = effective_date + existing.region = region + existing.tags_json = tags + existing.content_excerpt = remote_item.get("content_excerpt") + existing.error_message = remote_item.get("error_message") + repository.update(existing) + + return _merge_local_document_overlay(remote_item, existing) + + @router.get("") def list_knowledge_bases( page: int = 1, @@ -66,8 +184,23 @@ def list_knowledge_base_documents( page_size: int = 20, shared_ai_platform_client: SharedAIPlatformClient = Depends(get_shared_ai_platform_client), current_user: dict = Depends(require_current_user), + db: Session = Depends(get_db_session), ): - return success_response(shared_ai_platform_client.list_documents(current_user, knowledge_base_id, page, page_size)) + payload = shared_ai_platform_client.list_documents(current_user, knowledge_base_id, page, page_size) + items = payload.get("items", []) + if not isinstance(items, list) or not items: + return success_response(payload) + + repository = KnowledgeBaseDocumentRepository(db) + document_ids = [str(item.get("id") or "") for item in items if item.get("id")] + overlays = repository.list_by_ids(str(current_user.get("tenant_id") or ""), document_ids) + overlay_map = {overlay.id: overlay for overlay in overlays} + + payload["items"] = [ + _merge_local_document_overlay(item, overlay_map[item["id"]]) if item.get("id") in overlay_map else item + for item in items + ] + return success_response(payload) @router.post("/{knowledge_base_id}/documents") @@ -76,12 +209,31 @@ async def upload_knowledge_base_document( request: Request, file: UploadFile = File(...), note: Optional[str] = Form(default=None), + source_type: Optional[str] = Form(default=None), + authority: Optional[str] = Form(default=None), + effective_date: Optional[str] = Form(default=None), + region: Optional[str] = Form(default=None), + tags_json: Optional[str] = Form(default=None), shared_ai_platform_client: SharedAIPlatformClient = Depends(get_shared_ai_platform_client), audit_log_service: AuditLogService = Depends(get_audit_log_service), current_user: dict = Depends(require_current_user), + db: Session = Depends(get_db_session), ): + parsed_tags = _parse_tags_json(tags_json) content = await file.read() result = shared_ai_platform_client.upload_document(current_user, knowledge_base_id, file.filename, file.content_type, content) + result = _upsert_local_document_overlay( + db, + current_user=current_user, + remote_item=result, + metadata_override=_build_metadata_override( + source_type=source_type, + authority=authority, + effective_date=effective_date, + region=region, + tags=parsed_tags, + ), + ) audit_log_service.record_event( tenant_id=current_user.get("tenant_id"), user_id=current_user.get("id"), @@ -99,6 +251,11 @@ async def upload_knowledge_base_document( "index_status": result.get("index_status"), "error_message": result.get("error_message"), "note": note, + "source_type": result.get("source_type"), + "authority": result.get("authority"), + "effective_date": result.get("effective_date"), + "region": result.get("region"), + "tags": result.get("tags"), }, ip_address=request.client.host if request.client else None, user_agent=request.headers.get("user-agent"), @@ -113,10 +270,17 @@ async def retry_knowledge_base_document( request: Request, file: Optional[UploadFile] = File(default=None), note: Optional[str] = Form(default=None), + source_type: Optional[str] = Form(default=None), + authority: Optional[str] = Form(default=None), + effective_date: Optional[str] = Form(default=None), + region: Optional[str] = Form(default=None), + tags_json: Optional[str] = Form(default=None), shared_ai_platform_client: SharedAIPlatformClient = Depends(get_shared_ai_platform_client), audit_log_service: AuditLogService = Depends(get_audit_log_service), current_user: dict = Depends(require_current_user), + db: Session = Depends(get_db_session), ): + parsed_tags = _parse_tags_json(tags_json) content = await file.read() if file is not None else None result = shared_ai_platform_client.retry_document( current_user=current_user, @@ -126,6 +290,18 @@ async def retry_knowledge_base_document( mime_type=file.content_type if file is not None else None, content=content, ) + result = _upsert_local_document_overlay( + db, + current_user=current_user, + remote_item=result, + metadata_override=_build_metadata_override( + source_type=source_type, + authority=authority, + effective_date=effective_date, + region=region, + tags=parsed_tags, + ), + ) audit_log_service.record_event( tenant_id=current_user.get("tenant_id"), user_id=current_user.get("id"), @@ -145,6 +321,11 @@ async def retry_knowledge_base_document( "error_message": result.get("error_message"), "replacement_file": file is not None, "note": note, + "source_type": result.get("source_type"), + "authority": result.get("authority"), + "effective_date": result.get("effective_date"), + "region": result.get("region"), + "tags": result.get("tags"), }, ip_address=request.client.host if request.client else None, user_agent=request.headers.get("user-agent"), diff --git a/backend/app/repositories/knowledge_base_document_repository.py b/backend/app/repositories/knowledge_base_document_repository.py index 312b5305..a7bdb988 100644 --- a/backend/app/repositories/knowledge_base_document_repository.py +++ b/backend/app/repositories/knowledge_base_document_repository.py @@ -52,6 +52,16 @@ class KnowledgeBaseDocumentRepository: query = query.filter(KnowledgeBaseDocument.knowledge_base_id == knowledge_base_id) return query.first() + def list_by_ids(self, tenant_id: str, document_ids: list[str]) -> List[KnowledgeBaseDocument]: + if not document_ids: + return [] + return ( + self.db.query(KnowledgeBaseDocument) + .filter(KnowledgeBaseDocument.tenant_id == tenant_id) + .filter(KnowledgeBaseDocument.id.in_(document_ids)) + .all() + ) + def count_by_knowledge_base_ids(self, tenant_id: str, knowledge_base_ids: list[str]) -> dict[str, int]: if not knowledge_base_ids: return {} diff --git a/backend/tests/test_knowledge_bases_api.py b/backend/tests/test_knowledge_bases_api.py index 0919f430..68476c63 100644 --- a/backend/tests/test_knowledge_bases_api.py +++ b/backend/tests/test_knowledge_bases_api.py @@ -2,8 +2,12 @@ import unittest from fastapi import HTTPException from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import StaticPool from app.api.knowledge_bases import get_audit_log_service, get_shared_ai_platform_client, require_current_user +from app.db import Base, get_db_session from app.main import app @@ -167,12 +171,31 @@ class StubKnowledgeBaseService: class KnowledgeBasesApiTestCase(unittest.TestCase): def setUp(self): self.audit_log_service = StubAuditLogService() + self.engine = create_engine( + "sqlite://", + future=True, + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + self.testing_session_local = sessionmaker(autocommit=False, autoflush=False, bind=self.engine) + Base.metadata.create_all(bind=self.engine) + + def override_db(): + db = self.testing_session_local() + try: + yield db + finally: + db.close() + + app.dependency_overrides[get_db_session] = override_db app.dependency_overrides[get_shared_ai_platform_client] = lambda: StubKnowledgeBaseService() app.dependency_overrides[get_audit_log_service] = lambda: self.audit_log_service self.client = TestClient(app) def tearDown(self): app.dependency_overrides.clear() + Base.metadata.drop_all(bind=self.engine) + self.engine.dispose() def test_admin_can_list_knowledge_bases(self): app.dependency_overrides[require_current_user] = lambda: { @@ -262,6 +285,11 @@ class KnowledgeBasesApiTestCase(unittest.TestCase): "index_status": "completed", "error_message": None, "note": "首批规则", + "source_type": None, + "authority": None, + "effective_date": None, + "region": None, + "tags": None, }, ) @@ -298,9 +326,61 @@ class KnowledgeBasesApiTestCase(unittest.TestCase): "error_message": None, "replacement_file": True, "note": "替换为空文档", + "source_type": None, + "authority": None, + "effective_date": None, + "region": None, + "tags": None, }, ) + def test_uploaded_metadata_is_persisted_and_list_overlay_returns_it(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-admin", + "tenant_id": "tenant-demo-001", + "role": "admin", + } + + upload_response = self.client.post( + "/api/v1/knowledge-bases/kb-123/documents", + data={ + "note": "带法源元数据", + "source_type": "法律", + "authority": "全国人民代表大会", + "effective_date": "2021-01-01", + "region": "CN", + "tags_json": '["民法典","合同"]', + }, + files={"file": ("rules.txt", "合同解除规则".encode("utf-8"), "text/plain")}, + ) + + self.assertEqual(upload_response.status_code, 200) + + listed = self.client.get("/api/v1/knowledge-bases/kb-123/documents") + self.assertEqual(listed.status_code, 200) + item = listed.json()["data"]["items"][0] + self.assertEqual(item["source_type"], "法律") + self.assertEqual(item["authority"], "全国人民代表大会") + self.assertEqual(item["effective_date"], "2021-01-01") + self.assertEqual(item["region"], "CN") + self.assertEqual(item["tags"], ["民法典", "合同"]) + + def test_invalid_tags_json_is_rejected(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-admin", + "tenant_id": "tenant-demo-001", + "role": "admin", + } + + response = self.client.post( + "/api/v1/knowledge-bases/kb-123/documents", + data={"tags_json": '{"bad":"shape"}'}, + files={"file": ("rules.txt", "abc".encode("utf-8"), "text/plain")}, + ) + + self.assertEqual(response.status_code, 400) + self.assertEqual(response.json()["message"], "tags_json must be a JSON array of strings") + def test_member_cannot_access_knowledge_bases(self): app.dependency_overrides[require_current_user] = lambda: { "id": "user-member", diff --git a/frontend/src/app/knowledge-bases/page.tsx b/frontend/src/app/knowledge-bases/page.tsx index 8515b94b..e1c4f6f6 100644 --- a/frontend/src/app/knowledge-bases/page.tsx +++ b/frontend/src/app/knowledge-bases/page.tsx @@ -17,6 +17,11 @@ type KnowledgeBaseFormValues = { name: string; description?: string; note?: string; + source_type?: string; + authority?: string; + effective_date?: string; + region?: string; + tags?: string; }; const DOCUMENT_STATUS_META: Record = { @@ -361,7 +366,17 @@ export default function KnowledgeBasesPage() { setUploading(true); try { const note = uploadForm.getFieldValue("note")?.trim(); - const result = await uploadKnowledgeBaseDocument(selectedId, selectedFile, note); + const metadata = { + source_type: uploadForm.getFieldValue("source_type")?.trim() || undefined, + authority: uploadForm.getFieldValue("authority")?.trim() || undefined, + effective_date: uploadForm.getFieldValue("effective_date")?.trim() || undefined, + region: uploadForm.getFieldValue("region")?.trim() || undefined, + tags: ((uploadForm.getFieldValue("tags") as string | undefined) || "") + .split(/[;,，；]/) + .map((item) => item.trim()) + .filter(Boolean), + }; + const result = await uploadKnowledgeBaseDocument(selectedId, selectedFile, note, metadata); if (result.code !== 0) { messageApi.error(result.message || "上传文档失败"); return; @@ -372,7 +387,7 @@ export default function KnowledgeBasesPage() { : result.data.error_message || "文档上传完成，但当前未能完成入库", ); setFileList([]); - uploadForm.resetFields(["note"]); + uploadForm.resetFields(["note", "source_type", "authority", "effective_date", "region", "tags"]); await Promise.all([loadKnowledgeBases(), loadDocuments(selectedId)]); } catch { messageApi.error("上传文档失败，请稍后重试"); @@ -520,6 +535,21 @@ export default function KnowledgeBasesPage() { + + + + + + + + + + + + + + +