diff --git a/backend/app/api/legal_articles.py b/backend/app/api/legal_articles.py new file mode 100644 index 00000000..634542c5 --- /dev/null +++ b/backend/app/api/legal_articles.py @@ -0,0 +1,166 @@ +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Request +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.api.reviews import require_current_user +from app.core.response import success_response +from app.db import get_db_session +from app.services.audit_log_service import AuditLogService +from app.services.legal_article_service import LegalArticleService + +router = APIRouter(prefix="/legal-articles", tags=["legal-articles"]) +ROUTE_PERMISSION_SCOPE = "legal_article_library" +ROUTE_PERMISSION_ACTIONS = { + "list": "legal_articles:list", + "detail": "legal_articles:detail", + "bulk_upsert": "legal_articles:bulk_upsert", + "index_status": "legal_articles:index_status:update", +} + + +class LegalArticlePayload(BaseModel): + id: Optional[str] = None + source_id: str + source_title: str + source_type: str = "法律" + authority: str + effective_date: Optional[str] = None + validity_status: Optional[str] = None + region: str = "CN" + official_url: Optional[str] = None + article_no: str + chapter_title: Optional[str] = None + section_title: Optional[str] = None + article_text: str + tags: list[str] = Field(default_factory=list) + source_checksum: Optional[str] = None + text_checksum: Optional[str] = None + knowledge_base_id: Optional[str] = None + knowledge_document_id: Optional[str] = None + index_status: str = "pending" + published_status: str = "draft" + published_at: Optional[str] = None + published_by: Optional[str] = None + + +class BulkUpsertLegalArticlesRequest(BaseModel): + articles: list[LegalArticlePayload] + + +class UpdateLegalArticleIndexStatusRequest(BaseModel): + index_status: str + knowledge_base_id: Optional[str] = None + knowledge_document_id: Optional[str] = None + published_status: Optional[str] = None + + +def get_legal_article_service(db: Session = Depends(get_db_session)) -> LegalArticleService: + return LegalArticleService(db) + + +def get_audit_log_service(db: Session = Depends(get_db_session)) -> AuditLogService: + return AuditLogService(db) + + +@router.get("") +def list_legal_articles( + page: int = 1, + page_size: int = 20, + keyword: Optional[str] = None, + source_id: Optional[str] = None, + article_no: Optional[str] = None, + index_status: Optional[str] = None, + published_status: Optional[str] = None, + legal_article_service: LegalArticleService = Depends(get_legal_article_service), + current_user: dict = Depends(require_current_user), +): + result = legal_article_service.list_articles( + current_user, + page, + page_size, + keyword, + source_id, + article_no, + index_status, + published_status, + ) + return success_response(result) + + +@router.get("/{article_id}") +def get_legal_article( + article_id: str, + legal_article_service: LegalArticleService = Depends(get_legal_article_service), + current_user: dict = Depends(require_current_user), +): + result = legal_article_service.get_article(current_user, article_id) + if result is None: + raise HTTPException(status_code=404, detail="legal article not found") + return success_response(result) + + +@router.post("/bulk-upsert") +def bulk_upsert_legal_articles( + payload: BulkUpsertLegalArticlesRequest, + request: Request, + legal_article_service: LegalArticleService = Depends(get_legal_article_service), + audit_log_service: AuditLogService = Depends(get_audit_log_service), + current_user: dict = Depends(require_current_user), +): + articles_payload: list[dict[str, Any]] = [article.model_dump(mode="json") for article in payload.articles] + result = legal_article_service.bulk_upsert_articles(current_user, articles_payload) + audit_log_service.record_event( + tenant_id=current_user.get("tenant_id"), + user_id=current_user.get("id"), + action="bulk_upsert_legal_articles", + resource_type="legal_article", + resource_id="phase14-bulk-upsert", + detail_json={ + "created_count": result["created_count"], + "updated_count": result["updated_count"], + "total": result["total"], + "source_ids": sorted({article["source_id"] for article in articles_payload}), + }, + ip_address=request.client.host if request.client else None, + user_agent=request.headers.get("user-agent"), + ) + return success_response(result) + + +@router.post("/{article_id}/index-status") +def update_legal_article_index_status( + article_id: str, + payload: UpdateLegalArticleIndexStatusRequest, + request: Request, + legal_article_service: LegalArticleService = Depends(get_legal_article_service), + audit_log_service: AuditLogService = Depends(get_audit_log_service), + current_user: dict = Depends(require_current_user), +): + result = legal_article_service.update_index_status( + current_user, + article_id, + index_status=payload.index_status, + knowledge_base_id=payload.knowledge_base_id, + knowledge_document_id=payload.knowledge_document_id, + published_status=payload.published_status, + ) + if result is None: + raise HTTPException(status_code=404, detail="legal article not found") + audit_log_service.record_event( + tenant_id=current_user.get("tenant_id"), + user_id=current_user.get("id"), + action="update_legal_article_index_status", + resource_type="legal_article", + resource_id=article_id, + detail_json={ + "index_status": result["index_status"], + "published_status": result["published_status"], + "knowledge_base_id": result["knowledge_base_id"], + "knowledge_document_id": result["knowledge_document_id"], + }, + ip_address=request.client.host if request.client else None, + user_agent=request.headers.get("user-agent"), + ) + return success_response(result) diff --git a/backend/app/main.py b/backend/app/main.py index b58cfc23..c6f31eee 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -10,6 +10,7 @@ from app.api.conversations import router as conversations_router from app.api.files import router as files_router from app.api.governance import router as governance_router from app.api.knowledge_bases import router as knowledge_bases_router +from app.api.legal_articles import router as legal_articles_router from app.api.pilot_requests import router as pilot_requests_router from app.api.review_rules import router as review_rules_router from app.api.reviews import router as reviews_router @@ -38,6 +39,7 @@ app.include_router(conversations_router, prefix=settings.api_v1_prefix) app.include_router(files_router, prefix=settings.api_v1_prefix) app.include_router(governance_router, prefix=settings.api_v1_prefix) app.include_router(knowledge_bases_router, prefix=settings.api_v1_prefix) +app.include_router(legal_articles_router, prefix=settings.api_v1_prefix) app.include_router(pilot_requests_router, prefix=settings.api_v1_prefix) app.include_router(review_rules_router, prefix=settings.api_v1_prefix) app.include_router(templates_router, prefix=settings.api_v1_prefix) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 0078c5ee..83b34922 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -12,6 +12,7 @@ from app.models.governance_review_assignment import GovernanceReviewAssignment from app.models.governance_review_decision import GovernanceReviewDecision from app.models.knowledge_base import KnowledgeBase from app.models.knowledge_base_document import KnowledgeBaseDocument +from app.models.legal_article import LegalArticle from app.models.message import Message from app.models.pilot_request import PilotRequest from app.models.review_result import ReviewResult diff --git a/backend/app/models/legal_article.py b/backend/app/models/legal_article.py new file mode 100644 index 00000000..1f048e39 --- /dev/null +++ b/backend/app/models/legal_article.py @@ -0,0 +1,42 @@ +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import JSON, DateTime, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from app.db import Base + + +class LegalArticle(Base): + __tablename__ = "legal_articles" + + id: Mapped[str] = mapped_column(String(96), primary_key=True) + tenant_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + source_id: Mapped[str] = mapped_column(String(96), nullable=False, index=True) + source_title: Mapped[str] = mapped_column(String(255), nullable=False) + source_type: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + authority: Mapped[str] = mapped_column(String(255), nullable=False) + effective_date: Mapped[Optional[str]] = mapped_column(String(128), nullable=True) + validity_status: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + region: Mapped[str] = mapped_column(String(64), default="CN", index=True) + official_url: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + article_no: Mapped[str] = mapped_column(String(64), nullable=False, index=True) + chapter_title: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + section_title: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + article_text: Mapped[str] = mapped_column(Text, nullable=False) + tags_json: Mapped[Optional[list[str]]] = mapped_column(JSON, nullable=True) + source_checksum: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + text_checksum: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) + knowledge_base_id: Mapped[Optional[str]] = mapped_column(String(96), nullable=True, index=True) + knowledge_document_id: Mapped[Optional[str]] = mapped_column(String(96), nullable=True, index=True) + index_status: Mapped[str] = mapped_column(String(32), default="pending", index=True) + published_status: Mapped[str] = mapped_column(String(32), default="draft", index=True) + published_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True), nullable=True) + published_by: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), index=True) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), + default=lambda: datetime.now(timezone.utc), + onupdate=lambda: datetime.now(timezone.utc), + index=True, + ) diff --git a/backend/app/repositories/legal_article_repository.py b/backend/app/repositories/legal_article_repository.py new file mode 100644 index 00000000..91d21074 --- /dev/null +++ b/backend/app/repositories/legal_article_repository.py @@ -0,0 +1,82 @@ +from typing import Optional + +from sqlalchemy import or_ +from sqlalchemy.orm import Session + +from app.models.legal_article import LegalArticle + + +class LegalArticleRepository: + def __init__(self, db: Session): + self.db = db + + def list_paginated( + self, + tenant_id: str, + page: int, + page_size: int, + keyword: Optional[str] = None, + source_id: Optional[str] = None, + article_no: Optional[str] = None, + index_status: Optional[str] = None, + published_status: Optional[str] = None, + ) -> tuple[list[LegalArticle], int]: + query = self.db.query(LegalArticle).filter(LegalArticle.tenant_id == tenant_id) + query = self._apply_filters(query, keyword, source_id, article_no, index_status, published_status) + total = query.count() + items = ( + query.order_by(LegalArticle.source_id.asc(), LegalArticle.article_no.asc(), LegalArticle.updated_at.desc()) + .offset((page - 1) * page_size) + .limit(page_size) + .all() + ) + return items, total + + def get_by_id(self, article_id: str, tenant_id: str) -> Optional[LegalArticle]: + return ( + self.db.query(LegalArticle) + .filter(LegalArticle.id == article_id) + .filter(LegalArticle.tenant_id == tenant_id) + .first() + ) + + def get_by_source_article(self, tenant_id: str, source_id: str, article_no: str) -> Optional[LegalArticle]: + return ( + self.db.query(LegalArticle) + .filter(LegalArticle.tenant_id == tenant_id) + .filter(LegalArticle.source_id == source_id) + .filter(LegalArticle.article_no == article_no) + .first() + ) + + def create(self, article: LegalArticle) -> LegalArticle: + self.db.add(article) + self.db.commit() + self.db.refresh(article) + return article + + def update(self, article: LegalArticle) -> LegalArticle: + self.db.commit() + self.db.refresh(article) + return article + + @staticmethod + def _apply_filters(query, keyword, source_id, article_no, index_status, published_status): + if keyword: + like = f"%{keyword.strip()}%" + query = query.filter( + or_( + LegalArticle.source_title.ilike(like), + LegalArticle.article_no.ilike(like), + LegalArticle.article_text.ilike(like), + ) + ) + if source_id: + query = query.filter(LegalArticle.source_id == source_id.strip()) + if article_no: + query = query.filter(LegalArticle.article_no == article_no.strip()) + if index_status: + query = query.filter(LegalArticle.index_status == index_status.strip()) + if published_status: + query = query.filter(LegalArticle.published_status == published_status.strip()) + return query diff --git a/backend/app/services/legal_article_service.py b/backend/app/services/legal_article_service.py new file mode 100644 index 00000000..758a847b --- /dev/null +++ b/backend/app/services/legal_article_service.py @@ -0,0 +1,241 @@ +import hashlib +from datetime import datetime, timezone +from typing import Any, Optional + +from fastapi import HTTPException +from sqlalchemy.orm import Session + +from app.models.legal_article import LegalArticle +from app.repositories.legal_article_repository import LegalArticleRepository + + +class LegalArticleService: + def __init__(self, db: Session): + self.db = db + self.repository = LegalArticleRepository(db) + + def list_articles( + self, + current_user: dict, + page: int, + page_size: int, + keyword: Optional[str] = None, + source_id: Optional[str] = None, + article_no: Optional[str] = None, + index_status: Optional[str] = None, + published_status: Optional[str] = None, + ) -> dict: + self._require_admin(current_user) + normalized_page = max(page, 1) + normalized_page_size = min(max(page_size, 1), 200) + items, total = self.repository.list_paginated( + tenant_id=current_user["tenant_id"], + page=normalized_page, + page_size=normalized_page_size, + keyword=keyword, + source_id=source_id, + article_no=article_no, + index_status=index_status, + published_status=published_status, + ) + return { + "items": [self._to_dict(item) for item in items], + "page": normalized_page, + "page_size": normalized_page_size, + "total": total, + } + + def get_article(self, current_user: dict, article_id: str) -> Optional[dict]: + self._require_admin(current_user) + article = self.repository.get_by_id(article_id, current_user["tenant_id"]) + return self._to_dict(article) if article is not None else None + + def bulk_upsert_articles(self, current_user: dict, articles: list[dict[str, Any]]) -> dict: + self._require_admin(current_user) + created: list[dict] = [] + updated: list[dict] = [] + for payload in articles: + result, was_created = self.upsert_article(current_user, payload) + if was_created: + created.append(result) + else: + updated.append(result) + return { + "created_count": len(created), + "updated_count": len(updated), + "total": len(created) + len(updated), + "created": created, + "updated": updated, + } + + def upsert_article(self, current_user: dict, payload: dict[str, Any]) -> tuple[dict, bool]: + self._require_admin(current_user) + tenant_id = current_user["tenant_id"] + source_id = self._required_text(payload, "source_id") + article_no = self._required_text(payload, "article_no") + article_text = self._required_text(payload, "article_text") + existing = self.repository.get_by_source_article(tenant_id, source_id, article_no) + text_checksum = payload.get("text_checksum") or self._checksum(article_text) + + if existing is None: + article = LegalArticle( + id=payload.get("id") or self._build_article_id(tenant_id, source_id, article_no), + tenant_id=tenant_id, + source_id=source_id, + source_title=self._required_text(payload, "source_title"), + source_type=self._text_or_default(payload.get("source_type"), "法律"), + authority=self._text_or_default(payload.get("authority"), "待核验权威来源"), + effective_date=self._optional_text(payload.get("effective_date")), + validity_status=self._optional_text(payload.get("validity_status")), + region=self._text_or_default(payload.get("region"), "CN"), + official_url=self._optional_text(payload.get("official_url")), + article_no=article_no, + chapter_title=self._optional_text(payload.get("chapter_title")), + section_title=self._optional_text(payload.get("section_title")), + article_text=article_text, + tags_json=self._normalize_tags(payload.get("tags") or payload.get("tags_json")), + source_checksum=self._optional_text(payload.get("source_checksum")), + text_checksum=text_checksum, + knowledge_base_id=self._optional_text(payload.get("knowledge_base_id")), + knowledge_document_id=self._optional_text(payload.get("knowledge_document_id")), + index_status=self._text_or_default(payload.get("index_status"), "pending"), + published_status=self._text_or_default(payload.get("published_status"), "draft"), + published_at=self._parse_datetime(payload.get("published_at")), + published_by=self._optional_text(payload.get("published_by")) or current_user.get("id"), + ) + created = self.repository.create(article) + return self._to_dict(created), True + + existing.source_title = self._required_text(payload, "source_title") + existing.source_type = self._text_or_default(payload.get("source_type"), existing.source_type) + existing.authority = self._text_or_default(payload.get("authority"), existing.authority) + existing.effective_date = self._optional_text(payload.get("effective_date")) + existing.validity_status = self._optional_text(payload.get("validity_status")) + existing.region = self._text_or_default(payload.get("region"), existing.region) + existing.official_url = self._optional_text(payload.get("official_url")) + existing.chapter_title = self._optional_text(payload.get("chapter_title")) + existing.section_title = self._optional_text(payload.get("section_title")) + existing.article_text = article_text + existing.tags_json = self._normalize_tags(payload.get("tags") or payload.get("tags_json")) + existing.source_checksum = self._optional_text(payload.get("source_checksum")) + existing.text_checksum = text_checksum + existing.knowledge_base_id = self._optional_text(payload.get("knowledge_base_id")) + existing.knowledge_document_id = self._optional_text(payload.get("knowledge_document_id")) + existing.index_status = self._text_or_default(payload.get("index_status"), existing.index_status) + existing.published_status = self._text_or_default(payload.get("published_status"), existing.published_status) + existing.published_at = self._parse_datetime(payload.get("published_at")) + existing.published_by = self._optional_text(payload.get("published_by")) or existing.published_by or current_user.get("id") + updated = self.repository.update(existing) + return self._to_dict(updated), False + + def update_index_status( + self, + current_user: dict, + article_id: str, + *, + index_status: str, + knowledge_base_id: Optional[str] = None, + knowledge_document_id: Optional[str] = None, + published_status: Optional[str] = None, + ) -> Optional[dict]: + self._require_admin(current_user) + article = self.repository.get_by_id(article_id, current_user["tenant_id"]) + if article is None: + return None + article.index_status = self._text_or_default(index_status, article.index_status) + article.knowledge_base_id = self._optional_text(knowledge_base_id) or article.knowledge_base_id + article.knowledge_document_id = self._optional_text(knowledge_document_id) or article.knowledge_document_id + if published_status is not None: + article.published_status = self._text_or_default(published_status, article.published_status) + if article.published_status == "published" and article.published_at is None: + article.published_at = datetime.now(timezone.utc) + article.published_by = current_user.get("id") + updated = self.repository.update(article) + return self._to_dict(updated) + + @staticmethod + def _require_admin(current_user: dict) -> None: + if current_user.get("role") != "admin": + raise HTTPException(status_code=403, detail="forbidden") + + @staticmethod + def _required_text(payload: dict[str, Any], field_name: str) -> str: + value = payload.get(field_name) + if value is None or not str(value).strip(): + raise HTTPException(status_code=400, detail=f"{field_name} is required") + return str(value).strip() + + @staticmethod + def _optional_text(value: Any) -> Optional[str]: + if value is None or not str(value).strip(): + return None + return str(value).strip() + + @classmethod + def _text_or_default(cls, value: Any, default: str) -> str: + return cls._optional_text(value) or default + + @staticmethod + def _normalize_tags(value: Any) -> Optional[list[str]]: + if value is None: + return None + if not isinstance(value, list): + raise HTTPException(status_code=400, detail="tags must be a list of strings") + tags: list[str] = [] + for item in value: + text = str(item).strip() + if text and text not in tags: + tags.append(text) + return tags or None + + @staticmethod + def _parse_datetime(value: Any) -> Optional[datetime]: + if value is None or not str(value).strip(): + return None + text = str(value).strip() + if text.endswith("Z"): + text = f"{text[:-1]}+00:00" + try: + return datetime.fromisoformat(text) + except ValueError: + raise HTTPException(status_code=400, detail="published_at must be ISO datetime") + + @staticmethod + def _checksum(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + @classmethod + def _build_article_id(cls, tenant_id: str, source_id: str, article_no: str) -> str: + digest = hashlib.sha256(f"{tenant_id}:{source_id}:{article_no}".encode("utf-8")).hexdigest()[:20] + return f"lart-{digest}" + + @staticmethod + def _to_dict(article: LegalArticle) -> dict: + return { + "id": article.id, + "tenant_id": article.tenant_id, + "source_id": article.source_id, + "source_title": article.source_title, + "source_type": article.source_type, + "authority": article.authority, + "effective_date": article.effective_date, + "validity_status": article.validity_status, + "region": article.region, + "official_url": article.official_url, + "article_no": article.article_no, + "chapter_title": article.chapter_title, + "section_title": article.section_title, + "article_text": article.article_text, + "tags": article.tags_json or [], + "tags_json": article.tags_json, + "source_checksum": article.source_checksum, + "text_checksum": article.text_checksum, + "knowledge_base_id": article.knowledge_base_id, + "knowledge_document_id": article.knowledge_document_id, + "index_status": article.index_status, + "published_status": article.published_status, + "published_at": article.published_at.isoformat() if article.published_at else None, + "published_by": article.published_by, + "created_at": article.created_at.isoformat() if article.created_at else None, + "updated_at": article.updated_at.isoformat() if article.updated_at else None, + } diff --git a/backend/migrations/versions/20260516_0022_create_legal_articles.py b/backend/migrations/versions/20260516_0022_create_legal_articles.py new file mode 100644 index 00000000..bc74f0cc --- /dev/null +++ b/backend/migrations/versions/20260516_0022_create_legal_articles.py @@ -0,0 +1,89 @@ +"""create legal articles table""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + + +revision = "20260516_0022" +down_revision = "20260516_0021" +branch_labels = None +depends_on = None + + +def _table_names(inspector: sa.Inspector) -> set[str]: + return set(inspector.get_table_names()) + + +def _index_names(inspector: sa.Inspector, table_name: str) -> set[str]: + return {index["name"] for index in inspector.get_indexes(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "legal_articles" not in _table_names(inspector): + op.create_table( + "legal_articles", + sa.Column("id", sa.String(length=96), primary_key=True), + sa.Column("tenant_id", sa.String(length=64), nullable=False), + sa.Column("source_id", sa.String(length=96), nullable=False), + sa.Column("source_title", sa.String(length=255), nullable=False), + sa.Column("source_type", sa.String(length=64), nullable=False), + sa.Column("authority", sa.String(length=255), nullable=False), + sa.Column("effective_date", sa.String(length=128), nullable=True), + sa.Column("validity_status", sa.String(length=64), nullable=True), + sa.Column("region", sa.String(length=64), nullable=False, server_default="CN"), + sa.Column("official_url", sa.Text(), nullable=True), + sa.Column("article_no", sa.String(length=64), nullable=False), + sa.Column("chapter_title", sa.String(length=255), nullable=True), + sa.Column("section_title", sa.String(length=255), nullable=True), + sa.Column("article_text", sa.Text(), nullable=False), + sa.Column("tags_json", sa.JSON(), nullable=True), + sa.Column("source_checksum", sa.String(length=64), nullable=True), + sa.Column("text_checksum", sa.String(length=64), nullable=True), + sa.Column("knowledge_base_id", sa.String(length=96), nullable=True), + sa.Column("knowledge_document_id", sa.String(length=96), nullable=True), + sa.Column("index_status", sa.String(length=32), nullable=False, server_default="pending"), + sa.Column("published_status", sa.String(length=32), nullable=False, server_default="draft"), + sa.Column("published_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("published_by", sa.String(length=64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False), + ) + + inspector = sa.inspect(bind) + indexes = _index_names(inspector, "legal_articles") + index_specs = { + "ix_legal_articles_tenant_id": ["tenant_id"], + "ix_legal_articles_source_id": ["source_id"], + "ix_legal_articles_source_type": ["source_type"], + "ix_legal_articles_region": ["region"], + "ix_legal_articles_article_no": ["article_no"], + "ix_legal_articles_text_checksum": ["text_checksum"], + "ix_legal_articles_knowledge_base_id": ["knowledge_base_id"], + "ix_legal_articles_knowledge_document_id": ["knowledge_document_id"], + "ix_legal_articles_index_status": ["index_status"], + "ix_legal_articles_published_status": ["published_status"], + "ix_legal_articles_created_at": ["created_at"], + "ix_legal_articles_updated_at": ["updated_at"], + } + for index_name, columns in index_specs.items(): + if index_name not in indexes: + op.create_index(index_name, "legal_articles", columns, unique=False) + if "uq_legal_articles_tenant_source_article" not in indexes: + op.create_index( + "uq_legal_articles_tenant_source_article", + "legal_articles", + ["tenant_id", "source_id", "article_no"], + unique=True, + ) + + +def downgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if "legal_articles" not in _table_names(inspector): + return + op.drop_table("legal_articles") diff --git a/backend/tests/test_legal_articles_api.py b/backend/tests/test_legal_articles_api.py new file mode 100644 index 00000000..08b28292 --- /dev/null +++ b/backend/tests/test_legal_articles_api.py @@ -0,0 +1,208 @@ +import unittest + +from fastapi import HTTPException +from fastapi.testclient import TestClient + +from app.api.legal_articles import get_audit_log_service, get_legal_article_service, require_current_user +from app.main import app + + +class StubAuditLogService: + def __init__(self): + self.events = [] + + def record_event(self, **kwargs): + self.events.append(kwargs) + return kwargs + + +class StubLegalArticleService: + def list_articles( + self, + current_user, + page, + page_size, + keyword=None, + source_id=None, + article_no=None, + index_status=None, + published_status=None, + ): + if current_user["role"] != "admin": + raise HTTPException(status_code=403, detail="forbidden") + return { + "items": [ + { + "id": "lart-001", + "source_id": source_id or "LAW-CIVIL-CODE-2020", + "source_title": "中华人民共和国民法典", + "source_type": "法律", + "authority": "全国人民代表大会", + "effective_date": "2021-01-01", + "validity_status": "现行有效", + "region": "CN", + "official_url": "https://flk.npc.gov.cn/detail2.html?ZmY4MDgwODE3MjlkMWVmZTAxNzI5ZDUwYjVjNTAwYmY%3D", + "article_no": article_no or "第四百六十四条", + "chapter_title": "第三编 合同", + "section_title": "第一章 一般规定", + "article_text": "合同是民事主体之间设立、变更、终止民事法律关系的协议。", + "tags": ["phase14", "contract"], + "index_status": index_status or "indexed", + "published_status": published_status or "published", + "knowledge_base_id": "kb-001", + "knowledge_document_id": "doc-001", + "created_at": "2026-05-16T00:00:00+00:00", + "updated_at": "2026-05-16T00:00:00+00:00", + } + ], + "page": page, + "page_size": page_size, + "total": 1, + } + + def get_article(self, current_user, article_id): + if current_user["role"] != "admin": + raise HTTPException(status_code=403, detail="forbidden") + if article_id == "missing": + return None + return self.list_articles(current_user, 1, 20)["items"][0] + + def bulk_upsert_articles(self, current_user, articles): + if current_user["role"] != "admin": + raise HTTPException(status_code=403, detail="forbidden") + return { + "created_count": len(articles), + "updated_count": 0, + "total": len(articles), + "created": articles, + "updated": [], + } + + def update_index_status( + self, + current_user, + article_id, + *, + index_status, + knowledge_base_id=None, + knowledge_document_id=None, + published_status=None, + ): + if current_user["role"] != "admin": + raise HTTPException(status_code=403, detail="forbidden") + if article_id == "missing": + return None + item = self.list_articles(current_user, 1, 20, index_status=index_status, published_status=published_status)["items"][0] + item["id"] = article_id + item["knowledge_base_id"] = knowledge_base_id + item["knowledge_document_id"] = knowledge_document_id + return item + + +class LegalArticlesApiTestCase(unittest.TestCase): + def setUp(self): + self.audit_log_service = StubAuditLogService() + app.dependency_overrides[get_legal_article_service] = lambda: StubLegalArticleService() + app.dependency_overrides[get_audit_log_service] = lambda: self.audit_log_service + self.client = TestClient(app) + + def tearDown(self): + app.dependency_overrides.clear() + + def test_admin_can_list_legal_articles(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-admin", + "tenant_id": "tenant-demo-001", + "role": "admin", + } + + response = self.client.get("/api/v1/legal-articles", params={"source_id": "LAW-CIVIL-CODE-2020"}) + + self.assertEqual(response.status_code, 200) + body = response.json()["data"] + self.assertEqual(body["total"], 1) + self.assertEqual(body["items"][0]["source_id"], "LAW-CIVIL-CODE-2020") + self.assertEqual(body["items"][0]["index_status"], "indexed") + + def test_admin_can_get_legal_article_detail(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-admin", + "tenant_id": "tenant-demo-001", + "role": "admin", + } + + response = self.client.get("/api/v1/legal-articles/lart-001") + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data"]["article_no"], "第四百六十四条") + + def test_admin_can_bulk_upsert_legal_articles(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-admin", + "tenant_id": "tenant-demo-001", + "role": "admin", + } + + response = self.client.post( + "/api/v1/legal-articles/bulk-upsert", + json={ + "articles": [ + { + "source_id": "LAW-CIVIL-CODE-2020", + "source_title": "中华人民共和国民法典", + "source_type": "法律", + "authority": "全国人民代表大会", + "effective_date": "2021-01-01", + "region": "CN", + "article_no": "第四百六十四条", + "article_text": "合同是民事主体之间设立、变更、终止民事法律关系的协议。", + "tags": ["phase14", "contract"], + "index_status": "indexed", + "published_status": "published", + } + ] + }, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data"]["total"], 1) + self.assertEqual(self.audit_log_service.events[0]["action"], "bulk_upsert_legal_articles") + self.assertEqual(self.audit_log_service.events[0]["detail_json"]["source_ids"], ["LAW-CIVIL-CODE-2020"]) + + def test_admin_can_update_legal_article_index_status(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-admin", + "tenant_id": "tenant-demo-001", + "role": "admin", + } + + response = self.client.post( + "/api/v1/legal-articles/lart-001/index-status", + json={ + "index_status": "indexed", + "knowledge_base_id": "kb-001", + "knowledge_document_id": "doc-001", + "published_status": "published", + }, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["data"]["knowledge_document_id"], "doc-001") + self.assertEqual(self.audit_log_service.events[0]["action"], "update_legal_article_index_status") + + def test_member_cannot_access_legal_articles(self): + app.dependency_overrides[require_current_user] = lambda: { + "id": "user-member", + "tenant_id": "tenant-demo-001", + "role": "member", + } + + response = self.client.get("/api/v1/legal-articles") + + self.assertEqual(response.status_code, 403) + self.assertEqual(response.json()["message"], "forbidden") + self.assertEqual(self.audit_log_service.events, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/project-docs/AI合同起草与审查助手-Phase14法条全文级知识库实施方案-2026-05-16.md b/project-docs/AI合同起草与审查助手-Phase14法条全文级知识库实施方案-2026-05-16.md new file mode 100644 index 00000000..76227763 --- /dev/null +++ b/project-docs/AI合同起草与审查助手-Phase14法条全文级知识库实施方案-2026-05-16.md @@ -0,0 +1,92 @@ +# AI合同起草与审查助手-Phase14法条全文级知识库实施方案 + +日期：2026-05-16 + +## 1. 阶段定位 + +Phase13 已经完成“资料包正式吸收入库”，系统内已有法源清单、知识卡、标准条款、正式审核规则和合同模板。 + +Phase14 不再重复做资料包，而是补齐下一层能力： + +> 把权威法源中的具体法条拆到“条文级全文”，让系统能够按法条编号、来源、效力、标签、索引状态和知识库文档进行检索、引用和追踪。 + +## 2. 不混淆的边界 + +- Phase13：解决“有哪些法源、模板、规则、条款已经进入系统”。 +- Phase14：解决“具体第几条法律条文的全文是否已经进入系统，是否可检索、可引用、可追踪”。 +- 本阶段不宣称一次性覆盖全国全部法律法规全文，而是先建立可复用的条文级吸收机制，再按批次扩展。 + +## 3. 首批来源 + +首批从国家法律法规数据库可查询到的正式法律中抽取合同业务高频条文： + +- 《中华人民共和国民法典》 +- 《中华人民共和国电子签名法》 +- 《中华人民共和国个人信息保护法》 +- 《中华人民共和国劳动合同法》 +- 《中华人民共和国数据安全法》 +- 《中华人民共和国公司法》 + +这些来源先进入 `project-docs/phase14-legal-fulltext-library/legal-article-manifest.json`，每个来源记录： + +- 法源编号 +- 法源名称 +- 国家法律法规数据库 `bbbs` +- 发布机关 +- 公布日期 +- 生效日期 +- 效力状态 +- 条文编号 +- 业务标签 + +## 4. 实现口径 + +本阶段新增运行层能力： + +- 新增 `legal_articles` 表，保存条文级全文和治理状态。 +- 新增 `/api/v1/legal-articles` 管理接口： + - 列表检索 + - 详情查看 + - 批量 upsert + - 索引状态回填 +- 新增 `scripts/import-phase14-legal-fulltext.ps1`： + - 读取首批法条清单 + - 从国家法律法规数据库下载官方 docx + - 抽取指定法条全文 + - 生成条文级 markdown 文档并上传知识库 + - 回写 `legal_articles` + - 发布目标知识库 + - 生成闭环验证产物 + +## 5. 验收口径 + +Phase14 第一刀通过标准： + +- 所有清单内法条均能从官方文档抽取到全文。 +- 所有法条均上传到目标知识库且索引状态为 `indexed`。 +- 所有法条均写入 `legal_articles` 表。 +- 每条法条均有： + - `source_id` + - `source_title` + - `article_no` + - `article_text` + - `authority` + - `effective_date` + - `region` + - `official_url` + - `text_checksum` + - `knowledge_base_id` + - `knowledge_document_id` + - `index_status` + - `published_status` +- 导入脚本生成 `PHASE14_LEGAL_FULLTEXT_IMPORT_PASS` 产物。 + +## 6. 后续扩展 + +第一刀完成后，后续可以继续做： + +- 扩展到更多民法典合同编、物权编、侵权责任编条文。 +- 扩展司法解释、部门规章、地方示范文本。 +- 为审查结果增加“引用到具体法条全文”的证据卡片。 +- 做前端“法条全文库”页面。 +- 接入用户本地下载的法律法规资料包，按同一清单格式导入。 diff --git a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md index 4aa3919c..4a9c9356 100644 --- a/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md +++ b/project-docs/AI合同起草与审查助手-当前工作进度保存-2026-05-11.md @@ -2016,3 +2016,87 @@ ### 阻塞项 - 本轮无 required 阻塞项。 - 仍保留旧的非 required 项：外部告警 Webhook 未配置。 + +## 2026-05-16 Phase14 法条全文级知识库第一刀闭环 +### 本轮完成项 +- 已新增 Phase14 中文实施方案： + - `project-docs/AI合同起草与审查助手-Phase14法条全文级知识库实施方案-2026-05-16.md` +- 已新增首批法条全文清单： + - `project-docs/phase14-legal-fulltext-library/legal-article-manifest.json` + - 首批覆盖 6 个正式法源、33 条合同业务高频条文： + - 《中华人民共和国民法典》 + - 《中华人民共和国电子签名法》 + - 《中华人民共和国个人信息保护法》 + - 《中华人民共和国劳动合同法》 + - 《中华人民共和国数据安全法》 + - 《中华人民共和国公司法》 +- 已新增运行层 `legal_articles` 表和迁移： + - `backend/app/models/legal_article.py` + - `backend/migrations/versions/20260516_0022_create_legal_articles.py` +- 已新增法条全文管理 API： + - `GET /api/v1/legal-articles` + - `GET /api/v1/legal-articles/{article_id}` + - `POST /api/v1/legal-articles/bulk-upsert` + - `POST /api/v1/legal-articles/{article_id}/index-status` +- 已新增 Phase14 官方法条抓取和入库脚本： + - `scripts/import-phase14-legal-fulltext.ps1` + - 脚本从国家法律法规数据库下载官方 docx，抽取指定法条全文，上传知识库，回写 `legal_articles`，并发布目标知识库。 +- 已部署后端到 `192.168.3.90`，迁移和新 API 已进入生产后端。 +- 已完成 Phase14 正式导入： + - 目标知识库：`AI合同平台法条全文库（Phase14）` + - 目标知识库编号：`kb-de2dbb597834` + - 33 条法条全文全部上传、索引、写入运行表并标记为 published/indexed。 + +### 本轮验证 +- 后端编译： + - `.venv\Scripts\python.exe -m py_compile app\models\legal_article.py app\repositories\legal_article_repository.py app\services\legal_article_service.py app\api\legal_articles.py migrations\versions\20260516_0022_create_legal_articles.py` + - 结果：通过 +- 后端单测： + - `.venv\Scripts\python.exe -m unittest tests.test_legal_articles_api` + - 结果：`5 tests OK` + - `.venv\Scripts\python.exe -m unittest tests.test_knowledge_bases_api tests.test_templates_api tests.test_review_rules_api` + - 结果：`26 tests OK` +- 后端生产部署： + - 产物：`project-docs/qa-artifacts/phase7-backend-docker-deploy/2026-05-16T17-45-57/backend-docker-deploy-summary.md` + - 结果：`PHASE7_BACKEND_DOCKER_DEPLOY_WARN` + - 说明：required 检查失败数为 0；WARN 来自开发态未提交工作区和非 required 发布门禁提醒。 +- Phase14 法条全文导入： + - 首次脚本运行失败在 Windows PowerShell 官方 JSON 编码解码，已修复为 UTF-8 字节解码。 + - 最终产物：`project-docs/qa-artifacts/phase14-legal-fulltext-import/2026-05-16T17-56-15-843/phase14-legal-fulltext-import-summary.md` + - 结果：`PHASE14_LEGAL_FULLTEXT_IMPORT_PASS` + - 关键指标： + - `sources=6` + - `articles_extracted=33` + - `import_failures=0` + - `failed_required_checks=0` + - `failed_verifications=0` + - `knowledge_documents_indexed=pass` + - `legal_articles_runtime_table=pass` +- API 真相源抽查： + - `GET /api/v1/legal-articles?page=1&page_size=3&published_status=published&index_status=indexed` + - Node UTF-8 客户端验证总数：`total=33` + - 中文字段和条文正文可正常返回。 +- 导入后业务质量门禁： + - 产物：`project-docs/qa-artifacts/phase9-business-quality-gate/2026-05-16T17-56-58-537/business-quality-gate-summary.md` + - 结果：`PHASE9_BUSINESS_QUALITY_GATE_PASS_WITH_WEBHOOK_BLOCKED` + - 业务评分：review=100、kb=100、contract_generation=100。 + +### 当前状态 +- 现在可以明确说：系统已经具备“条文级法条全文知识库”的第一版生产能力。 +- 当前吸收口径是： + - 官方法源可追溯到国家法律法规数据库 `bbbs` + - 条文全文来自官方 docx 抽取 + - 每条法条有来源、条号、章节、正文、标签、checksum、知识库文档编号、索引状态和发布状态 + - 知识库层可检索，运行表层可治理 + +### 未完成项 +- Phase14 目前完成的是首批 6 个法源、33 条高频合同相关法条，不是全国全部法律法规全文。 +- 后续增强项： + - 扩展更多民法典合同编和相关司法解释条文 + - 增加前端“法条全文库”管理页面 + - 在审查报告中把风险依据进一步链接到 `legal_articles` 的具体条文全文 + - 支持用户本地下载的法律法规资料包按同一清单格式导入 + +### 阻塞项 +- 本轮无 required 阻塞项。 +- 仍保留旧的非 required 项：外部告警 Webhook 未配置。 diff --git a/project-docs/README.md b/project-docs/README.md index 00e426b0..dd65f32d 100644 --- a/project-docs/README.md +++ b/project-docs/README.md @@ -19,6 +19,7 @@ - `AI合同起草与审查助手-真人参与最小化与必要人工环节说明-2026-05-14.md` - `AI合同起草与审查助手-真人参与流程-会议汇报版-2026-05-14.md` - `AI合同起草与审查助手-正式生产部署演练与上线检查清单-2026-05-16.md` +- `AI合同起草与审查助手-Phase14法条全文级知识库实施方案-2026-05-16.md` ## 当前仍保留的专项资料 @@ -36,6 +37,7 @@ - `phase8-pilot-templates/` - `phase9-quality-fixtures/` - `phase13-legal-knowledge-template-library/` +- `phase14-legal-fulltext-library/` - `pilot-materials/` - `structured-review-samples/` - `automation-artifacts/` diff --git a/project-docs/phase14-legal-fulltext-library/legal-article-manifest.json b/project-docs/phase14-legal-fulltext-library/legal-article-manifest.json new file mode 100644 index 00000000..ed3b3cc4 --- /dev/null +++ b/project-docs/phase14-legal-fulltext-library/legal-article-manifest.json @@ -0,0 +1,125 @@ +{ + "name": "Phase14 法条全文级知识库首批清单", + "source_system": "国家法律法规数据库", + "source_homepage": "https://flk.npc.gov.cn/", + "generated_at": "2026-05-16", + "sources": [ + { + "source_id": "LAW-CIVIL-CODE-2020", + "title": "中华人民共和国民法典", + "bbbs": "ff808081729d1efe01729d50b5c500bf", + "source_type": "法律", + "authority": "全国人民代表大会", + "published_date": "2020-05-28", + "effective_date": "2021-01-01", + "validity_status": "现行有效", + "region": "CN", + "knowledge_domain": "civil-code-contract", + "articles": [ + { "article_no": "第四百六十四条", "chapter_title": "第三编 合同", "section_title": "第一章 一般规定", "tags": ["合同定义", "合同编", "民法典"] }, + { "article_no": "第四百六十五条", "chapter_title": "第三编 合同", "section_title": "第一章 一般规定", "tags": ["合同效力", "合同保护", "民法典"] }, + { "article_no": "第四百六十九条", "chapter_title": "第三编 合同", "section_title": "第二章 合同的订立", "tags": ["书面形式", "数据电文", "电子合同"] }, + { "article_no": "第四百七十条", "chapter_title": "第三编 合同", "section_title": "第二章 合同的订立", "tags": ["合同条款", "必备条款", "合同内容"] }, + { "article_no": "第四百九十条", "chapter_title": "第三编 合同", "section_title": "第二章 合同的订立", "tags": ["合同成立", "签名盖章", "履行成立"] }, + { "article_no": "第四百九十六条", "chapter_title": "第三编 合同", "section_title": "第二章 合同的订立", "tags": ["格式条款", "提示说明", "合同审查"] }, + { "article_no": "第四百九十七条", "chapter_title": "第三编 合同", "section_title": "第二章 合同的订立", "tags": ["格式条款", "无效条款", "风险审查"] }, + { "article_no": "第五百零九条", "chapter_title": "第三编 合同", "section_title": "第四章 合同的履行", "tags": ["全面履行", "诚信原则", "协助义务"] }, + { "article_no": "第五百七十七条", "chapter_title": "第三编 合同", "section_title": "第八章 违约责任", "tags": ["违约责任", "补救措施", "赔偿损失"] }, + { "article_no": "第五百八十五条", "chapter_title": "第三编 合同", "section_title": "第八章 违约责任", "tags": ["违约金", "损失赔偿", "违约责任"] } + ] + }, + { + "source_id": "LAW-E-SIGNATURE-2019", + "title": "中华人民共和国电子签名法", + "bbbs": "ff8080816f135f46016f2163e4261aa1", + "source_type": "法律", + "authority": "全国人民代表大会常务委员会", + "published_date": "2019-04-23", + "effective_date": "2019-04-23", + "validity_status": "现行有效", + "region": "CN", + "knowledge_domain": "electronic-signature", + "articles": [ + { "article_no": "第三条", "chapter_title": "第一章 总则", "section_title": null, "tags": ["电子签名", "民事活动", "适用范围"] }, + { "article_no": "第四条", "chapter_title": "第二章 数据电文", "section_title": null, "tags": ["数据电文", "书面形式", "电子合同"] }, + { "article_no": "第五条", "chapter_title": "第二章 数据电文", "section_title": null, "tags": ["原件形式", "数据电文", "证据"] }, + { "article_no": "第十三条", "chapter_title": "第三章 电子签名与认证", "section_title": null, "tags": ["可靠电子签名", "签名要件", "证据"] }, + { "article_no": "第十四条", "chapter_title": "第三章 电子签名与认证", "section_title": null, "tags": ["电子签名效力", "手写签名", "盖章"] } + ] + }, + { + "source_id": "LAW-PIPL-2021", + "title": "中华人民共和国个人信息保护法", + "bbbs": "ff8081817b6472a3017b656cc2040044", + "source_type": "法律", + "authority": "全国人民代表大会常务委员会", + "published_date": "2021-08-20", + "effective_date": "2021-11-01", + "validity_status": "现行有效", + "region": "CN", + "knowledge_domain": "personal-information-protection", + "articles": [ + { "article_no": "第十三条", "chapter_title": "第二章 个人信息处理规则", "section_title": "第一节 一般规定", "tags": ["处理依据", "个人信息", "数据合规"] }, + { "article_no": "第十四条", "chapter_title": "第二章 个人信息处理规则", "section_title": "第一节 一般规定", "tags": ["同意", "个人信息", "告知同意"] }, + { "article_no": "第十七条", "chapter_title": "第二章 个人信息处理规则", "section_title": "第一节 一般规定", "tags": ["告知义务", "隐私政策", "数据合规"] }, + { "article_no": "第二十三条", "chapter_title": "第二章 个人信息处理规则", "section_title": "第一节 一般规定", "tags": ["委托处理", "共同处理", "第三方"] }, + { "article_no": "第二十八条", "chapter_title": "第二章 个人信息处理规则", "section_title": "第二节 敏感个人信息的处理规则", "tags": ["敏感个人信息", "必要性", "严格保护"] }, + { "article_no": "第五十一条", "chapter_title": "第五章 个人信息处理者的义务", "section_title": null, "tags": ["合规制度", "安全措施", "个人信息处理者"] } + ] + }, + { + "source_id": "LAW-LABOR-CONTRACT-2012", + "title": "中华人民共和国劳动合同法", + "bbbs": "2c909fdd678bf17901678bf74d7106b3", + "source_type": "法律", + "authority": "全国人民代表大会常务委员会", + "published_date": "2012-12-28", + "effective_date": "2013-07-01", + "validity_status": "现行有效", + "region": "CN", + "knowledge_domain": "labor-contract", + "articles": [ + { "article_no": "第十条", "chapter_title": "第二章 劳动合同的订立", "section_title": null, "tags": ["书面劳动合同", "用工", "劳动关系"] }, + { "article_no": "第十七条", "chapter_title": "第二章 劳动合同的订立", "section_title": null, "tags": ["必备条款", "劳动合同", "合同内容"] }, + { "article_no": "第十九条", "chapter_title": "第二章 劳动合同的订立", "section_title": null, "tags": ["试用期", "期限", "劳动合同"] }, + { "article_no": "第二十二条", "chapter_title": "第二章 劳动合同的订立", "section_title": null, "tags": ["服务期", "培训费用", "违约金"] }, + { "article_no": "第二十三条", "chapter_title": "第二章 劳动合同的订立", "section_title": null, "tags": ["保密义务", "竞业限制", "劳动合同"] }, + { "article_no": "第二十四条", "chapter_title": "第二章 劳动合同的订立", "section_title": null, "tags": ["竞业限制", "适用人员", "期限"] } + ] + }, + { + "source_id": "LAW-DATA-SECURITY-2021", + "title": "中华人民共和国数据安全法", + "bbbs": "ff80818179f5e0800179f885c7e70392", + "source_type": "法律", + "authority": "全国人民代表大会常务委员会", + "published_date": "2021-06-10", + "effective_date": "2021-09-01", + "validity_status": "现行有效", + "region": "CN", + "knowledge_domain": "data-security", + "articles": [ + { "article_no": "第二十七条", "chapter_title": "第四章 数据安全保护义务", "section_title": null, "tags": ["数据安全", "管理制度", "技术措施"] }, + { "article_no": "第二十九条", "chapter_title": "第四章 数据安全保护义务", "section_title": null, "tags": ["风险监测", "补救措施", "数据安全"] }, + { "article_no": "第三十一条", "chapter_title": "第四章 数据安全保护义务", "section_title": null, "tags": ["重要数据", "出境安全", "数据合规"] } + ] + }, + { + "source_id": "LAW-COMPANY-2023", + "title": "中华人民共和国公司法", + "bbbs": "ff8081818c9108eb018cb6922f750c07", + "source_type": "法律", + "authority": "全国人民代表大会常务委员会", + "published_date": "2023-12-29", + "effective_date": "2024-07-01", + "validity_status": "现行有效", + "region": "CN", + "knowledge_domain": "company-commercial", + "articles": [ + { "article_no": "第十条", "chapter_title": "第一章 总则", "section_title": null, "tags": ["法定代表人", "公司代表", "主体资格"] }, + { "article_no": "第十一条", "chapter_title": "第一章 总则", "section_title": null, "tags": ["公司章程", "约束力", "治理文件"] }, + { "article_no": "第十二条", "chapter_title": "第一章 总则", "section_title": null, "tags": ["经营范围", "登记事项", "主体审查"] } + ] + } + ] +} diff --git a/scripts/import-phase14-legal-fulltext.ps1 b/scripts/import-phase14-legal-fulltext.ps1 new file mode 100644 index 00000000..ffb9ff44 --- /dev/null +++ b/scripts/import-phase14-legal-fulltext.ps1 @@ -0,0 +1,590 @@ +param( + [string]$ApiBaseUrl = "http://192.168.3.90:8001/api/v1", + [string]$ManifestPath = "", + [string]$ArtifactRoot = "", + [string]$AdminEmail = "admin@example.com", + [string]$AdminPassword = "", + [string]$AdminPasswordFile = "", + [string]$KnowledgeBaseName = "AI合同平台法条全文库（Phase14）", + [switch]$ForceReupload, + [switch]$SkipKnowledgeBasePublish +) + +$ErrorActionPreference = "Stop" + +Add-Type -AssemblyName System.Net.Http + +$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path +$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss-fff" + +if ([string]::IsNullOrWhiteSpace($ManifestPath)) { + $ManifestPath = Join-Path $repoRoot "project-docs\phase14-legal-fulltext-library\legal-article-manifest.json" +} +if (-not [System.IO.Path]::IsPathRooted($ManifestPath)) { + $ManifestPath = Join-Path $repoRoot $ManifestPath +} + +if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase14-legal-fulltext-import" +} +if (-not [System.IO.Path]::IsPathRooted($ArtifactRoot)) { + $ArtifactRoot = Join-Path $repoRoot $ArtifactRoot +} + +if ([string]::IsNullOrWhiteSpace($AdminPasswordFile)) { + $AdminPasswordFile = Join-Path $HOME ".ai-contract-platform\production-admin-password.txt" +} + +$artifactDir = Join-Path $ArtifactRoot $timestamp +$downloadDir = Join-Path $artifactDir "official-downloads" +New-Item -ItemType Directory -Force -Path $downloadDir | Out-Null + +$summaryJson = Join-Path $artifactDir "phase14-legal-fulltext-import-summary.json" +$summaryMd = Join-Path $artifactDir "phase14-legal-fulltext-import-summary.md" +$ApiBaseUrl = $ApiBaseUrl.TrimEnd("/") + +$checks = New-Object System.Collections.Generic.List[object] +$imports = New-Object System.Collections.Generic.List[object] +$verification = New-Object System.Collections.Generic.List[object] +$articlePayloads = New-Object System.Collections.Generic.List[object] +$articlePrefix = [string][char]0x7B2C +$articleSuffix = [string][char]0x6761 + +function Add-Check { + param([string]$Key, [string]$Label, [string]$Status, [string]$Detail, [bool]$Required = $true, [hashtable]$Meta = @{}) + $checks.Add([ordered]@{ key = $Key; label = $Label; status = $Status; required = $Required; detail = $Detail; meta = $Meta }) | Out-Null +} + +function Add-Import { + param([string]$SourceId, [string]$ArticleNo, [string]$Action, [string]$Status, [string]$TargetId, [string]$Detail, [hashtable]$Meta = @{}) + $imports.Add([ordered]@{ source_id = $SourceId; article_no = $ArticleNo; action = $Action; status = $Status; target_id = $TargetId; detail = $Detail; meta = $Meta }) | Out-Null +} + +function Add-Verification { + param([string]$Key, [string]$Status, [string]$Detail, [hashtable]$Meta = @{}) + $verification.Add([ordered]@{ key = $Key; status = $Status; detail = $Detail; meta = $Meta }) | Out-Null +} + +function Save-Json { + param([string]$Path, [object]$Value) + $Value | ConvertTo-Json -Depth 60 | Set-Content -Encoding UTF8 -Path $Path +} + +function Get-ResponseData { + param([object]$Response) + if ($null -eq $Response) { return $null } + if ($Response.PSObject.Properties.Name -contains "data") { return $Response.data } + return $Response +} + +function Invoke-ApiJson { + param([string]$Method, [string]$Url, [hashtable]$Headers = @{}, [object]$Body = $null) + $args = @{ Method = $Method; Uri = $Url; Headers = $Headers; ErrorAction = "Stop" } + if ($null -ne $Body) { + $args["ContentType"] = "application/json; charset=utf-8" + $args["Body"] = ($Body | ConvertTo-Json -Depth 60) + } + return Invoke-RestMethod @args +} + +function Invoke-FormRequest { + param( + [string]$Method, + [string]$Url, + [hashtable]$Headers = @{}, + [hashtable]$Fields = @{}, + [string]$FileField = "", + [string]$FileName = "", + [byte[]]$FileContent = $null, + [string]$FileContentType = "text/markdown; charset=utf-8" + ) + + $client = [System.Net.Http.HttpClient]::new() + try { + foreach ($headerName in $Headers.Keys) { + $client.DefaultRequestHeaders.Remove($headerName) | Out-Null + $client.DefaultRequestHeaders.Add($headerName, [string]$Headers[$headerName]) + } + + $form = [System.Net.Http.MultipartFormDataContent]::new() + try { + foreach ($fieldName in $Fields.Keys) { + $value = [string]$Fields[$fieldName] + $stringContent = [System.Net.Http.StringContent]::new($value, [System.Text.Encoding]::UTF8) + $form.Add($stringContent, $fieldName) + } + if ($FileContent -ne $null -and -not [string]::IsNullOrWhiteSpace($FileField)) { + $byteContent = [System.Net.Http.ByteArrayContent]::new($FileContent) + $byteContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse($FileContentType) + $form.Add($byteContent, $FileField, $FileName) + } + + $request = [System.Net.Http.HttpRequestMessage]::new([System.Net.Http.HttpMethod]::new($Method), $Url) + $request.Content = $form + $response = $client.SendAsync($request).GetAwaiter().GetResult() + $body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() + if (-not $response.IsSuccessStatusCode) { + throw "API request failed: $Method $Url, status=$([int]$response.StatusCode), response=$body" + } + if ([string]::IsNullOrWhiteSpace($body)) { return $null } + return $body | ConvertFrom-Json + } finally { + if ($null -ne $form) { $form.Dispose() } + } + } finally { + $client.Dispose() + } +} + +function Get-PagedApiItems { + param([string]$Url, [hashtable]$Headers) + $items = New-Object System.Collections.Generic.List[object] + $page = 1 + $pageSize = 200 + while ($true) { + $separator = if ($Url.Contains("?")) { "&" } else { "?" } + $response = Invoke-ApiJson -Method "GET" -Url "$Url${separator}page=$page&page_size=$pageSize" -Headers $Headers + $data = Get-ResponseData -Response $response + if ($null -eq $data -or $null -eq $data.items) { break } + foreach ($item in @($data.items)) { $items.Add($item) | Out-Null } + if ($items.Count -ge [int]$data.total -or @($data.items).Count -eq 0) { break } + $page += 1 + } + return $items.ToArray() +} + +function Get-AccessToken { + $password = $AdminPassword + if ([string]::IsNullOrWhiteSpace($password) -and -not [string]::IsNullOrWhiteSpace($env:AICP_ADMIN_PASSWORD)) { + $password = $env:AICP_ADMIN_PASSWORD + } + if ([string]::IsNullOrWhiteSpace($password) -and (Test-Path -LiteralPath $AdminPasswordFile -PathType Leaf)) { + $password = (Get-Content -LiteralPath $AdminPasswordFile -Raw).Trim() + } + if ([string]::IsNullOrWhiteSpace($password)) { + throw "Admin password is required. Use -AdminPassword, AICP_ADMIN_PASSWORD, or $AdminPasswordFile." + } + + $response = Invoke-ApiJson -Method "POST" -Url "$ApiBaseUrl/auth/login" -Body @{ account = $AdminEmail; password = $password } + $data = Get-ResponseData -Response $response + if ($null -eq $data -or [string]::IsNullOrWhiteSpace($data.access_token)) { + throw "Login response has no access_token." + } + return [string]$data.access_token +} + +function Get-AuthHeaders { + param([string]$Token) + return @{ Authorization = "Bearer $Token" } +} + +function Get-TextChecksum { + param([string]$Text) + $bytes = [System.Text.Encoding]::UTF8.GetBytes($Text) + $sha = [System.Security.Cryptography.SHA256]::Create() + try { + return ([System.BitConverter]::ToString($sha.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Get-Base64Url { + param([string]$Value) + return [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($Value)) +} + +function Get-OfficialDetailUrl { + param([string]$Bbbs) + return "https://flk.npc.gov.cn/detail2.html?$(Get-Base64Url -Value $Bbbs)" +} + +function Invoke-OfficialJson { + param([string]$Url) + $client = [System.Net.Http.HttpClient]::new() + try { + $client.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0") + $client.DefaultRequestHeaders.Add("Referer", "https://flk.npc.gov.cn/") + $bytes = $client.GetByteArrayAsync($Url).GetAwaiter().GetResult() + $text = [System.Text.Encoding]::UTF8.GetString($bytes) + return $text | ConvertFrom-Json + } finally { + $client.Dispose() + } +} + +function Get-OfficialLawDetail { + param([string]$Bbbs) + $encoded = [uri]::EscapeDataString($Bbbs) + $response = Invoke-OfficialJson -Url "https://flk.npc.gov.cn/law-search/search/flfgDetails?bbbs=$encoded" + if ($response.code -ne 200 -or $null -eq $response.data) { + throw "Official detail request failed for $Bbbs." + } + return $response.data +} + +function Get-OfficialDocxUrl { + param([string]$Bbbs) + $encoded = [uri]::EscapeDataString($Bbbs) + $response = Invoke-OfficialJson -Url "https://flk.npc.gov.cn/law-search/download/pc?format=docx&bbbs=$encoded&fileId=" + if ($response.code -ne 200 -or $null -eq $response.data -or [string]::IsNullOrWhiteSpace([string]$response.data.url)) { + throw "Official docx download URL request failed for $Bbbs." + } + return [string]$response.data.url +} + +function Save-OfficialDocx { + param([string]$Url, [string]$Path) + Invoke-WebRequest -Uri $Url -OutFile $Path -UseBasicParsing -Headers @{ "User-Agent" = "Mozilla/5.0"; "Referer" = "https://flk.npc.gov.cn/" } -ErrorAction Stop +} + +function Convert-DocxToParagraphs { + param([string]$DocxPath, [string]$WorkDir) + $zipPath = Join-Path $WorkDir "document.zip" + $unzipDir = Join-Path $WorkDir "document" + Copy-Item -LiteralPath $DocxPath -Destination $zipPath -Force + Expand-Archive -LiteralPath $zipPath -DestinationPath $unzipDir -Force + $documentXmlPath = Join-Path $unzipDir "word\document.xml" + if (-not (Test-Path -LiteralPath $documentXmlPath -PathType Leaf)) { + throw "word/document.xml not found in official docx." + } + $xml = Get-Content -LiteralPath $documentXmlPath -Raw -Encoding UTF8 + $paragraphs = New-Object System.Collections.Generic.List[string] + foreach ($paragraphMatch in [regex]::Matches($xml, "")) { + $parts = New-Object System.Collections.Generic.List[string] + foreach ($textMatch in [regex]::Matches($paragraphMatch.Value, "]*>([\s\S]*?)")) { + $parts.Add([System.Net.WebUtility]::HtmlDecode($textMatch.Groups[1].Value)) | Out-Null + } + $text = ($parts.ToArray() -join "").Trim() + if (-not [string]::IsNullOrWhiteSpace($text)) { + $paragraphs.Add($text) | Out-Null + } + } + return $paragraphs.ToArray() +} + +function Test-ArticleHeader { + param([string]$Text) + if ([string]::IsNullOrWhiteSpace($Text)) { return $false } + if (-not $Text.StartsWith($articlePrefix)) { return $false } + $suffixIndex = $Text.IndexOf($articleSuffix) + return $suffixIndex -gt 0 -and $suffixIndex -le 16 +} + +function Get-ArticleText { + param([string[]]$Paragraphs, [string]$ArticleNo) + $start = -1 + for ($i = 0; $i -lt $Paragraphs.Count; $i++) { + if ($Paragraphs[$i].StartsWith($ArticleNo)) { + $start = $i + break + } + } + if ($start -lt 0) { + throw "Article not found: $ArticleNo" + } + $selected = New-Object System.Collections.Generic.List[string] + for ($i = $start; $i -lt $Paragraphs.Count; $i++) { + if ($i -gt $start -and (Test-ArticleHeader -Text $Paragraphs[$i])) { + break + } + $selected.Add($Paragraphs[$i]) | Out-Null + } + return ($selected.ToArray() -join "`n").Trim() +} + +function ConvertTo-TagsJson { + param([object[]]$Values) + $tags = New-Object System.Collections.Generic.List[string] + foreach ($value in $Values) { + $text = ([string]$value).Trim() + if (-not [string]::IsNullOrWhiteSpace($text) -and -not $tags.Contains($text)) { + $tags.Add($text) | Out-Null + } + } + if ($tags.Count -eq 0) { return $null } + return ($tags.ToArray() | ConvertTo-Json -Compress) +} + +function Ensure-KnowledgeBase { + param([hashtable]$Headers) + $items = Get-PagedApiItems -Url "$ApiBaseUrl/knowledge-bases" -Headers $Headers + $existing = @($items | Where-Object { $_.name -eq $KnowledgeBaseName }) | Select-Object -First 1 + if ($existing) { return $existing } + $response = Invoke-FormRequest -Method "POST" -Url "$ApiBaseUrl/knowledge-bases" -Headers $Headers -Fields @{ + name = $KnowledgeBaseName + description = "Phase14 legal article fulltext library imported from official law sources." + } + return Get-ResponseData -Response $response +} + +function New-ArticleMarkdown { + param([object]$Source, [object]$Article, [string]$ArticleText, [string]$OfficialUrl) + return @" +# $($Source.title) $($Article.article_no) + +## Source metadata + +- source_id: $($Source.source_id) +- source_title: $($Source.title) +- source_type: $($Source.source_type) +- authority: $($Source.authority) +- published_date: $($Source.published_date) +- effective_date: $($Source.effective_date) +- validity_status: $($Source.validity_status) +- region: $($Source.region) +- official_url: $OfficialUrl +- chapter_title: $($Article.chapter_title) +- section_title: $($Article.section_title) + +## Article full text + +$ArticleText +"@ +} + +function Import-ArticleDocument { + param( + [hashtable]$Headers, + [string]$KnowledgeBaseId, + [object]$Source, + [object]$Article, + [string]$ArticleText, + [string]$OfficialUrl, + [object[]]$ExistingDocuments + ) + $fileArticleNo = (Get-TextChecksum -Text $Article.article_no).Substring(0, 8) + $fileName = "$($Source.source_id)-$fileArticleNo.md" + $existing = @($ExistingDocuments | Where-Object { $_.file_name -eq $fileName }) | Select-Object -First 1 + $tagsJson = ConvertTo-TagsJson -Values @($Source.source_id, $Source.knowledge_domain, "phase14", "legal-fulltext", $Article.tags) + $fields = @{ + note = "Phase14 legal article fulltext import." + source_type = $Source.source_type + authority = $Source.authority + effective_date = $Source.effective_date + region = $Source.region + } + if ($tagsJson) { $fields["tags_json"] = $tagsJson } + $content = New-ArticleMarkdown -Source $Source -Article $Article -ArticleText $ArticleText -OfficialUrl $OfficialUrl + $bytes = [System.Text.Encoding]::UTF8.GetBytes($content) + + if ($existing -and -not $ForceReupload) { + Add-Import -SourceId $Source.source_id -ArticleNo $Article.article_no -Action "skip_existing" -Status "skipped" -TargetId $existing.id -Detail "Existing knowledge document reused." -Meta @{ file_name = $fileName; status = $existing.status } + return $existing + } + if ($existing -and $ForceReupload) { + $response = Invoke-FormRequest -Method "POST" -Url "$ApiBaseUrl/knowledge-bases/$KnowledgeBaseId/documents/$($existing.id)/retry" -Headers $Headers -Fields $fields -FileField "file" -FileName $fileName -FileContent $bytes + $data = Get-ResponseData -Response $response + Add-Import -SourceId $Source.source_id -ArticleNo $Article.article_no -Action "retry_existing" -Status $data.status -TargetId $data.id -Detail $data.status_detail -Meta @{ file_name = $fileName } + return $data + } + + $response = Invoke-FormRequest -Method "POST" -Url "$ApiBaseUrl/knowledge-bases/$KnowledgeBaseId/documents" -Headers $Headers -Fields $fields -FileField "file" -FileName $fileName -FileContent $bytes + $created = Get-ResponseData -Response $response + Add-Import -SourceId $Source.source_id -ArticleNo $Article.article_no -Action "upload" -Status $created.status -TargetId $created.id -Detail $created.status_detail -Meta @{ file_name = $fileName } + return $created +} + +if (-not (Test-Path -LiteralPath $ManifestPath -PathType Leaf)) { + Add-Check -Key "manifest_exists" -Label "Manifest exists" -Status "fail" -Detail "Manifest missing: $ManifestPath" +} else { + Add-Check -Key "manifest_exists" -Label "Manifest exists" -Status "pass" -Detail $ManifestPath +} + +$manifest = $null +if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0) { + $manifest = Get-Content -LiteralPath $ManifestPath -Raw -Encoding UTF8 | ConvertFrom-Json + $sourceCount = @($manifest.sources).Count + $articleCount = 0 + foreach ($source in @($manifest.sources)) { $articleCount += @($source.articles).Count } + if ($sourceCount -gt 0 -and $articleCount -gt 0) { + Add-Check -Key "manifest_shape" -Label "Manifest shape" -Status "pass" -Detail "sources=$sourceCount, articles=$articleCount" -Meta @{ sources = $sourceCount; articles = $articleCount } + } else { + Add-Check -Key "manifest_shape" -Label "Manifest shape" -Status "fail" -Detail "Manifest has no sources or articles." -Meta @{ sources = $sourceCount; articles = $articleCount } + } +} + +$token = "" +$headers = @{} +$targetKnowledgeBase = $null +if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0) { + $token = Get-AccessToken + $headers = Get-AuthHeaders -Token $token + Add-Check -Key "api_login" -Label "API login" -Status "pass" -Detail "Admin API login succeeded." + $targetKnowledgeBase = Ensure-KnowledgeBase -Headers $headers + Add-Check -Key "target_knowledge_base" -Label "Target knowledge base" -Status "pass" -Detail $targetKnowledgeBase.name -Meta @{ knowledge_base_id = $targetKnowledgeBase.id } +} + +if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0) { + $existingDocs = Get-PagedApiItems -Url "$ApiBaseUrl/knowledge-bases/$($targetKnowledgeBase.id)/documents" -Headers $headers + foreach ($source in @($manifest.sources)) { + try { + $officialDetail = Get-OfficialLawDetail -Bbbs $source.bbbs + $officialUrl = Get-OfficialDetailUrl -Bbbs $source.bbbs + if ([string]$officialDetail.title -ne [string]$source.title) { + Add-Check -Key "official_title_$($source.source_id)" -Label "Official title check $($source.source_id)" -Status "fail" -Detail "Expected=$($source.title), actual=$($officialDetail.title)" + continue + } + $docxUrl = Get-OfficialDocxUrl -Bbbs $source.bbbs + $sourceDir = Join-Path $downloadDir $source.source_id + New-Item -ItemType Directory -Force -Path $sourceDir | Out-Null + $docxPath = Join-Path $sourceDir "official.docx" + Save-OfficialDocx -Url $docxUrl -Path $docxPath + $docxBytes = [System.IO.File]::ReadAllBytes($docxPath) + $sourceChecksum = Get-TextChecksum -Text ([Convert]::ToBase64String($docxBytes)) + $paragraphs = Convert-DocxToParagraphs -DocxPath $docxPath -WorkDir $sourceDir + Add-Check -Key "official_download_$($source.source_id)" -Label "Official download $($source.source_id)" -Status "pass" -Detail "paragraphs=$($paragraphs.Count)" -Meta @{ title = $source.title; bbbs = $source.bbbs; checksum = $sourceChecksum } + + foreach ($article in @($source.articles)) { + try { + $articleText = Get-ArticleText -Paragraphs $paragraphs -ArticleNo $article.article_no + $knowledgeDocument = Import-ArticleDocument -Headers $headers -KnowledgeBaseId $targetKnowledgeBase.id -Source $source -Article $article -ArticleText $articleText -OfficialUrl $officialUrl -ExistingDocuments $existingDocs + $articlePayloads.Add([ordered]@{ + source_id = $source.source_id + source_title = $source.title + source_type = $source.source_type + authority = $source.authority + effective_date = $source.effective_date + validity_status = $source.validity_status + region = $source.region + official_url = $officialUrl + article_no = $article.article_no + chapter_title = $article.chapter_title + section_title = $article.section_title + article_text = $articleText + tags = @($source.source_id, $source.knowledge_domain, "phase14", "legal-fulltext") + @($article.tags) + source_checksum = $sourceChecksum + text_checksum = Get-TextChecksum -Text $articleText + knowledge_base_id = $targetKnowledgeBase.id + knowledge_document_id = $knowledgeDocument.id + index_status = if ($knowledgeDocument.status -eq "indexed") { "indexed" } else { [string]$knowledgeDocument.status } + published_status = "published" + published_by = "CODEX_PHASE14_IMPORT" + }) | Out-Null + } catch { + Add-Import -SourceId $source.source_id -ArticleNo $article.article_no -Action "extract_upload" -Status "failed" -TargetId "" -Detail $_.Exception.Message + } + } + } catch { + Add-Check -Key "official_download_$($source.source_id)" -Label "Official download $($source.source_id)" -Status "fail" -Detail $_.Exception.Message + } + } +} + +$importFailures = @($imports | Where-Object { $_.status -eq "failed" }).Count +if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0 -and $importFailures -eq 0) { + $bulkResponse = Invoke-ApiJson -Method "POST" -Url "$ApiBaseUrl/legal-articles/bulk-upsert" -Headers $headers -Body @{ articles = $articlePayloads.ToArray() } + $bulkData = Get-ResponseData -Response $bulkResponse + Add-Check -Key "legal_articles_bulk_upsert" -Label "Legal articles bulk upsert" -Status "pass" -Detail "total=$($bulkData.total), created=$($bulkData.created_count), updated=$($bulkData.updated_count)" -Meta @{ total = $bulkData.total; created = $bulkData.created_count; updated = $bulkData.updated_count } + + if ($SkipKnowledgeBasePublish) { + Add-Import -SourceId "knowledge_base" -ArticleNo "" -Action "publish" -Status "skipped" -TargetId $targetKnowledgeBase.id -Detail "Skipped by parameter." + } else { + $publishResponse = Invoke-ApiJson -Method "POST" -Url "$ApiBaseUrl/knowledge-bases/$($targetKnowledgeBase.id)/publish" -Headers $headers + $publishedKb = Get-ResponseData -Response $publishResponse + Add-Import -SourceId "knowledge_base" -ArticleNo "" -Action "publish" -Status "published" -TargetId $publishedKb.id -Detail "Knowledge base published." -Meta @{ current_version_label = $publishedKb.current_version_label; published_at = $publishedKb.published_at } + } +} + +if ((@($checks | Where-Object { $_.required -and $_.status -eq "fail" })).Count -eq 0 -and $importFailures -eq 0) { + $expectedArticleCount = $articlePayloads.Count + $documentsAfter = Get-PagedApiItems -Url "$ApiBaseUrl/knowledge-bases/$($targetKnowledgeBase.id)/documents" -Headers $headers + $documentIds = @($articlePayloads | ForEach-Object { $_.knowledge_document_id }) + $targetDocs = @($documentsAfter | Where-Object { $documentIds -contains $_.id }) + $indexedDocs = @($targetDocs | Where-Object { $_.status -eq "indexed" }) + if ($targetDocs.Count -eq $expectedArticleCount -and $indexedDocs.Count -eq $expectedArticleCount) { + Add-Verification -Key "knowledge_documents_indexed" -Status "pass" -Detail "All legal article documents are indexed." -Meta @{ expected = $expectedArticleCount; target_documents = $targetDocs.Count; indexed = $indexedDocs.Count } + } else { + Add-Verification -Key "knowledge_documents_indexed" -Status "fail" -Detail "Legal article document indexing is incomplete." -Meta @{ expected = $expectedArticleCount; target_documents = $targetDocs.Count; indexed = $indexedDocs.Count } + } + + $apiArticles = Get-PagedApiItems -Url "$ApiBaseUrl/legal-articles?published_status=published&index_status=indexed" -Headers $headers + $expectedChecksums = @($articlePayloads | ForEach-Object { $_.text_checksum }) + $matchedArticles = @($apiArticles | Where-Object { $expectedChecksums -contains $_.text_checksum }) + if ($matchedArticles.Count -eq $expectedArticleCount) { + Add-Verification -Key "legal_articles_runtime_table" -Status "pass" -Detail "All legal articles are in runtime table." -Meta @{ expected = $expectedArticleCount; matched = $matchedArticles.Count } + } else { + Add-Verification -Key "legal_articles_runtime_table" -Status "fail" -Detail "Runtime legal article table is incomplete." -Meta @{ expected = $expectedArticleCount; matched = $matchedArticles.Count } + } +} + +$failedRequired = @($checks | Where-Object { $_.required -and $_.status -eq "fail" }).Count +$failedVerification = @($verification | Where-Object { $_.status -eq "fail" }).Count +$warnings = @($checks | Where-Object { $_.status -eq "warn" }).Count + @($verification | Where-Object { $_.status -eq "warn" }).Count +$overallStatus = if ($failedRequired -gt 0 -or $failedVerification -gt 0 -or $importFailures -gt 0) { "FAIL" } elseif ($warnings -gt 0) { "WARN" } else { "PASS" } + +$summary = [ordered]@{ + name = "phase14-legal-fulltext-import" + status = $overallStatus + generated_at = (Get-Date).ToUniversalTime().ToString("o") + api_base_url = $ApiBaseUrl + manifest_path = $ManifestPath + artifact_dir = $artifactDir + knowledge_base_name = $KnowledgeBaseName + knowledge_base_id = if ($targetKnowledgeBase) { $targetKnowledgeBase.id } else { "" } + checks = $checks.ToArray() + imports = $imports.ToArray() + verification = $verification.ToArray() + totals = [ordered]@{ + sources = if ($manifest) { @($manifest.sources).Count } else { 0 } + articles_extracted = $articlePayloads.Count + import_failures = $importFailures + failed_required_checks = $failedRequired + failed_verifications = $failedVerification + } +} +Save-Json -Path $summaryJson -Value $summary + +$md = New-Object System.Collections.Generic.List[string] +$md.Add("# Phase14 legal fulltext import report") | Out-Null +$md.Add("") | Out-Null +$md.Add("- status: $overallStatus") | Out-Null +$md.Add("- generated_at: $($summary.generated_at)") | Out-Null +$md.Add("- api_base_url: $ApiBaseUrl") | Out-Null +$md.Add("- manifest_path: $ManifestPath") | Out-Null +$md.Add("- knowledge_base_name: $KnowledgeBaseName") | Out-Null +$md.Add("- knowledge_base_id: $($summary.knowledge_base_id)") | Out-Null +$md.Add("") | Out-Null +$md.Add("## Totals") | Out-Null +$md.Add("") | Out-Null +$md.Add("| item | value |") | Out-Null +$md.Add("|---|---:|") | Out-Null +$md.Add("| sources | $($summary.totals.sources) |") | Out-Null +$md.Add("| articles_extracted | $($summary.totals.articles_extracted) |") | Out-Null +$md.Add("| import_failures | $($summary.totals.import_failures) |") | Out-Null +$md.Add("| failed_required_checks | $($summary.totals.failed_required_checks) |") | Out-Null +$md.Add("| failed_verifications | $($summary.totals.failed_verifications) |") | Out-Null +$md.Add("") | Out-Null +$md.Add("## Checks") | Out-Null +$md.Add("") | Out-Null +$md.Add("| key | required | status | detail |") | Out-Null +$md.Add("|---|---:|---|---|") | Out-Null +foreach ($check in $checks) { + $detail = ([string]$check.detail).Replace("|", "/") + $requiredText = if ($check.required) { "yes" } else { "no" } + $md.Add("| $($check.key) | $requiredText | $($check.status) | $detail |") | Out-Null +} +$md.Add("") | Out-Null +$md.Add("## Imports") | Out-Null +$md.Add("") | Out-Null +$md.Add("| source_id | article_no | action | status | target_id | detail |") | Out-Null +$md.Add("|---|---|---|---|---|---|") | Out-Null +foreach ($item in $imports) { + $detail = ([string]$item.detail).Replace("|", "/") + $md.Add("| $($item.source_id) | $($item.article_no) | $($item.action) | $($item.status) | $($item.target_id) | $detail |") | Out-Null +} +$md.Add("") | Out-Null +$md.Add("## Verification") | Out-Null +$md.Add("") | Out-Null +$md.Add("| key | status | detail |") | Out-Null +$md.Add("|---|---|---|") | Out-Null +foreach ($item in $verification) { + $detail = ([string]$item.detail).Replace("|", "/") + $md.Add("| $($item.key) | $($item.status) | $detail |") | Out-Null +} +$md | Set-Content -Encoding UTF8 -Path $summaryMd + +Write-Host "PHASE14_LEGAL_FULLTEXT_IMPORT_$overallStatus $artifactDir" +if ($overallStatus -eq "FAIL") { + throw "Phase14 legal fulltext import failed. See: $summaryMd" +}