yuqei
6abca4eb75
Security Quality Gate / npm audit, Bandit, pip-audit, detect-secrets (pull_request) Successful in 10m53s
1582 lines
72 KiB
Python
1582 lines
72 KiB
Python
import json
|
|
from urllib.parse import parse_qs
|
|
|
|
import httpx
|
|
|
|
from yuqei_sdk import (
|
|
AiPlatformClient,
|
|
AiQualityImprovementActionCreate,
|
|
AiQualityImprovementActionUpdate,
|
|
AiQualityLabelReviewUpdate,
|
|
AiQualityRuleConfigUpdate,
|
|
ContractReviewRequest,
|
|
KnowledgeDocumentCreate,
|
|
KnowledgeImportBatchCreate,
|
|
KnowledgeReindexRequest,
|
|
LegalQaRequest,
|
|
PromptTemplateCreate,
|
|
PromptTemplatePublishRequest,
|
|
PromptTemplateRollbackRequest,
|
|
ProviderConfigCreate,
|
|
ProviderConfigUpsertRequest,
|
|
RequestContext,
|
|
WorkerRunOnceRequest,
|
|
)
|
|
|
|
|
|
def test_ai_platform_client_posts_legal_qa_with_context_headers() -> None:
|
|
captured_request: httpx.Request | None = None
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal captured_request
|
|
captured_request = request
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"answer": "Use Article 19 and verify the contract term.",
|
|
"citations": [
|
|
{
|
|
"source_type": "law",
|
|
"title": "Labor Contract Law",
|
|
"reference": "Article 19 chunk-1",
|
|
"quote": "Probation period must stay within the statutory limit.",
|
|
"score": 0.82,
|
|
"locator": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"chunk_id": "chunk-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"keyword_score": 0.78,
|
|
"vector_score": 0.87,
|
|
"hybrid_score": 0.82,
|
|
"retrieval_strategy": "hybrid",
|
|
"matched_terms": ["probation"],
|
|
}
|
|
],
|
|
"primary_sources": [
|
|
{
|
|
"source_type": "law",
|
|
"title": "Labor Contract Law",
|
|
"reference": "Article 19 chunk-1",
|
|
"quote": "Probation period must stay within the statutory limit.",
|
|
"score": 0.82,
|
|
"locator": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"chunk_id": "chunk-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"keyword_score": 0.78,
|
|
"vector_score": 0.87,
|
|
"hybrid_score": 0.82,
|
|
"retrieval_strategy": "hybrid",
|
|
"matched_terms": ["probation"],
|
|
}
|
|
],
|
|
"confidence": 0.72,
|
|
},
|
|
)
|
|
|
|
client = AiPlatformClient(
|
|
"http://ai-platform.test",
|
|
transport=httpx.MockTransport(handler),
|
|
service_key="ai-platform-service-key", # pragma: allowlist secret
|
|
)
|
|
|
|
response = client.run_legal_qa(
|
|
LegalQaRequest(question="How should probation period be set?"),
|
|
context=RequestContext(trace_id="trace-1", tenant_id="tenant-1", user_id="user-1"),
|
|
)
|
|
|
|
assert response.citations[0].reference == "Article 19 chunk-1"
|
|
assert response.primary_sources[0].score == 0.82
|
|
assert response.primary_sources[0].hybrid_score == 0.82
|
|
assert response.primary_sources[0].retrieval_strategy == "hybrid"
|
|
assert captured_request is not None
|
|
assert captured_request.url.path == "/api/v1/legal/qa"
|
|
assert captured_request.headers["X-Trace-Id"] == "trace-1"
|
|
assert captured_request.headers["X-Tenant-Id"] == "tenant-1"
|
|
assert captured_request.headers["X-User-Id"] == "user-1"
|
|
assert captured_request.headers["X-AI-Platform-Service-Key"] == "ai-platform-service-key"
|
|
|
|
|
|
def test_ai_platform_client_posts_contract_review_with_context_headers() -> None:
|
|
captured_request: httpx.Request | None = None
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
nonlocal captured_request
|
|
captured_request = request
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"summary": "The contract has payment and acceptance risks.",
|
|
"risk_level": "high",
|
|
"risk_items": [
|
|
{
|
|
"id": "risk-payment-acceptance",
|
|
"title": "Payment and acceptance should be clarified.",
|
|
"severity": "high",
|
|
"suggestion": "Clarify delivery, acceptance period and payment trigger.",
|
|
"related_clause": "Payment",
|
|
"citation_refs": ["Civil Code Article 509"],
|
|
}
|
|
],
|
|
"citations": [
|
|
{
|
|
"source_type": "law",
|
|
"title": "Civil Code",
|
|
"reference": "Article 509",
|
|
"quote": "Parties shall fully perform their obligations.",
|
|
}
|
|
],
|
|
"primary_sources": [
|
|
{
|
|
"source_type": "law",
|
|
"title": "Civil Code",
|
|
"reference": "Article 509",
|
|
"quote": "Parties shall fully perform their obligations.",
|
|
}
|
|
],
|
|
"confidence": 0.78,
|
|
"audit_run_id": "run-review-1",
|
|
},
|
|
)
|
|
|
|
client = AiPlatformClient(
|
|
"http://ai-platform.test",
|
|
transport=httpx.MockTransport(handler),
|
|
)
|
|
|
|
response = client.run_contract_review(
|
|
ContractReviewRequest(
|
|
contract_title="Technical Service Contract",
|
|
contract_text="Payment is due after acceptance.",
|
|
review_focus="payment and acceptance",
|
|
),
|
|
context=RequestContext(trace_id="trace-review", tenant_id="tenant-1", user_id="contract-api"),
|
|
)
|
|
|
|
assert response.risk_level == "high"
|
|
assert response.risk_items[0].id == "risk-payment-acceptance"
|
|
assert response.citations[0].reference == "Article 509"
|
|
assert response.audit_run_id == "run-review-1"
|
|
assert captured_request is not None
|
|
assert captured_request.url.path == "/api/v1/legal/contract-review"
|
|
assert captured_request.headers["X-Trace-Id"] == "trace-review"
|
|
assert captured_request.headers["X-Tenant-Id"] == "tenant-1"
|
|
assert captured_request.headers["X-User-Id"] == "contract-api"
|
|
|
|
|
|
def test_ai_platform_client_covers_admin_data_endpoints() -> None:
|
|
seen_paths: list[str] = []
|
|
|
|
def quality_action_payload(
|
|
*,
|
|
status: str = "open",
|
|
resolution_note: str | None = None,
|
|
resolved_at: str | None = None,
|
|
) -> dict[str, object]:
|
|
return {
|
|
"id": "qa-action-1",
|
|
"dimension": "model",
|
|
"key": "deepseek / deepseek-chat",
|
|
"label": "deepseek / deepseek-chat",
|
|
"action_type": "tune_model",
|
|
"title": "Tune model config: deepseek / deepseek-chat",
|
|
"description": "Check provider timeout and model configuration.",
|
|
"owner": "admin",
|
|
"priority": "high",
|
|
"status": status,
|
|
"source_label_codes": ["source_limited"],
|
|
"source_run_ids": ["run-1"],
|
|
"root_cause_hint": "Model quality issues are concentrated.",
|
|
"linked_prompt_template_id": "prompt-1",
|
|
"linked_prompt_template_version": "v2",
|
|
"linked_knowledge_base_id": "laws-cn",
|
|
"linked_import_batch_id": "kb-batch-1",
|
|
"evaluation_days": 7,
|
|
"baseline_total_runs": 1,
|
|
"baseline_affected_runs": 1,
|
|
"baseline_pending_labels": 1,
|
|
"baseline_quality_score": 90.0,
|
|
"current_total_runs": 1,
|
|
"current_affected_runs": 1,
|
|
"current_pending_labels": 1,
|
|
"current_quality_score": 90.0,
|
|
"post_change_total_runs": 2,
|
|
"post_change_affected_runs": 0,
|
|
"post_change_pending_labels": 0,
|
|
"post_change_quality_score": 98.0,
|
|
"post_change_affected_rate": 0.0,
|
|
"post_change_observation_days": 3,
|
|
"affected_rate_delta": 0.0,
|
|
"pending_labels_delta": 0,
|
|
"quality_score_delta": 0.0,
|
|
"post_change_affected_rate_delta": -1.0,
|
|
"post_change_pending_labels_delta": -1,
|
|
"post_change_quality_score_delta": 8.0,
|
|
"related_change_actions_count": 2,
|
|
"effect_status": "unchanged",
|
|
"effect_summary": "Baseline affected rate 100%, post-change 0% over 2 run(s); pending labels delta -1; quality score delta +8.0.",
|
|
"resolution_note": resolution_note,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
"updated_at": "2026-06-22T00:02:00Z",
|
|
"resolved_at": resolved_at,
|
|
}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen_paths.append(request.url.path)
|
|
if request.url.path == "/api/v1/model-configs" and request.method == "POST":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["change_note"] == "Use production legal QA model."
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "provider-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"display_name": "DeepSeek Chat",
|
|
"base_url": None,
|
|
"temperature": 0.1,
|
|
"max_tokens": 4096,
|
|
"enabled": True,
|
|
"is_default": True,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/model-configs/changes":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "provider-change-1",
|
|
"provider_config_id": "provider-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"display_name": "DeepSeek Chat",
|
|
"before_config": None,
|
|
"after_config": {
|
|
"id": "provider-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"display_name": "DeepSeek Chat",
|
|
"base_url": None,
|
|
"temperature": 0.1,
|
|
"max_tokens": 4096,
|
|
"enabled": True,
|
|
"is_default": True,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
"changed_by": "admin",
|
|
"change_note": "Use production legal QA model.",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/model-configs":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "provider-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"display_name": "DeepSeek Chat",
|
|
"base_url": None,
|
|
"temperature": 0.1,
|
|
"max_tokens": 4096,
|
|
"enabled": True,
|
|
"is_default": True,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/prompt-templates" and request.method == "POST":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"content": "Search evidence first, then answer.",
|
|
"enabled": True,
|
|
"status": "draft",
|
|
"rollback_from_version": None,
|
|
"published_at": None,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/prompt-templates/legal_qa.default/publish":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["version"] == "v2"
|
|
assert payload["changed_by"] == "admin"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"content": "Search evidence first, then answer.",
|
|
"enabled": True,
|
|
"status": "published",
|
|
"rollback_from_version": None,
|
|
"published_at": "2026-06-22T00:01:00Z",
|
|
"updated_at": "2026-06-22T00:01:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/prompt-templates/legal_qa.default/rollback":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["from_version"] == "v1"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "prompt-rollback-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v1-rollback",
|
|
"content": "Use the stable prompt.",
|
|
"enabled": True,
|
|
"status": "published",
|
|
"rollback_from_version": "v1",
|
|
"published_at": "2026-06-22T00:02:00Z",
|
|
"updated_at": "2026-06-22T00:02:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/prompt-templates/changes":
|
|
assert request.url.params["prompt_template_id"] == "prompt-1"
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "prompt-change-1",
|
|
"prompt_template_id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"operation": "publish",
|
|
"before_template": None,
|
|
"after_template": {
|
|
"id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"content": "Search evidence first, then answer.",
|
|
"enabled": True,
|
|
"status": "published",
|
|
"rollback_from_version": None,
|
|
"published_at": "2026-06-22T00:01:00Z",
|
|
"updated_at": "2026-06-22T00:01:00Z",
|
|
},
|
|
"changed_by": "admin",
|
|
"change_note": "Publish prompt for quality action.",
|
|
"created_at": "2026-06-22T00:01:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/prompt-templates":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"content": "Search evidence first, then answer.",
|
|
"enabled": True,
|
|
"status": "published",
|
|
"rollback_from_version": None,
|
|
"published_at": "2026-06-22T00:01:00Z",
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/quality-rules/default" and request.method == "PUT":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["min_source_count"] == 1
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "default",
|
|
"min_citation_count": 1,
|
|
"min_source_count": 1,
|
|
"high_token_threshold": 9000,
|
|
"high_latency_ms": 12000,
|
|
"critical_latency_ms": 24000,
|
|
"warning_penalty": 7,
|
|
"error_penalty": 21,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/quality-rules/default":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "default",
|
|
"min_citation_count": 1,
|
|
"min_source_count": 2,
|
|
"high_token_threshold": 8000,
|
|
"high_latency_ms": 15000,
|
|
"critical_latency_ms": 30000,
|
|
"warning_penalty": 10,
|
|
"error_penalty": 30,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/runtime/worker-health":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"status": "ok",
|
|
"queue_counts": {"queued": 1, "running": 0, "succeeded": 2, "failed": 0},
|
|
"workers": [
|
|
{
|
|
"worker_name": "knowledge-worker-1",
|
|
"status": "ok",
|
|
"queue_name": "yuqei-v2",
|
|
"processed": 3,
|
|
"succeeded": 2,
|
|
"failed": 0,
|
|
"error_message": None,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
"stale_running_jobs": 0,
|
|
"checked_at": "2026-06-22T00:00:01Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/runtime/worker-runs/run-once":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["limit"] == 1
|
|
assert payload["worker_name"] == "manual-sdk-worker"
|
|
assert payload["triggered_by"] == "admin"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "worker-run-1",
|
|
"worker_name": "manual-sdk-worker",
|
|
"trigger_type": "manual",
|
|
"triggered_by": "admin",
|
|
"queue_name": "manual",
|
|
"status": "succeeded",
|
|
"limit": 1,
|
|
"processed": 1,
|
|
"succeeded": 1,
|
|
"failed": 0,
|
|
"error_message": None,
|
|
"duration_ms": 15,
|
|
"started_at": "2026-06-22T00:00:02Z",
|
|
"completed_at": "2026-06-22T00:00:03Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/runtime/worker-runs":
|
|
assert request.url.params["limit"] == "5"
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "worker-run-1",
|
|
"worker_name": "manual-sdk-worker",
|
|
"trigger_type": "manual",
|
|
"triggered_by": "admin",
|
|
"queue_name": "manual",
|
|
"status": "succeeded",
|
|
"limit": 1,
|
|
"processed": 1,
|
|
"succeeded": 1,
|
|
"failed": 0,
|
|
"error_message": None,
|
|
"duration_ms": 15,
|
|
"started_at": "2026-06-22T00:00:02Z",
|
|
"completed_at": "2026-06-22T00:00:03Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-import-batches" and request.method == "POST":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "civil-code-lease",
|
|
"source_type": "law_import",
|
|
"status": "indexed",
|
|
"documents_count": 0,
|
|
"chunks_count": 0,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-bases/laws-cn/documents" and request.method == "POST":
|
|
body = request.read()
|
|
assert b'filename="lease.md"' in body
|
|
assert b"Deposit refund rules" in body
|
|
assert request.headers["X-Trace-Id"] == "trace-upload"
|
|
return httpx.Response(
|
|
202,
|
|
json={
|
|
"id": "ingest-upload-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "lease.md",
|
|
"source_type": "file_upload",
|
|
"status": "queued",
|
|
"parse_status": "pending",
|
|
"ocr_status": "not_required",
|
|
"import_batch_id": None,
|
|
"document_id": None,
|
|
"title": "Lease Upload",
|
|
"reference": "lease.md",
|
|
"mime_type": "text/markdown",
|
|
"file_size_bytes": 43,
|
|
"storage_backend": "local",
|
|
"storage_bucket": "ai-platform-uploads",
|
|
"storage_object_key": "knowledge-ingestion/test.md",
|
|
"documents_count": 0,
|
|
"chunks_count": 0,
|
|
"retry_count": 0,
|
|
"max_retries": 3,
|
|
"error_message": None,
|
|
"worker_name": None,
|
|
"idempotency_key": None,
|
|
"created_at": "2026-06-22T00:03:00Z",
|
|
"updated_at": "2026-06-22T00:03:00Z",
|
|
"started_at": None,
|
|
"completed_at": None,
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-ingestion-jobs" and request.method == "GET":
|
|
assert request.url.params["knowledge_base_id"] == "laws-cn"
|
|
assert request.url.params["status"] == "queued"
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "ingest-upload-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "lease.md",
|
|
"source_type": "file_upload",
|
|
"status": "queued",
|
|
"parse_status": "pending",
|
|
"ocr_status": "not_required",
|
|
"import_batch_id": None,
|
|
"document_id": None,
|
|
"title": "Lease Upload",
|
|
"reference": "lease.md",
|
|
"mime_type": "text/markdown",
|
|
"file_size_bytes": 43,
|
|
"storage_backend": "local",
|
|
"storage_bucket": "ai-platform-uploads",
|
|
"storage_object_key": "knowledge-ingestion/test.md",
|
|
"documents_count": 0,
|
|
"chunks_count": 0,
|
|
"retry_count": 0,
|
|
"max_retries": 3,
|
|
"error_message": None,
|
|
"worker_name": None,
|
|
"idempotency_key": None,
|
|
"created_at": "2026-06-22T00:03:00Z",
|
|
"updated_at": "2026-06-22T00:03:00Z",
|
|
"started_at": None,
|
|
"completed_at": None,
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-ingestion-jobs/ingest-upload-1/retry":
|
|
assert request.url.params["triggered_by"] == "admin"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "ingest-upload-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "lease.md",
|
|
"source_type": "file_upload",
|
|
"status": "queued",
|
|
"parse_status": "pending",
|
|
"ocr_status": "not_required",
|
|
"import_batch_id": None,
|
|
"document_id": None,
|
|
"title": "Lease Upload",
|
|
"reference": "lease.md",
|
|
"mime_type": "text/markdown",
|
|
"file_size_bytes": 43,
|
|
"storage_backend": "local",
|
|
"storage_bucket": "ai-platform-uploads",
|
|
"storage_object_key": "knowledge-ingestion/test.md",
|
|
"documents_count": 0,
|
|
"chunks_count": 0,
|
|
"retry_count": 1,
|
|
"max_retries": 3,
|
|
"error_message": None,
|
|
"worker_name": "admin",
|
|
"idempotency_key": None,
|
|
"created_at": "2026-06-22T00:03:00Z",
|
|
"updated_at": "2026-06-22T00:04:00Z",
|
|
"started_at": None,
|
|
"completed_at": None,
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-import-batches":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "civil-code-lease",
|
|
"source_type": "law_import",
|
|
"status": "indexed",
|
|
"documents_count": 1,
|
|
"chunks_count": 1,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-import-batches/kb-batch-1":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"batch": {
|
|
"id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "civil-code-lease",
|
|
"source_type": "law_import",
|
|
"status": "indexed",
|
|
"documents_count": 1,
|
|
"chunks_count": 1,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
"documents": [
|
|
{
|
|
"id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"index_status": "indexed",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
"chunks": [
|
|
{
|
|
"id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"chunk_index": 1,
|
|
"locator": "chunk-1",
|
|
"embedding_provider": "local-hash",
|
|
"embedding_model": "local-hash-v1",
|
|
"embedding_dimension": 64,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-import-batches/kb-batch-1/documents":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"index_status": "indexed",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-import-batches/kb-batch-1/chunks":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"chunk_index": 1,
|
|
"locator": "chunk-1",
|
|
"embedding_provider": "local-hash",
|
|
"embedding_model": "local-hash-v1",
|
|
"embedding_dimension": 64,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-import-batches/kb-batch-1/reindex":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"documents_count": 1,
|
|
"chunks_count": 1,
|
|
"embedding_provider": "openai-compatible",
|
|
"embedding_model": "text-embedding-prod",
|
|
"embedding_dimension": 1536,
|
|
"document_ids": ["doc-1"],
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-documents" and request.method == "POST":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"index_status": "indexed",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-documents/doc-1/chunks":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"chunk_index": 1,
|
|
"locator": "chunk-1",
|
|
"embedding_provider": "local-hash",
|
|
"embedding_model": "local-hash-v1",
|
|
"embedding_dimension": 64,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-documents/search":
|
|
query = parse_qs(request.url.query.decode())
|
|
assert query["keyword"] == ["lease"]
|
|
assert query["limit"] == ["3"]
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"index_status": "indexed",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-documents/reindex":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["knowledge_base_id"] == "laws-cn"
|
|
assert payload["triggered_by"] == "admin"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"documents_count": 1,
|
|
"chunks_count": 1,
|
|
"embedding_provider": "openai-compatible",
|
|
"embedding_model": "text-embedding-prod",
|
|
"embedding_dimension": 1536,
|
|
"document_ids": ["doc-1"],
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-chunks/search":
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"chunk": {
|
|
"id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"chunk_index": 1,
|
|
"locator": "chunk-1",
|
|
"embedding_provider": "local-hash",
|
|
"embedding_model": "local-hash-v1",
|
|
"embedding_dimension": 64,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
"score": 0.82,
|
|
"keyword_score": 0.76,
|
|
"vector_score": 0.89,
|
|
"hybrid_score": 0.82,
|
|
"retrieval_strategy": "hybrid",
|
|
"matched_terms": ["lease"],
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/knowledge-chunks/search/explain":
|
|
query = parse_qs(request.url.query.decode())
|
|
assert query["keyword"] == ["lease"]
|
|
assert query["limit"] == ["3"]
|
|
assert query["candidate_limit"] == ["9"]
|
|
assert query["strategy"] == ["hybrid"]
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"keyword": "lease",
|
|
"normalized_terms": ["lease"],
|
|
"retrieval_strategy": "hybrid",
|
|
"knowledge_base_ids": [],
|
|
"limit": 3,
|
|
"candidate_limit": 9,
|
|
"candidate_count": 1,
|
|
"included_count": 1,
|
|
"retrieval_backend": "json-explain",
|
|
"embedding_provider": "local-hash",
|
|
"embedding_model": "local-hash-v1",
|
|
"query_embedding_dimension": 64,
|
|
"keyword_threshold": 0.0,
|
|
"vector_threshold": 0.35,
|
|
"hybrid_keyword_weight": 0.55,
|
|
"hybrid_vector_weight": 0.45,
|
|
"results": [
|
|
{
|
|
"chunk": {
|
|
"id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"import_batch_id": "kb-batch-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"content": "A lease contract defines rent and use of leased property.",
|
|
"chunk_index": 1,
|
|
"locator": "chunk-1",
|
|
"embedding_provider": "local-hash",
|
|
"embedding_model": "local-hash-v1",
|
|
"embedding_dimension": 64,
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
"score": 0.82,
|
|
"keyword_score": 0.76,
|
|
"vector_score": 0.89,
|
|
"hybrid_score": 0.82,
|
|
"retrieval_strategy": "hybrid",
|
|
"matched_terms": ["lease"],
|
|
"included": True,
|
|
"rank": 1,
|
|
"filter_reason": None,
|
|
}
|
|
],
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/audit/ai-runs":
|
|
assert request.url.params.get("archive_status") == "archived"
|
|
return httpx.Response(
|
|
200,
|
|
json=[
|
|
{
|
|
"id": "run-1",
|
|
"operation": "legal_qa",
|
|
"provider_config_id": "provider-1",
|
|
"prompt_template_id": "prompt-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"provider_status": "succeeded",
|
|
"provider_error": None,
|
|
"prompt_preview": "Search evidence first, then answer. Question: How should lease deposit refund be handled?",
|
|
"question": "How should lease deposit refund be handled?",
|
|
"answer": "The contract should clearly state deposit refund conditions.",
|
|
"citations_count": 1,
|
|
"retrieval_query": "How should lease deposit refund be handled?",
|
|
"retrieval_strategy": "hybrid",
|
|
"retrieved_chunk_ids": ["chunk-1"],
|
|
"retrieved_sources": [
|
|
{
|
|
"chunk_id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"locator": "chunk-1",
|
|
"score": 0.82,
|
|
"keyword_score": 0.76,
|
|
"vector_score": 0.89,
|
|
"hybrid_score": 0.82,
|
|
"retrieval_strategy": "hybrid",
|
|
"matched_terms": ["lease"],
|
|
}
|
|
],
|
|
"prompt_tokens": 12,
|
|
"completion_tokens": 8,
|
|
"total_tokens": 20,
|
|
"estimated_cost": 0.00028,
|
|
"latency_ms": 12,
|
|
"quality_status": "passed",
|
|
"quality_score": 100,
|
|
"quality_labels": [
|
|
{
|
|
"code": "citation_present",
|
|
"severity": "info",
|
|
"title": "Citation present",
|
|
"detail": "The answer returned 1 citation.",
|
|
"review_status": "pending",
|
|
"review_note": None,
|
|
"reviewed_by": None,
|
|
"reviewed_at": None,
|
|
}
|
|
],
|
|
"quality_summary": "passed: 0 error(s), 0 warning(s), score 100.",
|
|
"redaction_status": "redacted",
|
|
"redaction_policy": "default",
|
|
"retention_until": "2026-06-30T00:00:00Z",
|
|
"archive_status": "archived",
|
|
"archived_at": "2026-07-01T00:00:00Z",
|
|
"status": "succeeded",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
)
|
|
if request.url.path == "/api/v1/audit/ai-runs/run-1":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "run-1",
|
|
"operation": "legal_qa",
|
|
"provider_config_id": "provider-1",
|
|
"prompt_template_id": "prompt-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"provider_status": "succeeded",
|
|
"provider_error": None,
|
|
"prompt_preview": "Search evidence first, then answer. Question: How should lease deposit refund be handled?",
|
|
"question": "How should lease deposit refund be handled?",
|
|
"answer": "The contract should clearly state deposit refund conditions.",
|
|
"citations_count": 1,
|
|
"retrieval_query": "How should lease deposit refund be handled?",
|
|
"retrieval_strategy": "hybrid",
|
|
"retrieved_chunk_ids": ["chunk-1"],
|
|
"retrieved_sources": [
|
|
{
|
|
"chunk_id": "chunk-1",
|
|
"document_id": "doc-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"title": "Civil Code",
|
|
"source_type": "law",
|
|
"reference": "Article 703",
|
|
"locator": "chunk-1",
|
|
"score": 0.82,
|
|
"keyword_score": 0.76,
|
|
"vector_score": 0.89,
|
|
"hybrid_score": 0.82,
|
|
"retrieval_strategy": "hybrid",
|
|
"matched_terms": ["lease"],
|
|
}
|
|
],
|
|
"prompt_tokens": 12,
|
|
"completion_tokens": 8,
|
|
"total_tokens": 20,
|
|
"estimated_cost": 0.00028,
|
|
"latency_ms": 12,
|
|
"quality_status": "passed",
|
|
"quality_score": 100,
|
|
"quality_labels": [
|
|
{
|
|
"code": "citation_present",
|
|
"severity": "info",
|
|
"title": "Citation present",
|
|
"detail": "The answer returned 1 citation.",
|
|
"review_status": "pending",
|
|
"review_note": None,
|
|
"reviewed_by": None,
|
|
"reviewed_at": None,
|
|
}
|
|
],
|
|
"quality_summary": "passed: 0 error(s), 0 warning(s), score 100.",
|
|
"status": "succeeded",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/audit/quality-dashboard":
|
|
query = parse_qs(request.url.query.decode())
|
|
assert query["days"] == ["7"]
|
|
assert query["pending_limit"] == ["5"]
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"summary": {
|
|
"days": 7,
|
|
"total_runs": 1,
|
|
"passed_runs": 1,
|
|
"needs_review_runs": 0,
|
|
"failed_quality_runs": 0,
|
|
"fallback_runs": 0,
|
|
"source_issue_runs": 0,
|
|
"token_issue_runs": 0,
|
|
"latency_issue_runs": 0,
|
|
"pending_labels": 1,
|
|
"pass_rate": 1.0,
|
|
"fallback_rate": 0.0,
|
|
"source_issue_rate": 0.0,
|
|
"latency_issue_rate": 0.0,
|
|
},
|
|
"daily_metrics": [
|
|
{
|
|
"date": "2026-06-22",
|
|
"total_runs": 1,
|
|
"passed_runs": 1,
|
|
"needs_review_runs": 0,
|
|
"failed_quality_runs": 0,
|
|
"fallback_runs": 0,
|
|
"source_issue_runs": 0,
|
|
"token_issue_runs": 0,
|
|
"latency_issue_runs": 0,
|
|
"pending_labels": 1,
|
|
"pass_rate": 1.0,
|
|
"fallback_rate": 0.0,
|
|
"source_issue_rate": 0.0,
|
|
"latency_issue_rate": 0.0,
|
|
}
|
|
],
|
|
"pending_labels": [
|
|
{
|
|
"run_id": "run-1",
|
|
"label_code": "citation_present",
|
|
"severity": "info",
|
|
"title": "Citation present",
|
|
"detail": "The answer returned 1 citation.",
|
|
"review_status": "pending",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
"question": "How should lease deposit refund be handled?",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"quality_status": "passed",
|
|
"quality_score": 100,
|
|
}
|
|
],
|
|
"attribution": {
|
|
"by_model": [
|
|
{
|
|
"dimension": "model",
|
|
"key": "deepseek / deepseek-chat",
|
|
"label": "deepseek / deepseek-chat",
|
|
"total_runs": 1,
|
|
"affected_runs": 1,
|
|
"warning_labels": 1,
|
|
"error_labels": 0,
|
|
"pending_labels": 1,
|
|
"fallback_runs": 0,
|
|
"average_quality_score": 90.0,
|
|
"top_label_codes": ["source_limited"],
|
|
"root_cause_hint": "该模型相关质量异常较集中，优先对比其他模型的回答质量和成本。",
|
|
}
|
|
],
|
|
"by_prompt_template": [],
|
|
"by_knowledge_base": [],
|
|
"by_label_code": [
|
|
{
|
|
"dimension": "label_code",
|
|
"key": "source_limited",
|
|
"label": "Source limited",
|
|
"total_runs": 1,
|
|
"affected_runs": 1,
|
|
"warning_labels": 1,
|
|
"error_labels": 0,
|
|
"pending_labels": 1,
|
|
"fallback_runs": 0,
|
|
"average_quality_score": 90.0,
|
|
"top_label_codes": ["source_limited"],
|
|
"root_cause_hint": "来源数量偏少，优先补充文档或放宽检索召回策略。",
|
|
}
|
|
],
|
|
},
|
|
"improvement_actions": [quality_action_payload()],
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/audit/quality-improvement-actions" and request.method == "POST":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["dimension"] == "model"
|
|
assert payload["action_type"] == "tune_model"
|
|
assert payload["source_label_codes"] == ["source_limited"]
|
|
assert payload["linked_prompt_template_id"] == "prompt-1"
|
|
return httpx.Response(200, json=quality_action_payload())
|
|
if request.url.path == "/api/v1/audit/quality-improvement-actions":
|
|
query = parse_qs(request.url.query.decode())
|
|
assert query["status"] == ["open"]
|
|
assert query["owner"] == ["admin"]
|
|
assert query["priority"] == ["high"]
|
|
assert query["action_type"] == ["tune_model"]
|
|
assert query["dimension"] == ["model"]
|
|
assert query["limit"] == ["5"]
|
|
return httpx.Response(200, json=[quality_action_payload()])
|
|
if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1" and request.method == "GET":
|
|
return httpx.Response(200, json=quality_action_payload())
|
|
if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1/executions":
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"action_id": "qa-action-1",
|
|
"model_config_changes": [
|
|
{
|
|
"id": "provider-change-1",
|
|
"provider_config_id": "provider-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"display_name": "DeepSeek Chat",
|
|
"before_config": None,
|
|
"after_config": {
|
|
"id": "provider-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"display_name": "DeepSeek Chat",
|
|
"base_url": None,
|
|
"temperature": 0.1,
|
|
"max_tokens": 4096,
|
|
"enabled": True,
|
|
"is_default": True,
|
|
"updated_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
"changed_by": "admin",
|
|
"change_note": "Use production legal QA model.",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
"prompt_template_changes": [
|
|
{
|
|
"id": "prompt-change-1",
|
|
"prompt_template_id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"operation": "publish",
|
|
"before_template": None,
|
|
"after_template": {
|
|
"id": "prompt-1",
|
|
"name": "legal_qa.default",
|
|
"version": "v2",
|
|
"content": "Search evidence first, then answer.",
|
|
"enabled": True,
|
|
"status": "published",
|
|
"rollback_from_version": None,
|
|
"published_at": "2026-06-22T00:01:00Z",
|
|
"updated_at": "2026-06-22T00:01:00Z",
|
|
},
|
|
"changed_by": "admin",
|
|
"change_note": "Publish prompt for quality action.",
|
|
"created_at": "2026-06-22T00:01:00Z",
|
|
}
|
|
],
|
|
"knowledge_ingestion_jobs": [
|
|
{
|
|
"id": "ingest-upload-1",
|
|
"knowledge_base_id": "laws-cn",
|
|
"source_name": "lease.md",
|
|
"source_type": "file_upload",
|
|
"status": "succeeded",
|
|
"parse_status": "parsed",
|
|
"ocr_status": "not_required",
|
|
"import_batch_id": "kb-batch-1",
|
|
"document_id": "doc-1",
|
|
"documents_count": 1,
|
|
"chunks_count": 1,
|
|
"retry_count": 0,
|
|
"max_retries": 3,
|
|
"error_message": None,
|
|
"worker_name": "api-sync-ingest",
|
|
"idempotency_key": None,
|
|
"created_at": "2026-06-22T00:02:00Z",
|
|
"updated_at": "2026-06-22T00:02:00Z",
|
|
"started_at": "2026-06-22T00:02:00Z",
|
|
"completed_at": "2026-06-22T00:02:00Z",
|
|
}
|
|
],
|
|
"knowledge_reindex_executions": [
|
|
{
|
|
"id": "reindex-1",
|
|
"document_id": None,
|
|
"knowledge_base_id": "laws-cn",
|
|
"import_batch_id": "kb-batch-1",
|
|
"documents_count": 1,
|
|
"chunks_count": 1,
|
|
"embedding_provider": "openai-compatible",
|
|
"embedding_model": "text-embedding-prod",
|
|
"embedding_dimension": 1536,
|
|
"document_ids": ["doc-1"],
|
|
"status": "succeeded",
|
|
"triggered_by": "admin",
|
|
"execution_note": "Rebuild index for quality action.",
|
|
"created_at": "2026-06-22T00:03:00Z",
|
|
}
|
|
],
|
|
"related_trace_runs": [
|
|
{
|
|
"id": "run-1",
|
|
"operation": "legal_qa",
|
|
"provider_config_id": "provider-1",
|
|
"prompt_template_id": "prompt-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"provider_status": "succeeded",
|
|
"provider_error": None,
|
|
"prompt_preview": "Search evidence first, then answer. Question: How should lease deposit refund be handled?",
|
|
"question": "How should lease deposit refund be handled?",
|
|
"answer": "The contract should clearly state deposit refund conditions.",
|
|
"citations_count": 1,
|
|
"retrieval_query": "How should lease deposit refund be handled?",
|
|
"retrieval_strategy": "hybrid",
|
|
"retrieved_chunk_ids": ["chunk-1"],
|
|
"retrieved_sources": [],
|
|
"prompt_tokens": 12,
|
|
"completion_tokens": 8,
|
|
"total_tokens": 20,
|
|
"estimated_cost": 0.00028,
|
|
"latency_ms": 12,
|
|
"quality_status": "passed",
|
|
"quality_score": 100,
|
|
"quality_labels": [],
|
|
"quality_summary": "passed: 0 error(s), 0 warning(s), score 100.",
|
|
"status": "succeeded",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
}
|
|
],
|
|
"impacted_trace_runs": [
|
|
{
|
|
"run_id": "run-2",
|
|
"operation": "legal_qa",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"quality_status": "passed",
|
|
"quality_score": 96,
|
|
"matched_label_codes": [],
|
|
"issue_label_codes": [],
|
|
"pending_label_codes": [],
|
|
"created_at": "2026-06-22T00:06:00Z",
|
|
}
|
|
],
|
|
"effect_breakdown": [
|
|
{
|
|
"label_code": "missing_citation",
|
|
"current_total_runs": 3,
|
|
"current_affected_runs": 1,
|
|
"current_pending_labels": 1,
|
|
"current_affected_rate": 0.3333,
|
|
"post_change_total_runs": 1,
|
|
"post_change_affected_runs": 0,
|
|
"post_change_pending_labels": 0,
|
|
"post_change_affected_rate": 0,
|
|
"affected_rate_delta": -0.3333,
|
|
"pending_labels_delta": -1,
|
|
}
|
|
],
|
|
},
|
|
)
|
|
if request.url.path == "/api/v1/audit/quality-improvement-actions/qa-action-1" and request.method == "PATCH":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert payload["status"] == "resolved"
|
|
assert payload["resolution_note"] == "Provider config was tuned."
|
|
assert payload["linked_import_batch_id"] == "kb-batch-2"
|
|
return httpx.Response(
|
|
200,
|
|
json=quality_action_payload(
|
|
status="resolved",
|
|
resolution_note="Provider config was tuned.",
|
|
resolved_at="2026-06-22T00:05:00Z",
|
|
),
|
|
)
|
|
if request.url.path == "/api/v1/audit/ai-runs/run-1/quality-labels/citation_present":
|
|
payload = json.loads(request.read().decode("utf-8"))
|
|
assert request.method == "PATCH"
|
|
assert payload["review_status"] == "resolved"
|
|
return httpx.Response(
|
|
200,
|
|
json={
|
|
"id": "run-1",
|
|
"operation": "legal_qa",
|
|
"provider_config_id": "provider-1",
|
|
"prompt_template_id": "prompt-1",
|
|
"provider": "deepseek",
|
|
"model": "deepseek-chat",
|
|
"provider_status": "succeeded",
|
|
"provider_error": None,
|
|
"prompt_preview": "Search evidence first, then answer. Question: How should lease deposit refund be handled?",
|
|
"question": "How should lease deposit refund be handled?",
|
|
"answer": "The contract should clearly state deposit refund conditions.",
|
|
"citations_count": 1,
|
|
"retrieval_query": "How should lease deposit refund be handled?",
|
|
"retrieval_strategy": "hybrid",
|
|
"retrieved_chunk_ids": ["chunk-1"],
|
|
"retrieved_sources": [],
|
|
"prompt_tokens": 12,
|
|
"completion_tokens": 8,
|
|
"total_tokens": 20,
|
|
"estimated_cost": 0.00028,
|
|
"latency_ms": 12,
|
|
"quality_status": "passed",
|
|
"quality_score": 100,
|
|
"quality_labels": [
|
|
{
|
|
"code": "citation_present",
|
|
"severity": "info",
|
|
"title": "Citation present",
|
|
"detail": "The answer returned 1 citation.",
|
|
"review_status": "resolved",
|
|
"review_note": "已确认依据有效。",
|
|
"reviewed_by": "admin",
|
|
"reviewed_at": "2026-06-22T00:01:00Z",
|
|
}
|
|
],
|
|
"quality_summary": "passed: 0 error(s), 0 warning(s), score 100.",
|
|
"status": "succeeded",
|
|
"created_at": "2026-06-22T00:00:00Z",
|
|
},
|
|
)
|
|
return httpx.Response(404)
|
|
|
|
client = AiPlatformClient("http://ai-platform.test", transport=httpx.MockTransport(handler))
|
|
|
|
assert client.upsert_model_config(
|
|
ProviderConfigUpsertRequest(
|
|
provider="deepseek",
|
|
model="deepseek-chat",
|
|
display_name="DeepSeek Chat",
|
|
temperature=0.1,
|
|
is_default=True,
|
|
changed_by="admin",
|
|
change_note="Use production legal QA model.",
|
|
)
|
|
).model == "deepseek-chat"
|
|
assert client.list_model_configs()[0].provider == "deepseek"
|
|
model_changes = client.list_model_config_changes(provider_config_id="provider-1")
|
|
assert model_changes[0].after_config.model == "deepseek-chat"
|
|
assert model_changes[0].change_note == "Use production legal QA model."
|
|
assert client.upsert_prompt_template(
|
|
PromptTemplateCreate(name="legal_qa.default", version="v2", content="Search evidence first.")
|
|
).version == "v2"
|
|
assert client.list_prompt_templates()[0].name == "legal_qa.default"
|
|
assert client.publish_prompt_template(
|
|
"legal_qa.default",
|
|
PromptTemplatePublishRequest(
|
|
version="v2",
|
|
changed_by="admin",
|
|
change_note="Publish prompt for quality action.",
|
|
),
|
|
).status == "published"
|
|
prompt_changes = client.list_prompt_template_changes(prompt_template_id="prompt-1")
|
|
assert prompt_changes[0].id == "prompt-change-1"
|
|
assert prompt_changes[0].after_template.status == "published"
|
|
rolled_back_prompt = client.rollback_prompt_template(
|
|
"legal_qa.default",
|
|
PromptTemplateRollbackRequest(from_version="v1", target_version="v1-rollback"),
|
|
)
|
|
assert rolled_back_prompt.version == "v1-rollback"
|
|
assert rolled_back_prompt.rollback_from_version == "v1"
|
|
assert client.get_quality_rule_config().min_source_count == 2
|
|
assert client.upsert_quality_rule_config(
|
|
AiQualityRuleConfigUpdate(
|
|
min_citation_count=1,
|
|
min_source_count=1,
|
|
high_token_threshold=9000,
|
|
high_latency_ms=12000,
|
|
critical_latency_ms=24000,
|
|
warning_penalty=7,
|
|
error_penalty=21,
|
|
)
|
|
).warning_penalty == 7
|
|
runtime_health = client.get_worker_runtime_health()
|
|
assert runtime_health.status == "ok"
|
|
assert runtime_health.queue_counts["queued"] == 1
|
|
assert runtime_health.workers[0].worker_name == "knowledge-worker-1"
|
|
manual_run = client.run_worker_once(
|
|
WorkerRunOnceRequest(limit=1, worker_name="manual-sdk-worker", triggered_by="admin")
|
|
)
|
|
assert manual_run.id == "worker-run-1"
|
|
assert manual_run.processed == 1
|
|
assert manual_run.succeeded == 1
|
|
assert client.list_worker_run_executions(limit=5)[0].worker_name == "manual-sdk-worker"
|
|
assert client.create_knowledge_import_batch(
|
|
KnowledgeImportBatchCreate(
|
|
knowledge_base_id="laws-cn",
|
|
source_name="civil-code-lease",
|
|
source_type="law_import",
|
|
)
|
|
).id == "kb-batch-1"
|
|
upload_result = client.upload_knowledge_document(
|
|
knowledge_base_id="laws-cn",
|
|
filename="lease.md",
|
|
content=b"Deposit refund rules must be written clearly.",
|
|
title="Lease Upload",
|
|
reference="lease.md",
|
|
context=RequestContext(trace_id="trace-upload"),
|
|
)
|
|
assert upload_result.id == "ingest-upload-1"
|
|
assert upload_result.status == "queued"
|
|
assert upload_result.parse_status == "pending"
|
|
assert upload_result.title == "Lease Upload"
|
|
assert upload_result.storage_object_key == "knowledge-ingestion/test.md"
|
|
ingestion_jobs = client.list_knowledge_ingestion_jobs(knowledge_base_id="laws-cn", status="queued")
|
|
assert ingestion_jobs[0].id == "ingest-upload-1"
|
|
assert ingestion_jobs[0].storage_bucket == "ai-platform-uploads"
|
|
assert client.retry_knowledge_ingestion_job("ingest-upload-1", triggered_by="admin").retry_count == 1
|
|
assert client.list_knowledge_import_batches(knowledge_base_id="laws-cn")[0].chunks_count == 1
|
|
batch_detail = client.get_knowledge_import_batch_detail("kb-batch-1")
|
|
assert batch_detail.batch.id == "kb-batch-1"
|
|
assert batch_detail.documents[0].id == "doc-1"
|
|
assert batch_detail.chunks[0].id == "chunk-1"
|
|
assert client.list_knowledge_import_batch_documents("kb-batch-1")[0].title == "Civil Code"
|
|
assert client.list_knowledge_import_batch_chunks("kb-batch-1")[0].locator == "chunk-1"
|
|
assert client.add_knowledge_document(
|
|
KnowledgeDocumentCreate(
|
|
import_batch_id="kb-batch-1",
|
|
knowledge_base_id="laws-cn",
|
|
title="Civil Code",
|
|
source_type="law",
|
|
reference="Article 703",
|
|
content="A lease contract defines rent and use of leased property.",
|
|
)
|
|
).reference == "Article 703"
|
|
assert client.list_knowledge_chunks("doc-1")[0].locator == "chunk-1"
|
|
assert client.list_knowledge_chunks("doc-1")[0].embedding_provider == "local-hash"
|
|
assert client.list_knowledge_chunks("doc-1")[0].embedding_model == "local-hash-v1"
|
|
assert client.search_knowledge_documents("lease", limit=3)[0].title == "Civil Code"
|
|
reindex_result = client.reindex_knowledge_documents(
|
|
KnowledgeReindexRequest(
|
|
knowledge_base_id="laws-cn",
|
|
triggered_by="admin",
|
|
execution_note="Rebuild index for quality action.",
|
|
)
|
|
)
|
|
assert reindex_result.embedding_provider == "openai-compatible"
|
|
assert reindex_result.embedding_dimension == 1536
|
|
assert reindex_result.document_ids == ["doc-1"]
|
|
batch_reindex_result = client.reindex_knowledge_import_batch("kb-batch-1")
|
|
assert batch_reindex_result.document_ids == ["doc-1"]
|
|
chunk_result = client.search_knowledge_chunks("lease", limit=3)
|
|
assert chunk_result[0].score == 0.82
|
|
assert chunk_result[0].hybrid_score == 0.82
|
|
explain = client.explain_knowledge_search("lease", limit=3, candidate_limit=9)
|
|
assert explain.embedding_provider == "local-hash"
|
|
assert explain.results[0].included is True
|
|
assert explain.results[0].rank == 1
|
|
audit = client.list_ai_run_audits(limit=5, archive_status="archived")[0]
|
|
assert audit.operation == "legal_qa"
|
|
assert audit.provider == "deepseek"
|
|
assert audit.model == "deepseek-chat"
|
|
assert audit.provider_status == "succeeded"
|
|
assert audit.provider_error is None
|
|
assert audit.prompt_preview is not None
|
|
assert "lease deposit" in audit.prompt_preview
|
|
assert audit.prompt_tokens == 12
|
|
assert audit.completion_tokens == 8
|
|
assert audit.total_tokens == 20
|
|
assert audit.estimated_cost == 0.00028
|
|
assert audit.retrieved_chunk_ids == ["chunk-1"]
|
|
assert audit.retrieved_sources[0].reference == "Article 703"
|
|
assert audit.quality_status == "passed"
|
|
assert audit.quality_score == 100
|
|
assert audit.quality_labels[0].code == "citation_present"
|
|
assert audit.quality_labels[0].review_status == "pending"
|
|
assert audit.quality_summary == "passed: 0 error(s), 0 warning(s), score 100."
|
|
assert audit.redaction_status == "redacted"
|
|
assert audit.redaction_policy == "default"
|
|
assert audit.archive_status == "archived"
|
|
assert audit.archived_at == "2026-07-01T00:00:00Z"
|
|
audit_detail = client.get_ai_run_audit("run-1")
|
|
assert audit_detail.id == "run-1"
|
|
assert audit_detail.retrieved_sources[0].hybrid_score == 0.82
|
|
assert audit_detail.quality_labels[0].code == "citation_present"
|
|
dashboard = client.get_quality_dashboard(days=7, pending_limit=5)
|
|
assert dashboard.summary.pass_rate == 1.0
|
|
assert dashboard.daily_metrics[0].date == "2026-06-22"
|
|
assert dashboard.pending_labels[0].run_id == "run-1"
|
|
assert dashboard.attribution.by_model[0].key == "deepseek / deepseek-chat"
|
|
assert dashboard.attribution.by_label_code[0].top_label_codes == ["source_limited"]
|
|
assert dashboard.attribution.by_label_code[0].root_cause_hint is not None
|
|
assert dashboard.improvement_actions[0].id == "qa-action-1"
|
|
created_action = client.create_quality_improvement_action(
|
|
AiQualityImprovementActionCreate(
|
|
dimension="model",
|
|
key="deepseek / deepseek-chat",
|
|
label="deepseek / deepseek-chat",
|
|
action_type="tune_model",
|
|
description="Check provider timeout and model configuration.",
|
|
owner="admin",
|
|
priority="high",
|
|
source_label_codes=["source_limited"],
|
|
source_run_ids=["run-1"],
|
|
linked_prompt_template_id="prompt-1",
|
|
evaluation_days=7,
|
|
)
|
|
)
|
|
assert created_action.status == "open"
|
|
assert created_action.linked_prompt_template_id == "prompt-1"
|
|
assert created_action.linked_import_batch_id == "kb-batch-1"
|
|
assert created_action.baseline_affected_runs == 1
|
|
assert created_action.effect_status == "unchanged"
|
|
assert created_action.post_change_total_runs == 2
|
|
assert created_action.related_change_actions_count == 2
|
|
listed_actions = client.list_quality_improvement_actions(
|
|
status="open",
|
|
owner="admin",
|
|
priority="high",
|
|
action_type="tune_model",
|
|
dimension="model",
|
|
limit=5,
|
|
)
|
|
assert listed_actions[0].id == "qa-action-1"
|
|
assert listed_actions[0].effect_summary is not None
|
|
action_detail = client.get_quality_improvement_action("qa-action-1")
|
|
assert action_detail.id == "qa-action-1"
|
|
assert action_detail.source_run_ids == ["run-1"]
|
|
assert action_detail.effect_summary is not None
|
|
action_executions = client.get_quality_improvement_action_executions("qa-action-1")
|
|
assert action_executions.model_config_changes[0].after_config.model == "deepseek-chat"
|
|
assert action_executions.prompt_template_changes[0].version == "v2"
|
|
assert action_executions.knowledge_ingestion_jobs[0].id == "ingest-upload-1"
|
|
assert action_executions.knowledge_reindex_executions[0].import_batch_id == "kb-batch-1"
|
|
assert action_executions.related_trace_runs[0].id == "run-1"
|
|
assert action_executions.impacted_trace_runs[0].run_id == "run-2"
|
|
assert action_executions.effect_breakdown[0].label_code == "missing_citation"
|
|
assert action_executions.effect_breakdown[0].affected_rate_delta < 0
|
|
updated_action = client.update_quality_improvement_action(
|
|
"qa-action-1",
|
|
AiQualityImprovementActionUpdate(
|
|
status="resolved",
|
|
resolution_note="Provider config was tuned.",
|
|
linked_import_batch_id="kb-batch-2",
|
|
),
|
|
)
|
|
assert updated_action.status == "resolved"
|
|
assert updated_action.resolved_at == "2026-06-22T00:05:00Z"
|
|
assert updated_action.post_change_total_runs == 2
|
|
assert updated_action.effect_summary and "post-change" in updated_action.effect_summary
|
|
updated_audit = client.update_ai_run_quality_label(
|
|
"run-1",
|
|
"citation_present",
|
|
AiQualityLabelReviewUpdate(
|
|
review_status="resolved",
|
|
review_note="已确认依据有效。",
|
|
reviewed_by="admin",
|
|
),
|
|
)
|
|
assert updated_audit.quality_labels[0].review_status == "resolved"
|
|
assert updated_audit.quality_labels[0].reviewed_at == "2026-06-22T00:01:00Z"
|
|
assert "/api/v1/audit/ai-runs" in seen_paths
|
|
assert "/api/v1/audit/ai-runs/run-1" in seen_paths
|
|
assert "/api/v1/audit/quality-dashboard" in seen_paths
|
|
assert "/api/v1/audit/quality-improvement-actions" in seen_paths
|
|
assert "/api/v1/audit/quality-improvement-actions/qa-action-1" in seen_paths
|
|
assert "/api/v1/audit/quality-improvement-actions/qa-action-1/executions" in seen_paths
|
|
assert "/api/v1/quality-rules/default" in seen_paths
|