feat: extend ocr scanned sample coverage
This commit is contained in:
@@ -130,6 +130,14 @@ REPORT_PDF_MARKERS = (
|
||||
"风险项:",
|
||||
"原始文本摘要:",
|
||||
)
|
||||
OCR_SIGNATURE_NOISE_PATTERN = re.compile(
|
||||
r"^((甲方|乙方|双方|法定代表人|授权代表)((?(签字|签章|盖章))?)|(签字|签章|盖章|签署页|签字页|盖章页|签署栏|本页无正文)|(party\s*a(\s+signature)?)|(party\s*b(\s+signature)?)|(signature|signed))$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
OCR_SIGNATURE_DATE_PATTERN = re.compile(
|
||||
r"^((日期|签署日期|签约日期|date)[::]?\s*[\d\s\-./年月日]+)$",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
OCR_GATE_PASS = "PASS"
|
||||
OCR_GATE_WARN = "WARN"
|
||||
OCR_GATE_BLOCK = "BLOCK"
|
||||
@@ -980,6 +988,10 @@ class ReviewService:
|
||||
continue
|
||||
if PDF_PAGE_NOISE_PATTERN.fullmatch(line):
|
||||
continue
|
||||
if len(line) <= 24 and OCR_SIGNATURE_NOISE_PATTERN.fullmatch(line):
|
||||
continue
|
||||
if len(line) <= 28 and OCR_SIGNATURE_DATE_PATTERN.fullmatch(line):
|
||||
continue
|
||||
if PDF_NOISE_LINE_PATTERN.search(line):
|
||||
continue
|
||||
if line.lower().count(" obj") or line.lower().count("endobj"):
|
||||
|
||||
@@ -171,6 +171,101 @@ def make_cn_image_contract() -> bytes:
|
||||
return pdf_buffer.getvalue()
|
||||
|
||||
|
||||
def make_cn_multi_page_contract() -> bytes:
|
||||
font = _load_cjk_font(30)
|
||||
pages = [
|
||||
[
|
||||
"采购合同扫描样本(第1页)",
|
||||
"第一条 付款节点:甲方应在验收合格并收到发票后十五个工作日内付款。",
|
||||
"第二条 验收标准:系统核心流程可用且严重缺陷关闭后视为验收完成。",
|
||||
],
|
||||
[
|
||||
"采购合同扫描样本(第2页)",
|
||||
"第三条 保密义务:乙方应对商业秘密和技术资料承担三年保密义务。",
|
||||
"第四条 违约责任:如乙方延期交付,双方应按约承担违约责任。",
|
||||
],
|
||||
]
|
||||
|
||||
pdf_buffer = io.BytesIO()
|
||||
pdf = canvas.Canvas(pdf_buffer)
|
||||
for lines in pages:
|
||||
image = Image.new("RGB", (980, 620), "white")
|
||||
draw = ImageDraw.Draw(image)
|
||||
y = 56
|
||||
for line in lines:
|
||||
draw.text((40, y), line, fill="black", font=font)
|
||||
y += 94
|
||||
pdf.drawImage(ImageReader(image), 30, 220, width=550, height=360)
|
||||
pdf.showPage()
|
||||
pdf.save()
|
||||
return pdf_buffer.getvalue()
|
||||
|
||||
|
||||
def make_table_scan_pdf() -> bytes:
|
||||
image = Image.new("RGB", (1100, 760), "white")
|
||||
draw = ImageDraw.Draw(image)
|
||||
font = _load_cjk_font(28)
|
||||
|
||||
draw.text((40, 30), "表格型采购合同扫描样本", fill="black", font=font)
|
||||
left = 40
|
||||
top = 100
|
||||
col_widths = [220, 760]
|
||||
row_height = 90
|
||||
rows = [
|
||||
("条款项", "内容"),
|
||||
("付款节点", "验收合格并收到发票后十五个工作日内付款"),
|
||||
("验收标准", "核心功能可用且严重缺陷关闭"),
|
||||
("保密义务", "商业秘密和技术资料保密三年"),
|
||||
("违约责任", "延期交付按约承担违约责任"),
|
||||
]
|
||||
|
||||
for row_index, row in enumerate(rows):
|
||||
current_top = top + row_index * row_height
|
||||
x = left
|
||||
for cell_index, cell_text in enumerate(row):
|
||||
width = col_widths[cell_index]
|
||||
draw.rectangle((x, current_top, x + width, current_top + row_height), outline="black", width=2)
|
||||
draw.text((x + 16, current_top + 28), cell_text, fill="black", font=font)
|
||||
x += width
|
||||
|
||||
pdf_buffer = io.BytesIO()
|
||||
pdf = canvas.Canvas(pdf_buffer)
|
||||
pdf.drawImage(ImageReader(image), 24, 180, width=560, height=390)
|
||||
pdf.save()
|
||||
return pdf_buffer.getvalue()
|
||||
|
||||
|
||||
def make_signature_noise_scan_pdf() -> bytes:
|
||||
font = _load_cjk_font(28)
|
||||
pages = [
|
||||
[
|
||||
"服务合同扫描样本",
|
||||
"甲方应在验收合格并收到发票后十五个工作日内付款。",
|
||||
"乙方应对商业秘密和技术资料承担保密义务。",
|
||||
],
|
||||
[
|
||||
"签署页",
|
||||
"甲方(盖章)",
|
||||
"乙方(签字)",
|
||||
"签署日期:2026年05月04日",
|
||||
],
|
||||
]
|
||||
|
||||
pdf_buffer = io.BytesIO()
|
||||
pdf = canvas.Canvas(pdf_buffer)
|
||||
for lines in pages:
|
||||
image = Image.new("RGB", (900, 620), "white")
|
||||
draw = ImageDraw.Draw(image)
|
||||
y = 70
|
||||
for line in lines:
|
||||
draw.text((50, y), line, fill="black", font=font)
|
||||
y += 100
|
||||
pdf.drawImage(ImageReader(image), 36, 230, width=520, height=340)
|
||||
pdf.showPage()
|
||||
pdf.save()
|
||||
return pdf_buffer.getvalue()
|
||||
|
||||
|
||||
def make_image_only_pdf() -> bytes:
|
||||
return _make_image_contract(low_res=False)
|
||||
|
||||
@@ -260,6 +355,41 @@ def build_cases(sample_contract_path: Optional[str]) -> List[PdfCase]:
|
||||
require_preview=["采购合同扫描样本", "甲方应在验收合格并收到发票后十五个工作日内付款。"],
|
||||
notes="OCR 第一批实现后,中文扫描合同应至少进入正式审查,并暴露 OCR 门禁结果。",
|
||||
),
|
||||
PdfCase(
|
||||
case_id="C-004",
|
||||
category="C",
|
||||
label="多页中文扫描合同 PDF(OCR第三批)",
|
||||
expectation="ocr_supported",
|
||||
file_name="gate-cn-multipage.pdf",
|
||||
build_bytes=make_cn_multi_page_contract,
|
||||
allow_ocr_statuses=["PASS", "WARN"],
|
||||
require_preview=["采购合同扫描样本(第1页)", "第一条 付款节点:甲方应在验收合格并收到发票后十五个工作日内付款。"],
|
||||
notes="OCR 第三批:多页中文扫描合同应能进入正式审查。",
|
||||
),
|
||||
PdfCase(
|
||||
case_id="C-005",
|
||||
category="C",
|
||||
label="表格型扫描 PDF(OCR第三批)",
|
||||
expectation="ocr_supported",
|
||||
file_name="gate-table-scan.pdf",
|
||||
build_bytes=make_table_scan_pdf,
|
||||
allow_ocr_statuses=["PASS", "WARN"],
|
||||
require_preview=["表格型采购合同扫描样本", "付款节点"],
|
||||
notes="OCR 第三批:表格型扫描 PDF 至少应进入正式审查,并暴露 OCR 门禁结果。",
|
||||
),
|
||||
PdfCase(
|
||||
case_id="C-006",
|
||||
category="C",
|
||||
label="盖章/签章干扰扫描 PDF(OCR第三批)",
|
||||
expectation="ocr_supported",
|
||||
file_name="gate-signature-scan.pdf",
|
||||
build_bytes=make_signature_noise_scan_pdf,
|
||||
allow_ocr_statuses=["PASS", "WARN"],
|
||||
require_preview=["服务合同扫描样本", "甲方应在验收合格并收到发票后十五个工作日内付款。"],
|
||||
forbid_preview=["甲方(盖章)", "乙方(签字)", "签署日期:2026年05月04日", "签署页"],
|
||||
require_clause_absent=["甲方(盖章)", "乙方(签字)", "签署日期:2026年05月04日", "签署页"],
|
||||
notes="OCR 第三批:签章页噪声不应污染正文摘要和条款列表。",
|
||||
),
|
||||
]
|
||||
|
||||
if sample_contract_path:
|
||||
|
||||
@@ -26,12 +26,51 @@ type ReviewFormValues = {
|
||||
|
||||
const reviewHighlights = ["支持 PDF / DOCX / TXT 上传", "可叠加补充文本说明", "提交后自动进入结果详情页"];
|
||||
|
||||
type SubmissionGuidance = {
|
||||
type: "info" | "warning" | "error";
|
||||
title: string;
|
||||
description: string;
|
||||
};
|
||||
|
||||
function buildSubmissionGuidance(message: string): SubmissionGuidance | null {
|
||||
if (message.includes("疑似平台审查报告 PDF")) {
|
||||
return {
|
||||
type: "warning",
|
||||
title: "当前上传的是平台审查报告,不是合同原文",
|
||||
description: "请改为上传合同原文 PDF / DOCX / TXT,或直接粘贴合同正文后再审查。",
|
||||
};
|
||||
}
|
||||
if (message.includes("OCR 识别质量不足")) {
|
||||
return {
|
||||
type: "warning",
|
||||
title: "扫描件识别质量不足",
|
||||
description: "建议上传更清晰的扫描件、可复制文本 PDF / DOCX / TXT,或直接粘贴合同正文。清晰扫描件已支持 OCR,但低质量扫描件仍会被门禁拦截。",
|
||||
};
|
||||
}
|
||||
if (message.includes("扫描件/图片型 PDF") || message.includes("OCR")) {
|
||||
return {
|
||||
type: "info",
|
||||
title: "检测到扫描件或图片型 PDF",
|
||||
description: "系统会先尝试 OCR;若识别质量不足,会建议你改传更清晰扫描件、可复制文本 PDF / DOCX / TXT,或直接粘贴合同正文。",
|
||||
};
|
||||
}
|
||||
if (message.includes("无法从上传文件中提取可审查文本")) {
|
||||
return {
|
||||
type: "warning",
|
||||
title: "当前文件未形成可审查正文",
|
||||
description: "请确认文件是可复制文本 PDF / DOCX / TXT;如果是扫描件,请提高扫描清晰度或直接粘贴合同正文。",
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function ReviewPage() {
|
||||
const router = useRouter();
|
||||
const [form] = Form.useForm<ReviewFormValues>();
|
||||
const [messageApi, contextHolder] = message.useMessage();
|
||||
const [fileList, setFileList] = useState<UploadFile[]>([]);
|
||||
const [submitting, setSubmitting] = useState(false);
|
||||
const [submissionGuidance, setSubmissionGuidance] = useState<SubmissionGuidance | null>(null);
|
||||
const submitLockRef = useRef(false);
|
||||
const selectedFile = fileList[0];
|
||||
const selectedFileName = useMemo(() => selectedFile?.name ?? "", [selectedFile]);
|
||||
@@ -53,19 +92,32 @@ export default function ReviewPage() {
|
||||
{ label: "目标输出", value: "风险总览 / 条款摘要 / 报告下载" },
|
||||
];
|
||||
|
||||
const handleSubmit = async (values: ReviewFormValues) => {
|
||||
const handleSubmitFailed = () => {
|
||||
setSubmitting(false);
|
||||
submitLockRef.current = false;
|
||||
};
|
||||
|
||||
const requestSubmit = () => {
|
||||
if (submitLockRef.current || submitting) {
|
||||
return;
|
||||
}
|
||||
submitLockRef.current = true;
|
||||
setSubmitting(true);
|
||||
void form.submit();
|
||||
};
|
||||
|
||||
const handleSubmit = async (values: ReviewFormValues) => {
|
||||
const text = values.text?.trim();
|
||||
const selectedOriginFile = selectedFile?.originFileObj;
|
||||
|
||||
if (!selectedOriginFile && !text) {
|
||||
setSubmissionGuidance(null);
|
||||
messageApi.error("请上传合同文件或填写合同文本");
|
||||
return;
|
||||
}
|
||||
|
||||
if (selectedOriginFile && !isSupportedReviewFile(selectedOriginFile.name)) {
|
||||
setSubmissionGuidance(null);
|
||||
messageApi.error("仅支持 PDF / DOCX / TXT 文件");
|
||||
return;
|
||||
}
|
||||
@@ -84,21 +136,21 @@ export default function ReviewPage() {
|
||||
if (selectedOriginFile) {
|
||||
formData.append("file", selectedOriginFile);
|
||||
}
|
||||
|
||||
submitLockRef.current = true;
|
||||
setSubmitting(true);
|
||||
|
||||
try {
|
||||
const result = await createReview(formData);
|
||||
|
||||
if (result.code !== 0) {
|
||||
setSubmissionGuidance(buildSubmissionGuidance(result.message || ""));
|
||||
messageApi.error(result.message || "提交审查失败");
|
||||
return;
|
||||
}
|
||||
|
||||
setSubmissionGuidance(null);
|
||||
messageApi.success(selectedOriginFile ? "审查任务已创建,文件内容已纳入审查" : "审查任务已创建");
|
||||
router.push(`/reviews/${result.data.id}`);
|
||||
} catch {
|
||||
} catch (error: unknown) {
|
||||
const message = error instanceof Error ? error.message : "提交失败,请稍后重试";
|
||||
setSubmissionGuidance(buildSubmissionGuidance(message));
|
||||
messageApi.error("提交失败,请稍后重试");
|
||||
} finally {
|
||||
setSubmitting(false);
|
||||
@@ -144,7 +196,7 @@ export default function ReviewPage() {
|
||||
</Card>
|
||||
</section>
|
||||
|
||||
<Form form={form} layout="vertical" initialValues={{ review_as: "party_a" }} onFinish={handleSubmit}>
|
||||
<Form form={form} layout="vertical" initialValues={{ review_as: "party_a" }} onFinish={handleSubmit} onFinishFailed={handleSubmitFailed}>
|
||||
<Row gutter={[16, 16]}>
|
||||
<Col xs={24} lg={14}>
|
||||
<Space direction="vertical" size="middle" style={{ width: "100%" }}>
|
||||
@@ -155,6 +207,14 @@ export default function ReviewPage() {
|
||||
type="info"
|
||||
message="上传文件后,系统会先抽取文件文本,再进入风险识别与条款摘要链路。"
|
||||
/>
|
||||
{submissionGuidance ? (
|
||||
<Alert
|
||||
showIcon
|
||||
type={submissionGuidance.type}
|
||||
message={submissionGuidance.title}
|
||||
description={submissionGuidance.description}
|
||||
/>
|
||||
) : null}
|
||||
<Dragger
|
||||
accept=".pdf,.docx,.txt"
|
||||
beforeUpload={(file) => {
|
||||
@@ -236,7 +296,7 @@ export default function ReviewPage() {
|
||||
message="提交后将自动进入详情页,可查看风险总览、条款摘要和报告下载。"
|
||||
/>
|
||||
|
||||
<Button block type="primary" htmlType="submit" loading={submitting} disabled={submitting}>
|
||||
<Button block type="primary" htmlType="button" loading={submitting} disabled={submitting} onClick={requestSubmit}>
|
||||
提交审查
|
||||
</Button>
|
||||
</Card>
|
||||
|
||||
@@ -100,6 +100,29 @@
|
||||
- 已新增:
|
||||
- `project-docs/中文扫描合同样本库清单-2026-05-04.csv`
|
||||
|
||||
### 第三批推进结果
|
||||
|
||||
1. **多页中文扫描合同**
|
||||
- 已接入 `C-004`
|
||||
- 目标:`PASS/WARN`
|
||||
|
||||
2. **表格型扫描 PDF**
|
||||
- 已接入 `C-005`
|
||||
- 目标:`PASS/WARN`
|
||||
|
||||
3. **盖章页 / 签章页干扰清洗**
|
||||
- 已在后端清洗逻辑中增加签章页噪声规则
|
||||
- 已接入 `C-006`
|
||||
- 目标:正文摘要不再混入签章页短噪声
|
||||
|
||||
4. **前端 OCR 状态展示**
|
||||
- 审查详情页已显示 `OCR Quality Gate`
|
||||
- 用户可直接看到 `PASS / WARN / BLOCK`
|
||||
|
||||
5. **低质量扫描件的人机交互提示**
|
||||
- 提交页已按错误信息细分提示
|
||||
- 扫描件 / 报告型 PDF / OCR 低质量都会返回更具体的用户提示
|
||||
|
||||
## 4. 未在第一批实现的内容
|
||||
|
||||
1. 扫描件中文复杂版式专项优化
|
||||
@@ -118,9 +141,9 @@
|
||||
|
||||
## 6. 下一批建议
|
||||
|
||||
第二批最自然的工作是:
|
||||
下一批最自然的工作是:
|
||||
|
||||
1. 中文扫描合同样本库落地
|
||||
2. OCR 后清洗进一步加强
|
||||
3. 低质量扫描件的边界提示优化
|
||||
4. 前端详情页展示 `OCR PASS/WARN/BLOCK`
|
||||
1. 用真实业务扫描样本替换当前动态生成样本
|
||||
2. 多页中文扫描合同与表格型扫描 PDF 持续调优到更稳定的 `PASS`
|
||||
3. 盖章页 / 签章页清洗继续细化
|
||||
4. 低质量扫描件的人机交互提示再丰富
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
sample_id,title,document_type,language,current_stage,current_expectation,gate_case,source_mode,notes
|
||||
C-003,中文扫描采购合同(清晰),scanned_pdf,zh-CN,ocr_first_batch,PASS/WARN,C-003,dynamic_generated,当前已接入 PDF 验收门禁;用于验证中文 OCR 进入正式审查链路
|
||||
C-004,中文扫描采购合同(低清晰度),scanned_pdf,zh-CN,planned_second_batch,WARN/BLOCK,,planned_fixture,下一批用于验证中文低质量扫描件的 OCR 质量门禁
|
||||
C-005,中文扫描服务合同(多页),scanned_pdf,zh-CN,planned_second_batch,PASS/WARN,,planned_fixture,下一批用于验证多页扫描件、跨页拼接与页眉页脚清洗
|
||||
C-006,中文扫描劳动合同(盖章页),scanned_pdf,zh-CN,planned_second_batch,PASS/WARN,,planned_fixture,下一批用于验证盖章、签章页与正文分离
|
||||
C-007,中文扫描框架协议(表格较多),scanned_pdf,zh-CN,planned_second_batch,WARN/BLOCK,,planned_fixture,下一批用于验证表格型扫描 PDF 的 OCR 与后清洗
|
||||
C-004,中文扫描采购合同(低清晰度),scanned_pdf,zh-CN,ocr_third_batch,WARN/BLOCK,C-002,dynamic_generated,OCR 第三批:低质量中文扫描件由 OCR 质量门禁控制
|
||||
C-005,中文扫描采购合同(多页),scanned_pdf,zh-CN,ocr_third_batch,PASS/WARN,C-004,dynamic_generated,OCR 第三批:多页中文扫描合同应进入正式审查
|
||||
C-006,中文扫描采购合同(表格型),scanned_pdf,zh-CN,ocr_third_batch,PASS/WARN,C-005,dynamic_generated,OCR 第三批:表格型扫描 PDF 至少应进入正式审查
|
||||
C-007,中文扫描服务合同(盖章/签章页干扰),scanned_pdf,zh-CN,ocr_third_batch,PASS/WARN,C-006,dynamic_generated,OCR 第三批:盖章/签章页噪声不应污染正文摘要
|
||||
|
||||
|
Reference in New Issue
Block a user