1192 lines
58 KiB
PowerShell
1192 lines
58 KiB
PowerShell
param(
|
|
[string]$RemoteHost = "192.168.3.90",
|
|
[string]$RemoteUser = "root",
|
|
[string]$BackendContainer = "ai-contract-backend-restore",
|
|
[string]$SharedAIContainer = "shared-ai-platform",
|
|
[string]$BackendBaseUrl = "http://192.168.3.90:8001",
|
|
[string]$FrontendBaseUrl = "http://192.168.3.90:3003",
|
|
[string]$SshKeyPath = "$HOME\.ssh\id_rsa",
|
|
[switch]$UseSshpass,
|
|
[string]$TenantId = "tenant-demo-001",
|
|
[int]$ChatWaitSeconds = 35,
|
|
[string]$ArtifactRoot = "",
|
|
[string]$BrowsePath = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Save-Json {
|
|
param([string]$Path, [object]$Value)
|
|
$Value | ConvertTo-Json -Depth 100 | Set-Content -Encoding UTF8 -Path $Path
|
|
}
|
|
|
|
function Read-JsonFile {
|
|
param([string]$Path)
|
|
return Get-Content -LiteralPath $Path -Raw -Encoding UTF8 | ConvertFrom-Json
|
|
}
|
|
|
|
function ConvertTo-ShellSingleQuote {
|
|
param([string]$Value)
|
|
return "'" + ($Value -replace "'", "'`"`"`'") + "'"
|
|
}
|
|
|
|
function Add-Check {
|
|
param(
|
|
[System.Collections.Generic.List[object]]$Checks,
|
|
[string]$Key,
|
|
[string]$Label,
|
|
[ValidateSet("pass", "fail", "warn", "skip", "blocked")]
|
|
[string]$Status,
|
|
[string]$Detail,
|
|
[bool]$Required = $true,
|
|
[hashtable]$Meta = @{}
|
|
)
|
|
|
|
$Checks.Add([ordered]@{
|
|
key = $Key
|
|
label = $Label
|
|
status = $Status
|
|
required = $Required
|
|
detail = $Detail
|
|
meta = $Meta
|
|
}) | Out-Null
|
|
}
|
|
|
|
function Resolve-SshpassPath {
|
|
$command = Get-Command sshpass.exe -ErrorAction SilentlyContinue
|
|
if ($null -ne $command) {
|
|
return $command.Source
|
|
}
|
|
|
|
$candidates = @(
|
|
(Join-Path $env:LOCALAPPDATA "Microsoft\WinGet\Packages\xhcoding.sshpass-win32_Microsoft.Winget.Source_8wekyb3d8bbwe\sshpass.exe")
|
|
)
|
|
foreach ($candidate in $candidates) {
|
|
if (Test-Path -LiteralPath $candidate -PathType Leaf) {
|
|
return (Resolve-Path -LiteralPath $candidate).Path
|
|
}
|
|
}
|
|
|
|
return $null
|
|
}
|
|
|
|
function Resolve-BrowsePath {
|
|
param([string]$RequestedPath)
|
|
if (-not [string]::IsNullOrWhiteSpace($RequestedPath) -and (Test-Path -LiteralPath $RequestedPath -PathType Leaf)) {
|
|
return (Resolve-Path -LiteralPath $RequestedPath).Path
|
|
}
|
|
|
|
$candidates = @(
|
|
(Join-Path $HOME ".claude\skills\gstack\browse\dist\browse.exe"),
|
|
(Join-Path $HOME ".Codex\skills\gstack\browse\dist\browse.exe"),
|
|
(Join-Path $HOME ".codex\skills\gstack\browse\dist\browse.exe")
|
|
)
|
|
foreach ($candidate in $candidates) {
|
|
if (Test-Path -LiteralPath $candidate -PathType Leaf) {
|
|
return (Resolve-Path -LiteralPath $candidate).Path
|
|
}
|
|
}
|
|
|
|
$command = Get-Command browse.exe -ErrorAction SilentlyContinue
|
|
if ($null -ne $command) {
|
|
return $command.Source
|
|
}
|
|
|
|
return $null
|
|
}
|
|
|
|
function Resolve-PythonPath {
|
|
$python = Get-Command python -ErrorAction SilentlyContinue
|
|
if ($null -ne $python) {
|
|
return $python.Source
|
|
}
|
|
$py = Get-Command py -ErrorAction SilentlyContinue
|
|
if ($null -ne $py) {
|
|
return $py.Source
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Invoke-LoggedCommand {
|
|
param(
|
|
[string]$Name,
|
|
[string]$FilePath,
|
|
[string[]]$Arguments,
|
|
[string]$LogDir,
|
|
[switch]$AllowFailure
|
|
)
|
|
|
|
$logPath = Join-Path $LogDir "$Name.log"
|
|
$previousErrorActionPreference = $ErrorActionPreference
|
|
$ErrorActionPreference = "Continue"
|
|
try {
|
|
$output = & $FilePath @Arguments 2>&1
|
|
$exitCode = if ($LASTEXITCODE -is [int]) { $LASTEXITCODE } else { 0 }
|
|
} finally {
|
|
$ErrorActionPreference = $previousErrorActionPreference
|
|
}
|
|
|
|
$outputText = ($output -join [Environment]::NewLine)
|
|
$outputText | Set-Content -Encoding UTF8 -Path $logPath
|
|
if ($exitCode -ne 0 -and -not $AllowFailure) {
|
|
throw "$Name failed with exit code $exitCode. See $logPath"
|
|
}
|
|
return [ordered]@{
|
|
exit_code = $exitCode
|
|
output = $outputText
|
|
log = $logPath
|
|
}
|
|
}
|
|
|
|
function Invoke-JsonPost {
|
|
param([string]$Url, [hashtable]$Body)
|
|
$jsonBody = $Body | ConvertTo-Json -Depth 30
|
|
return Invoke-RestMethod -Method Post -Uri $Url -ContentType "application/json" -Body $jsonBody -TimeoutSec 30
|
|
}
|
|
|
|
function Invoke-HttpProbe {
|
|
param([string]$Url, [hashtable]$Headers = @{})
|
|
try {
|
|
$response = Invoke-WebRequest -UseBasicParsing -Method Get -Uri $Url -Headers $Headers -TimeoutSec 30
|
|
return [ordered]@{
|
|
status_code = [int]$response.StatusCode
|
|
ok = ([int]$response.StatusCode -ge 200 -and [int]$response.StatusCode -lt 300)
|
|
content_length = if ($null -ne $response.Content) { $response.Content.Length } else { 0 }
|
|
content = $response.Content
|
|
}
|
|
} catch {
|
|
$statusCode = $null
|
|
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
|
$statusCode = [int]$_.Exception.Response.StatusCode
|
|
}
|
|
return [ordered]@{
|
|
status_code = $statusCode
|
|
ok = $false
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
}
|
|
|
|
function New-SmokeAccount {
|
|
param(
|
|
[string]$ApiBase,
|
|
[string]$RunStamp,
|
|
[string]$ArtifactDir
|
|
)
|
|
|
|
$username = ("codex-product-e2e-{0}@example.com" -f ($RunStamp -replace "[^0-9A-Za-z]", "")).ToLowerInvariant()
|
|
$password = "E2e!" + ([guid]::NewGuid().ToString("N").Substring(0, 18)) + "a1"
|
|
$displayName = "Codex Product E2E"
|
|
|
|
$registerResponse = Invoke-JsonPost -Url "$ApiBase/auth/register" -Body @{
|
|
username = $username
|
|
password = $password
|
|
display_name = $displayName
|
|
}
|
|
if ([int]$registerResponse.code -ne 0) {
|
|
throw "E2E account register failed: $($registerResponse.message)"
|
|
}
|
|
|
|
$loginResponse = Invoke-JsonPost -Url "$ApiBase/auth/login" -Body @{
|
|
account = $username
|
|
password = $password
|
|
}
|
|
if ([int]$loginResponse.code -ne 0) {
|
|
throw "E2E account login failed: $($loginResponse.message)"
|
|
}
|
|
|
|
$safePayload = [ordered]@{
|
|
username = $username
|
|
user = $loginResponse.data.user
|
|
token_present = -not [string]::IsNullOrWhiteSpace([string]$loginResponse.data.access_token)
|
|
}
|
|
Save-Json -Path (Join-Path $ArtifactDir "e2e-account.json") -Value $safePayload
|
|
|
|
return [ordered]@{
|
|
username = $username
|
|
access_token = [string]$loginResponse.data.access_token
|
|
user = $loginResponse.data.user
|
|
}
|
|
}
|
|
|
|
function Invoke-BrowseCommand {
|
|
param(
|
|
[string]$Name,
|
|
[string[]]$Arguments,
|
|
[string]$LogDir,
|
|
[switch]$AllowFailure
|
|
)
|
|
return Invoke-LoggedCommand -Name $Name -FilePath $script:ResolvedBrowsePath -Arguments $Arguments -LogDir $LogDir -AllowFailure:$AllowFailure
|
|
}
|
|
|
|
function Test-PageTextHealthy {
|
|
param([string]$Text)
|
|
if ([string]::IsNullOrWhiteSpace($Text)) {
|
|
return $false
|
|
}
|
|
if ($Text -match "Secure Access|Account Access|Login|not found|does not exist|failed to load") {
|
|
return $false
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-ChatTextHealthy {
|
|
param([string]$Text, [string]$NetworkText)
|
|
if ([string]::IsNullOrWhiteSpace($Text)) {
|
|
return $false
|
|
}
|
|
if ($Text -match "timeout|Timeout|error|Error|incomplete|Incomplete|failed|Failed|network|Network") {
|
|
return $false
|
|
}
|
|
if ($NetworkText -match "pending") {
|
|
return $false
|
|
}
|
|
return $true
|
|
}
|
|
|
|
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot "..")).Path
|
|
if (-not (Test-Path "$HOME\.Codex\skills\gstack\bin")) {
|
|
throw "GSTACK_MISSING: gstack is required for AI-assisted work in this repo."
|
|
}
|
|
|
|
if ([string]::IsNullOrWhiteSpace($ArtifactRoot)) {
|
|
$ArtifactRoot = Join-Path $repoRoot "project-docs\qa-artifacts\phase15-product-e2e"
|
|
}
|
|
$timestamp = Get-Date -Format "yyyy-MM-ddTHH-mm-ss"
|
|
$artifactDir = Join-Path $ArtifactRoot $timestamp
|
|
New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null
|
|
|
|
$summaryJsonPath = Join-Path $artifactDir "phase15-product-e2e-summary.json"
|
|
$summaryMdPath = Join-Path $artifactDir "phase15-product-e2e-summary.md"
|
|
$apiSummaryPath = Join-Path $artifactDir "phase15-product-e2e-api.json"
|
|
$browserSummaryPath = Join-Path $artifactDir "phase15-product-e2e-browser.json"
|
|
$checks = New-Object System.Collections.Generic.List[object]
|
|
$apiBase = $BackendBaseUrl.TrimEnd("/") + "/api/v1"
|
|
|
|
Add-Check -Checks $checks -Key "gstack" -Label "gstack prerequisite" -Status "pass" -Detail "GSTACK_OK"
|
|
|
|
$health = Invoke-HttpProbe -Url ($BackendBaseUrl.TrimEnd("/") + "/health")
|
|
Add-Check -Checks $checks -Key "backend_health" -Label "backend health" -Status $(if ($health.ok) { "pass" } else { "fail" }) -Detail "GET /health status=$($health.status_code)" -Meta @{ url = ($BackendBaseUrl.TrimEnd("/") + "/health") }
|
|
|
|
$readiness = Invoke-HttpProbe -Url ($apiBase + "/system/readiness")
|
|
Add-Check -Checks $checks -Key "backend_readiness" -Label "backend readiness" -Status $(if ($readiness.ok) { "pass" } else { "fail" }) -Detail "GET /api/v1/system/readiness status=$($readiness.status_code)" -Meta @{ url = ($apiBase + "/system/readiness") }
|
|
|
|
try {
|
|
$smokeAccount = New-SmokeAccount -ApiBase $apiBase -RunStamp $timestamp -ArtifactDir $artifactDir
|
|
Add-Check -Checks $checks -Key "e2e_account" -Label "E2E account" -Status "pass" -Detail "registered and logged in E2E account" -Meta @{ username = $smokeAccount.username; user_id = $smokeAccount.user.id; role = $smokeAccount.user.role }
|
|
} catch {
|
|
Add-Check -Checks $checks -Key "e2e_account" -Label "E2E account" -Status "fail" -Detail $_.Exception.Message
|
|
$smokeAccount = $null
|
|
}
|
|
|
|
if ($null -ne $smokeAccount -and -not [string]::IsNullOrWhiteSpace([string]$smokeAccount.access_token)) {
|
|
$sharedAIProbe = Invoke-HttpProbe -Url ($apiBase + "/system/shared-ai") -Headers @{ Authorization = "Bearer $($smokeAccount.access_token)" }
|
|
$sharedAIStatus = if ($sharedAIProbe.ok) { "pass" } else { "warn" }
|
|
Add-Check -Checks $checks -Key "shared_ai_config" -Label "shared AI config" -Status $sharedAIStatus -Required:$false -Detail "GET /api/v1/system/shared-ai status=$($sharedAIProbe.status_code)" -Meta @{ url = ($apiBase + "/system/shared-ai") }
|
|
} else {
|
|
Add-Check -Checks $checks -Key "shared_ai_config" -Label "shared AI config" -Status "skip" -Required:$false -Detail "E2E account token unavailable"
|
|
}
|
|
|
|
$authMode = ""
|
|
$sshpassPath = $null
|
|
if ($UseSshpass -or -not [string]::IsNullOrWhiteSpace($env:SSHPASS)) {
|
|
$sshpassPath = Resolve-SshpassPath
|
|
if ([string]::IsNullOrWhiteSpace($sshpassPath)) {
|
|
throw "sshpass.exe not found. Install sshpass or use SSH key auth."
|
|
}
|
|
if ([string]::IsNullOrWhiteSpace($env:SSHPASS)) {
|
|
throw "SSHPASS environment variable is empty."
|
|
}
|
|
$authMode = "sshpass"
|
|
} elseif (Test-Path -LiteralPath $SshKeyPath -PathType Leaf) {
|
|
$authMode = "key"
|
|
} else {
|
|
throw "SSH key not found and SSHPASS is not configured."
|
|
}
|
|
|
|
$target = "${RemoteUser}@${RemoteHost}"
|
|
$remoteWorkDir = "/tmp/ai-contract-phase15-product-e2e-$timestamp"
|
|
if ($authMode -eq "sshpass") {
|
|
$sshFile = $sshpassPath
|
|
$scpFile = $sshpassPath
|
|
$sshPrefix = @("-e", "ssh")
|
|
$scpPrefix = @("-e", "scp")
|
|
} else {
|
|
$sshFile = "ssh"
|
|
$scpFile = "scp"
|
|
$sshPrefix = @("-i", $SshKeyPath)
|
|
$scpPrefix = @("-i", $SshKeyPath)
|
|
}
|
|
|
|
$sshCommon = @("-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10")
|
|
$scpCommon = @("-o", "StrictHostKeyChecking=accept-new", "-o", "ConnectTimeout=10")
|
|
if ($authMode -eq "key") {
|
|
$sshCommon = @("-o", "BatchMode=yes") + $sshCommon
|
|
$scpCommon = @("-o", "BatchMode=yes") + $scpCommon
|
|
}
|
|
|
|
$seedScriptPath = Join-Path $artifactDir "phase15_product_e2e_seed.py"
|
|
$seedScript = @'
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from typing import Any, Dict
|
|
|
|
from app.db import SessionLocal
|
|
from app.services.legal_document_element_schema_service import LegalDocumentElementSchemaService
|
|
from app.services.legal_document_type_service import LegalDocumentTypeService
|
|
from app.services.template_service import TemplateService
|
|
|
|
|
|
def compact_seed_result(value: Dict[str, Any]) -> Dict[str, Any]:
|
|
return {
|
|
"created_count": len(value.get("created") or []),
|
|
"existing_count": len(value.get("existing") or []),
|
|
"skipped_count": len(value.get("skipped") or []),
|
|
"total": value.get("total"),
|
|
}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output-dir", required=True)
|
|
parser.add_argument("--tenant-id", required=True)
|
|
args = parser.parse_args()
|
|
|
|
os.makedirs(args.output_dir, exist_ok=True)
|
|
current_user = {
|
|
"id": "phase15-product-e2e-admin",
|
|
"tenant_id": args.tenant_id,
|
|
"display_name": "Phase15 Product E2E",
|
|
"role": "admin",
|
|
}
|
|
db = SessionLocal()
|
|
try:
|
|
document_type_result = LegalDocumentTypeService(db).seed_default_document_types(current_user)
|
|
schema_result = LegalDocumentElementSchemaService(db).seed_default_element_schemas(current_user)
|
|
template_result = TemplateService(db).seed_default_templates(current_user)
|
|
finally:
|
|
db.close()
|
|
|
|
summary = {
|
|
"status": "PASS",
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"document_types": compact_seed_result(document_type_result),
|
|
"element_schemas": compact_seed_result(schema_result),
|
|
"templates": compact_seed_result(template_result),
|
|
}
|
|
output_path = os.path.join(args.output_dir, "phase15-product-e2e-seed.json")
|
|
with open(output_path, "w", encoding="utf-8") as handle:
|
|
json.dump(summary, handle, ensure_ascii=False, indent=2)
|
|
print(json.dumps({"status": "PASS", "output_path": output_path}, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
'@
|
|
$seedScript | Set-Content -Encoding UTF8 -Path $seedScriptPath
|
|
|
|
try {
|
|
Invoke-LoggedCommand -Name "remote-mkdir" -FilePath $sshFile -Arguments ($sshPrefix + $sshCommon + @($target, "rm -rf $(ConvertTo-ShellSingleQuote $remoteWorkDir) && mkdir -p $(ConvertTo-ShellSingleQuote $remoteWorkDir)")) -LogDir $artifactDir | Out-Null
|
|
Invoke-LoggedCommand -Name "scp-seed-script" -FilePath $scpFile -Arguments ($scpPrefix + $scpCommon + @($seedScriptPath, "${target}:$remoteWorkDir/phase15_product_e2e_seed.py")) -LogDir $artifactDir | Out-Null
|
|
|
|
$containerSeedScript = "$remoteWorkDir/phase15_product_e2e_seed.py"
|
|
$containerOutput = "$remoteWorkDir/output"
|
|
$containerRunCommand = "cd /app && PYTHONPATH=/app python $(ConvertTo-ShellSingleQuote $containerSeedScript) --output-dir $(ConvertTo-ShellSingleQuote $containerOutput) --tenant-id $(ConvertTo-ShellSingleQuote $TenantId)"
|
|
$remoteCommandParts = @(
|
|
"set +e",
|
|
"docker exec $BackendContainer sh -lc $(ConvertTo-ShellSingleQuote "rm -rf $remoteWorkDir && mkdir -p $containerOutput")",
|
|
"prep_exit=`$?",
|
|
"if [ `$prep_exit -ne 0 ]; then exit `$prep_exit; fi",
|
|
"docker cp $remoteWorkDir/phase15_product_e2e_seed.py ${BackendContainer}:$containerSeedScript",
|
|
"copy_in_exit=`$?",
|
|
"if [ `$copy_in_exit -ne 0 ]; then exit `$copy_in_exit; fi",
|
|
"docker exec $BackendContainer sh -lc $(ConvertTo-ShellSingleQuote $containerRunCommand)",
|
|
"seed_exit=`$?",
|
|
"docker cp ${BackendContainer}:$containerOutput $remoteWorkDir/output >/dev/null 2>&1 || true",
|
|
"exit `$seed_exit"
|
|
)
|
|
$remoteSeed = Invoke-LoggedCommand -Name "remote-seed-defaults" -FilePath $sshFile -Arguments ($sshPrefix + $sshCommon + @($target, ($remoteCommandParts -join " ; "))) -LogDir $artifactDir -AllowFailure
|
|
Invoke-LoggedCommand -Name "scp-seed-output" -FilePath $scpFile -Arguments ($scpPrefix + $scpCommon + @("-r", "${target}:$remoteWorkDir/output/.", $artifactDir)) -LogDir $artifactDir | Out-Null
|
|
$seedSummary = Read-JsonFile -Path (Join-Path $artifactDir "phase15-product-e2e-seed.json")
|
|
Add-Check -Checks $checks -Key "remote_seed_defaults" -Label "remote seed defaults" -Status $(if ($remoteSeed.exit_code -eq 0 -and $seedSummary.status -eq "PASS") { "pass" } else { "fail" }) -Detail "document_types existing=$($seedSummary.document_types.existing_count); templates existing=$($seedSummary.templates.existing_count)"
|
|
} catch {
|
|
Add-Check -Checks $checks -Key "remote_seed_defaults" -Label "remote seed defaults" -Status "fail" -Detail $_.Exception.Message
|
|
$seedSummary = $null
|
|
}
|
|
|
|
$pythonPath = Resolve-PythonPath
|
|
if ([string]::IsNullOrWhiteSpace($pythonPath)) {
|
|
Add-Check -Checks $checks -Key "python" -Label "local Python" -Status "fail" -Detail "python executable not found"
|
|
} else {
|
|
Add-Check -Checks $checks -Key "python" -Label "local Python" -Status "pass" -Detail "python executable found" -Meta @{ path = $pythonPath }
|
|
}
|
|
|
|
$apiScriptPath = Join-Path $artifactDir "phase15_product_e2e_api.py"
|
|
$apiScript = @'
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import socket
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from http.client import IncompleteRead
|
|
from typing import Any, Dict, Iterable, List, Optional, Tuple
|
|
from uuid import uuid4
|
|
|
|
|
|
DOCUMENT_CASES = [
|
|
{
|
|
"case_key": "investment_agreement",
|
|
"document_type_code": "investment_agreement",
|
|
"expected_prompt_key": "investment_agreement_default",
|
|
},
|
|
{
|
|
"case_key": "valuation_adjustment_agreement",
|
|
"document_type_code": "valuation_adjustment_agreement",
|
|
"expected_prompt_key": "valuation_adjustment_agreement_default",
|
|
},
|
|
{
|
|
"case_key": "shareholder_resolution",
|
|
"document_type_code": "shareholder_resolution",
|
|
"expected_prompt_key": "shareholder_resolution_default",
|
|
},
|
|
{
|
|
"case_key": "power_of_attorney",
|
|
"document_type_code": "power_of_attorney",
|
|
"expected_prompt_key": "power_of_attorney_default",
|
|
},
|
|
]
|
|
|
|
|
|
def env(name: str, default: str = "") -> str:
|
|
return os.environ.get(name, default).strip()
|
|
|
|
|
|
BACKEND_BASE_URL = env("PHASE15_E2E_BACKEND_BASE_URL").rstrip("/")
|
|
API_BASE = f"{BACKEND_BASE_URL}/api/v1"
|
|
TOKEN = env("PHASE15_E2E_ACCESS_TOKEN")
|
|
OUT_DIR = env("PHASE15_E2E_OUTPUT_DIR")
|
|
|
|
|
|
class ApiError(RuntimeError):
|
|
def __init__(self, message: str, status_code: Optional[int] = None, body: Optional[str] = None):
|
|
super().__init__(message)
|
|
self.status_code = status_code
|
|
self.body = body
|
|
|
|
|
|
def auth_headers(extra: Optional[Dict[str, str]] = None) -> Dict[str, str]:
|
|
headers = {"Authorization": f"Bearer {TOKEN}", "User-Agent": "phase15-product-e2e"}
|
|
if extra:
|
|
headers.update(extra)
|
|
return headers
|
|
|
|
|
|
def request_raw(method: str, path: str, body: Optional[bytes] = None, headers: Optional[Dict[str, str]] = None, timeout: int = 45) -> Tuple[int, Dict[str, str], bytes]:
|
|
url = path if path.startswith("http") else f"{API_BASE}{path}"
|
|
request = urllib.request.Request(url, data=body, method=method, headers=auth_headers(headers))
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout) as response:
|
|
return response.status, dict(response.headers.items()), response.read()
|
|
except urllib.error.HTTPError as exc:
|
|
payload = exc.read()
|
|
body_text = payload.decode("utf-8", errors="replace")
|
|
raise ApiError(f"HTTP {exc.code} {method} {url}", status_code=exc.code, body=body_text) from exc
|
|
except Exception as exc:
|
|
raise ApiError(f"{type(exc).__name__}: {exc}") from exc
|
|
|
|
|
|
def api_json(method: str, path: str, payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
|
|
body = None
|
|
headers = None
|
|
if payload is not None:
|
|
body = json.dumps(payload, ensure_ascii=False).encode("utf-8")
|
|
headers = {"Content-Type": "application/json"}
|
|
_, _, raw = request_raw(method, path, body=body, headers=headers)
|
|
decoded = json.loads(raw.decode("utf-8"))
|
|
if int(decoded.get("code", 0)) != 0:
|
|
raise ApiError(f"API code {decoded.get('code')}: {decoded.get('message')}")
|
|
return decoded.get("data")
|
|
|
|
|
|
def api_bytes(path: str) -> Tuple[int, Dict[str, str], bytes]:
|
|
return request_raw("GET", path, timeout=60)
|
|
|
|
|
|
def encode_multipart(fields: Dict[str, str]) -> Tuple[bytes, str]:
|
|
boundary = f"----phase15-e2e-{uuid4().hex}"
|
|
chunks: List[bytes] = []
|
|
for name, value in fields.items():
|
|
chunks.append(f"--{boundary}\r\n".encode("utf-8"))
|
|
chunks.append(f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode("utf-8"))
|
|
chunks.append(str(value).encode("utf-8"))
|
|
chunks.append(b"\r\n")
|
|
chunks.append(f"--{boundary}--\r\n".encode("utf-8"))
|
|
return b"".join(chunks), f"multipart/form-data; boundary={boundary}"
|
|
|
|
|
|
def api_multipart(path: str, fields: Dict[str, str]) -> Dict[str, Any]:
|
|
body, content_type = encode_multipart(fields)
|
|
_, _, raw = request_raw("POST", path, body=body, headers={"Content-Type": content_type}, timeout=90)
|
|
decoded = json.loads(raw.decode("utf-8"))
|
|
if int(decoded.get("code", 0)) != 0:
|
|
raise ApiError(f"API code {decoded.get('code')}: {decoded.get('message')}")
|
|
return decoded.get("data")
|
|
|
|
|
|
def option_value(option: Any) -> str:
|
|
if isinstance(option, dict):
|
|
return str(option.get("value") or option.get("label") or "")
|
|
return str(option or "")
|
|
|
|
|
|
def sample_field_value(document_type_code: str, field: Dict[str, Any]) -> Any:
|
|
key = str(field.get("key") or "").lower()
|
|
field_type = str(field.get("type") or "").lower()
|
|
options = field.get("options") or []
|
|
if field_type in {"select", "radio"} and options:
|
|
return option_value(options[0])
|
|
if field_type == "date":
|
|
return "2026-05-25"
|
|
if field_type in {"money", "number"}:
|
|
return "1000000"
|
|
if field_type == "percentage":
|
|
return "20"
|
|
if field_type == "checkbox":
|
|
return True
|
|
if field_type == "clause_toggle":
|
|
return True
|
|
if field_type == "party":
|
|
return {"name": "Alpha Phase15 E2E Ltd.", "contact": "Codex E2E"}
|
|
if "party_a" in key or "investor" in key or "principal" in key:
|
|
return "Alpha Phase15 E2E Ltd."
|
|
if "party_b" in key or "target" in key or "agent" in key:
|
|
return "Beta Phase15 E2E Ltd."
|
|
if "company" in key:
|
|
return "Beta Phase15 E2E Ltd."
|
|
if "dispute" in key:
|
|
return "PRC law applies; disputes will be submitted to arbitration."
|
|
if "payment" in key or "price" in key or "fee" in key:
|
|
return "Payment is tied to delivery, acceptance, invoice issuance, and breach remedies."
|
|
if "risk" in key or "liability" in key or "warranty" in key:
|
|
return "Liability, warranty, breach, notice, termination, and evidence retention must be reviewed."
|
|
if "notice" in key or "filing" in key or "registration" in key:
|
|
return "Registration, filing, notice delivery, and evidence retention checklist."
|
|
if document_type_code == "investment_agreement":
|
|
return "Investment closing, governance rights, representations, warranties, transfer restrictions, and exit mechanism."
|
|
if document_type_code == "valuation_adjustment_agreement":
|
|
return "Valuation adjustment, performance commitment, compensation trigger, repurchase arrangement, and company law constraints."
|
|
if document_type_code == "shareholder_resolution":
|
|
return "Shareholder meeting notice, voting rights, resolutions, authorization, filing materials, and company charter basis."
|
|
if document_type_code == "power_of_attorney":
|
|
return "Authorization scope, delegation boundary, seal requirements, revocation notice, and evidence retention."
|
|
return f"Phase15 E2E value for {field.get('key') or 'field'}"
|
|
|
|
|
|
def build_element_values(schema_json: Dict[str, Any], document_type_code: str) -> Dict[str, Any]:
|
|
values: Dict[str, Any] = {}
|
|
for section in schema_json.get("sections") or []:
|
|
for field in section.get("fields") or []:
|
|
key = str(field.get("key") or "").strip()
|
|
if not key:
|
|
continue
|
|
if field.get("required") is True or field.get("type") in {"select", "radio"}:
|
|
values[key] = sample_field_value(document_type_code, field)
|
|
elif any(marker in key.lower() for marker in ("risk", "dispute", "payment", "filing", "notice", "liability", "registration")):
|
|
values[key] = sample_field_value(document_type_code, field)
|
|
return values
|
|
|
|
|
|
def selected_runtime(runtime: Optional[Dict[str, Any]]) -> Dict[str, Any]:
|
|
runtime = runtime or {}
|
|
keys = [
|
|
"available",
|
|
"prompt_asset_id",
|
|
"prompt_asset_version_id",
|
|
"prompt_key",
|
|
"prompt_type",
|
|
"version_label",
|
|
"prompt_version",
|
|
"template_key",
|
|
"target_scene",
|
|
"source",
|
|
"match_strategy",
|
|
"evaluation_status",
|
|
"evaluation_score",
|
|
]
|
|
return {key: runtime.get(key) for key in keys if key in runtime}
|
|
|
|
|
|
def collect_legal_ref_count(result: Optional[Dict[str, Any]], report: Optional[Dict[str, Any]], generated: Optional[Dict[str, Any]], detail: Optional[Dict[str, Any]]) -> Dict[str, int]:
|
|
generated_refs = len((generated or {}).get("related_legal_articles") or (detail or {}).get("related_legal_articles") or [])
|
|
risk_refs = 0
|
|
for item in ((result or {}).get("risk_items") or []):
|
|
risk_refs += len(item.get("legal_article_refs") or [])
|
|
report_refs = int((report or {}).get("legal_article_ref_count") or 0)
|
|
return {
|
|
"generation_related_legal_articles": generated_refs,
|
|
"review_risk_legal_article_refs": risk_refs,
|
|
"report_legal_article_refs": report_refs,
|
|
"total": generated_refs + risk_refs + report_refs,
|
|
}
|
|
|
|
|
|
def pass_bool(value: bool) -> str:
|
|
return "pass" if value else "fail"
|
|
|
|
|
|
def run_case(case: Dict[str, Any]) -> Dict[str, Any]:
|
|
code = case["document_type_code"]
|
|
expected_prompt_key = case["expected_prompt_key"]
|
|
started_at = time.time()
|
|
result: Dict[str, Any] = {
|
|
"case_key": case["case_key"],
|
|
"document_type_code": code,
|
|
"expected_prompt_key": expected_prompt_key,
|
|
"status": "fail",
|
|
"checks": {},
|
|
}
|
|
try:
|
|
templates = api_json("GET", f"/contracts/templates?document_type_code={urllib.parse.quote(code)}")
|
|
result["template_count"] = len(templates or [])
|
|
if not templates:
|
|
raise RuntimeError(f"no enabled template for {code}")
|
|
template = templates[0]
|
|
result["template_id"] = template.get("id")
|
|
result["template_name"] = template.get("name")
|
|
result["checks"]["template_available"] = "pass"
|
|
|
|
schema = api_json("GET", f"/legal-document-types/{urllib.parse.quote(code)}/element-schema")
|
|
schema_json = schema.get("schema_json") or {}
|
|
elements = build_element_values(schema_json, code)
|
|
result["element_schema_id"] = schema.get("id")
|
|
result["element_count"] = len(elements)
|
|
result["checks"]["element_schema_available"] = pass_bool(bool(schema_json.get("sections")))
|
|
|
|
generated = api_multipart(
|
|
"/contracts/generate",
|
|
{
|
|
"template_id": str(template.get("id") or ""),
|
|
"document_type_code": code,
|
|
"elements_json": json.dumps(elements, ensure_ascii=False),
|
|
"variable_values_json": "{}",
|
|
"export_format": "html",
|
|
"options_json": json.dumps({"e2e": "phase15_product_e2e", "case_key": code}, ensure_ascii=False),
|
|
},
|
|
)
|
|
contract_id = str(generated.get("id") or "")
|
|
if not contract_id:
|
|
raise RuntimeError("generated contract id missing")
|
|
result["generated_contract_id"] = contract_id
|
|
result["file_name"] = generated.get("file_name")
|
|
result["checks"]["draft_generated"] = pass_bool(bool(generated.get("generated_content") or generated.get("generated_html")))
|
|
|
|
runtime = (((generated.get("metadata") or {}).get("prompt_runtime") or {}).get("contract_generation") or {})
|
|
result["runtime"] = selected_runtime(runtime)
|
|
result["checks"]["generation_prompt_runtime"] = pass_bool(
|
|
runtime.get("available") is True
|
|
and runtime.get("source") == "published_prompt_asset"
|
|
and runtime.get("prompt_key") == expected_prompt_key
|
|
)
|
|
|
|
detail = api_json("GET", f"/contracts/{urllib.parse.quote(contract_id)}")
|
|
result["checks"]["contract_detail"] = pass_bool(detail.get("id") == contract_id)
|
|
|
|
preview = api_json("GET", f"/contracts/{urllib.parse.quote(contract_id)}/preview")
|
|
result["checks"]["contract_preview"] = pass_bool(bool(preview.get("preview_content")))
|
|
|
|
download_status, download_headers, download_bytes = api_bytes(f"/contracts/{urllib.parse.quote(contract_id)}/download")
|
|
result["contract_download"] = {
|
|
"status_code": download_status,
|
|
"content_type": download_headers.get("Content-Type"),
|
|
"bytes": len(download_bytes),
|
|
}
|
|
result["checks"]["contract_export_download"] = pass_bool(download_status == 200 and len(download_bytes) > 0)
|
|
|
|
versions = api_json("GET", f"/contracts/{urllib.parse.quote(contract_id)}/versions")
|
|
version_items = versions.get("items") if isinstance(versions, dict) else versions
|
|
result["version_count"] = len(version_items or [])
|
|
result["checks"]["versions_list"] = pass_bool(len(version_items or []) >= 1)
|
|
if version_items:
|
|
version_id = str((version_items[0] or {}).get("id") or "")
|
|
result["first_version_id"] = version_id
|
|
version_status, _, version_bytes = api_bytes(f"/contracts/{urllib.parse.quote(contract_id)}/versions/{urllib.parse.quote(version_id)}/download?export_format=docx")
|
|
result["version_download"] = {"status_code": version_status, "bytes": len(version_bytes)}
|
|
result["checks"]["version_export_download"] = pass_bool(version_status == 200 and len(version_bytes) > 0)
|
|
|
|
auto_review = (generated.get("auto_review") or detail.get("auto_review") or {})
|
|
result["auto_review"] = {
|
|
"review_id": auto_review.get("review_id"),
|
|
"status": auto_review.get("status"),
|
|
"risk_level": auto_review.get("risk_level"),
|
|
"detail_url": auto_review.get("detail_url"),
|
|
"chat_url_present": bool(auto_review.get("chat_url")),
|
|
"report_download_url": auto_review.get("report_download_url"),
|
|
}
|
|
review_id = str(auto_review.get("review_id") or "")
|
|
result["checks"]["auto_review_created"] = pass_bool(bool(review_id))
|
|
result["checks"]["auto_review_completed"] = pass_bool(auto_review.get("status") in {"completed", "partial"})
|
|
|
|
review_detail = None
|
|
review_result = None
|
|
review_report = None
|
|
if review_id:
|
|
review_detail = api_json("GET", f"/reviews/{urllib.parse.quote(review_id)}")
|
|
review_result = api_json("GET", f"/reviews/{urllib.parse.quote(review_id)}/result")
|
|
review_report = api_json("GET", f"/reviews/{urllib.parse.quote(review_id)}/report")
|
|
report_status, report_headers, report_bytes = api_bytes(f"/reviews/{urllib.parse.quote(review_id)}/report/download")
|
|
result["review_report_download"] = {
|
|
"status_code": report_status,
|
|
"content_type": report_headers.get("Content-Type"),
|
|
"bytes": len(report_bytes),
|
|
"pdf_signature": report_bytes.startswith(b"%PDF"),
|
|
}
|
|
result["checks"]["review_detail"] = pass_bool(review_detail.get("id") == review_id)
|
|
result["checks"]["review_result"] = pass_bool(isinstance(review_result, dict))
|
|
result["checks"]["review_report"] = pass_bool(bool(review_report.get("download_url") or review_report.get("file_name")))
|
|
result["checks"]["review_report_download"] = pass_bool(report_status == 200 and report_bytes.startswith(b"%PDF"))
|
|
|
|
legal_counts = collect_legal_ref_count(review_result, review_report, generated, detail)
|
|
result["legal_article_ref_counts"] = legal_counts
|
|
result["checks"]["legal_article_references"] = "pass" if legal_counts["total"] > 0 else "warn"
|
|
|
|
required_check_names = [
|
|
"template_available",
|
|
"element_schema_available",
|
|
"draft_generated",
|
|
"generation_prompt_runtime",
|
|
"contract_detail",
|
|
"contract_preview",
|
|
"contract_export_download",
|
|
"auto_review_created",
|
|
"auto_review_completed",
|
|
"review_detail",
|
|
"review_result",
|
|
"review_report",
|
|
"review_report_download",
|
|
]
|
|
result["status"] = "pass" if all(result["checks"].get(name) == "pass" for name in required_check_names) else "fail"
|
|
except Exception as exc:
|
|
result["status"] = "fail"
|
|
result["error"] = str(exc)
|
|
if isinstance(exc, ApiError):
|
|
result["error_status_code"] = exc.status_code
|
|
result["error_body"] = exc.body
|
|
result["traceback"] = traceback.format_exc(limit=5)
|
|
finally:
|
|
result["duration_seconds"] = round(time.time() - started_at, 3)
|
|
return result
|
|
|
|
|
|
def main() -> int:
|
|
if not BACKEND_BASE_URL or not TOKEN or not OUT_DIR:
|
|
raise RuntimeError("PHASE15_E2E_BACKEND_BASE_URL, PHASE15_E2E_ACCESS_TOKEN, and PHASE15_E2E_OUTPUT_DIR are required")
|
|
os.makedirs(OUT_DIR, exist_ok=True)
|
|
cases = [run_case(case) for case in DOCUMENT_CASES]
|
|
generation_passed = sum(1 for item in cases if item.get("status") == "pass")
|
|
legal_ref_passed = sum(1 for item in cases if (item.get("legal_article_ref_counts") or {}).get("total", 0) > 0)
|
|
report_download_passed = sum(1 for item in cases if item.get("checks", {}).get("review_report_download") == "pass")
|
|
contract_download_passed = sum(1 for item in cases if item.get("checks", {}).get("contract_export_download") == "pass")
|
|
auto_review_passed = sum(1 for item in cases if item.get("checks", {}).get("auto_review_completed") == "pass")
|
|
summary = {
|
|
"status": "PASS" if generation_passed == len(DOCUMENT_CASES) else "FAIL",
|
|
"generated_at": datetime.now(timezone.utc).isoformat(),
|
|
"backend_base_url": BACKEND_BASE_URL,
|
|
"document_cases": cases,
|
|
"metrics": {
|
|
"document_cases_total": len(DOCUMENT_CASES),
|
|
"document_cases_passed": generation_passed,
|
|
"auto_review_completed": auto_review_passed,
|
|
"contract_export_download_passed": contract_download_passed,
|
|
"review_report_download_passed": report_download_passed,
|
|
"legal_article_reference_cases_passed": legal_ref_passed,
|
|
},
|
|
"manual_signoff_gaps": [
|
|
{
|
|
"code": "legal_article_refs_missing",
|
|
"severity": "blocker",
|
|
"detail": "One or more E2E cases completed generation/review/report but returned zero legal article references.",
|
|
"affected_cases": [
|
|
item.get("case_key")
|
|
for item in cases
|
|
if (item.get("legal_article_ref_counts") or {}).get("total", 0) == 0
|
|
],
|
|
}
|
|
] if legal_ref_passed < len(DOCUMENT_CASES) else [],
|
|
}
|
|
with open(os.path.join(OUT_DIR, "phase15-product-e2e-api.json"), "w", encoding="utf-8") as handle:
|
|
json.dump(summary, handle, ensure_ascii=False, indent=2)
|
|
print(json.dumps({"status": summary["status"], "metrics": summary["metrics"]}, ensure_ascii=False))
|
|
return 0 if summary["status"] == "PASS" else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|
|
'@
|
|
$apiScript | Set-Content -Encoding UTF8 -Path $apiScriptPath
|
|
|
|
$apiSummary = $null
|
|
if (-not [string]::IsNullOrWhiteSpace($pythonPath) -and $null -ne $smokeAccount -and -not [string]::IsNullOrWhiteSpace([string]$smokeAccount.access_token)) {
|
|
$previousBackendBase = $env:PHASE15_E2E_BACKEND_BASE_URL
|
|
$previousToken = $env:PHASE15_E2E_ACCESS_TOKEN
|
|
$previousOutDir = $env:PHASE15_E2E_OUTPUT_DIR
|
|
try {
|
|
$env:PHASE15_E2E_BACKEND_BASE_URL = $BackendBaseUrl.TrimEnd("/")
|
|
$env:PHASE15_E2E_ACCESS_TOKEN = $smokeAccount.access_token
|
|
$env:PHASE15_E2E_OUTPUT_DIR = $artifactDir
|
|
$apiRun = Invoke-LoggedCommand -Name "api-product-e2e" -FilePath $pythonPath -Arguments @($apiScriptPath) -LogDir $artifactDir -AllowFailure
|
|
if (Test-Path -LiteralPath $apiSummaryPath) {
|
|
$apiSummary = Read-JsonFile -Path $apiSummaryPath
|
|
Add-Check -Checks $checks -Key "api_product_e2e" -Label "API product E2E" -Status $(if ($apiRun.exit_code -eq 0 -and $apiSummary.status -eq "PASS") { "pass" } else { "fail" }) -Detail "documents=$($apiSummary.metrics.document_cases_passed)/$($apiSummary.metrics.document_cases_total); reports=$($apiSummary.metrics.review_report_download_passed)/$($apiSummary.metrics.document_cases_total); legal_refs=$($apiSummary.metrics.legal_article_reference_cases_passed)/$($apiSummary.metrics.document_cases_total)" -Meta @{ artifact = $apiSummaryPath }
|
|
} else {
|
|
Add-Check -Checks $checks -Key "api_product_e2e" -Label "API product E2E" -Status "fail" -Detail "API summary artifact was not produced"
|
|
}
|
|
} catch {
|
|
Add-Check -Checks $checks -Key "api_product_e2e" -Label "API product E2E" -Status "fail" -Detail $_.Exception.Message
|
|
} finally {
|
|
$env:PHASE15_E2E_BACKEND_BASE_URL = $previousBackendBase
|
|
$env:PHASE15_E2E_ACCESS_TOKEN = $previousToken
|
|
$env:PHASE15_E2E_OUTPUT_DIR = $previousOutDir
|
|
}
|
|
} else {
|
|
Add-Check -Checks $checks -Key "api_product_e2e" -Label "API product E2E" -Status "fail" -Detail "Python or E2E token unavailable"
|
|
}
|
|
|
|
$script:ResolvedBrowsePath = Resolve-BrowsePath -RequestedPath $BrowsePath
|
|
$browserResults = [ordered]@{
|
|
contract_detail_pages = New-Object System.Collections.Generic.List[object]
|
|
review_detail_pages = New-Object System.Collections.Generic.List[object]
|
|
continue_followup_paths = New-Object System.Collections.Generic.List[object]
|
|
business_auxiliary_chat_paths = New-Object System.Collections.Generic.List[object]
|
|
}
|
|
|
|
$businessAuxiliaryCases = @(
|
|
[ordered]@{ case_key = "obligation_extraction"; message = "performance obligation extraction for this investment agreement" },
|
|
[ordered]@{ case_key = "filing_materials_checklist"; message = "filing materials checklist for shareholder registration change" },
|
|
[ordered]@{ case_key = "negotiation_strategy"; message = "negotiation strategy for valuation adjustment and repurchase clauses" },
|
|
[ordered]@{ case_key = "legal_opinion_boundary_fallback"; message = "legal opinion boundary fallback and human review disclaimer" }
|
|
)
|
|
|
|
if ([string]::IsNullOrWhiteSpace($script:ResolvedBrowsePath)) {
|
|
Add-Check -Checks $checks -Key "gstack_browse" -Label "gstack browse" -Status "fail" -Detail "browse.exe not found"
|
|
} elseif ($null -eq $smokeAccount -or [string]::IsNullOrWhiteSpace([string]$smokeAccount.access_token)) {
|
|
Add-Check -Checks $checks -Key "gstack_browse" -Label "gstack browse" -Status "fail" -Detail "E2E account token unavailable"
|
|
} else {
|
|
try {
|
|
Add-Check -Checks $checks -Key "gstack_browse" -Label "gstack browse" -Status "pass" -Detail "browse executable found" -Meta @{ path = $script:ResolvedBrowsePath }
|
|
Invoke-BrowseCommand -Name "browse-status" -Arguments @("status") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-login-page" -Arguments @("goto", ($FrontendBaseUrl.TrimEnd("/") + "/login")) -LogDir $artifactDir -AllowFailure | Out-Null
|
|
|
|
$authStore = [ordered]@{
|
|
state = [ordered]@{
|
|
accessToken = $smokeAccount.access_token
|
|
currentUser = $smokeAccount.user
|
|
}
|
|
version = 0
|
|
}
|
|
$authStoreString = $authStore | ConvertTo-Json -Compress -Depth 60
|
|
Invoke-BrowseCommand -Name "browse-set-access-token" -Arguments @("storage", "set", "access_token", ([string]$smokeAccount.access_token)) -LogDir $artifactDir | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-set-auth-store" -Arguments @("storage", "set", "auth-store", ([string]$authStoreString)) -LogDir $artifactDir | Out-Null
|
|
|
|
$generatedCases = @()
|
|
if ($null -ne $apiSummary) {
|
|
$generatedCases = @($apiSummary.document_cases | Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_.generated_contract_id) })
|
|
}
|
|
|
|
foreach ($case in $generatedCases) {
|
|
$contractId = [string]$case.generated_contract_id
|
|
$safeCase = ([string]$case.case_key) -replace "[^0-9A-Za-z_-]", "_"
|
|
$contractUrl = $FrontendBaseUrl.TrimEnd("/") + "/contracts/$contractId"
|
|
Invoke-BrowseCommand -Name "browse-clear-console-contract-$safeCase" -Arguments @("console", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-clear-network-contract-$safeCase" -Arguments @("network", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
$goto = Invoke-BrowseCommand -Name "browse-goto-contract-$safeCase" -Arguments @("goto", $contractUrl) -LogDir $artifactDir -AllowFailure
|
|
Start-Sleep -Milliseconds 1200
|
|
$finalUrl = (Invoke-BrowseCommand -Name "browse-url-contract-$safeCase" -Arguments @("url") -LogDir $artifactDir -AllowFailure).output.Trim()
|
|
$textOutput = (Invoke-BrowseCommand -Name "browse-text-contract-$safeCase" -Arguments @("text") -LogDir $artifactDir -AllowFailure).output
|
|
$consoleOutput = (Invoke-BrowseCommand -Name "browse-console-contract-$safeCase" -Arguments @("console", "--errors") -LogDir $artifactDir -AllowFailure).output
|
|
$contractPassed = (
|
|
($finalUrl -eq $contractUrl) -and
|
|
(Test-PageTextHealthy -Text $textOutput) -and
|
|
($consoleOutput -match "\(no console errors\)")
|
|
)
|
|
$browserResults["contract_detail_pages"].Add([ordered]@{
|
|
case_key = $case.case_key
|
|
contract_id = $contractId
|
|
status = if ($contractPassed) { "pass" } else { "fail" }
|
|
requested_url = $contractUrl
|
|
final_url = $finalUrl
|
|
goto_exit_code = $goto.exit_code
|
|
console_clean = ($consoleOutput -match "\(no console errors\)")
|
|
}) | Out-Null
|
|
|
|
$reviewId = [string]$case.auto_review.review_id
|
|
if (-not [string]::IsNullOrWhiteSpace($reviewId)) {
|
|
$reviewUrl = $FrontendBaseUrl.TrimEnd("/") + "/reviews/$reviewId"
|
|
Invoke-BrowseCommand -Name "browse-clear-console-review-$safeCase" -Arguments @("console", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-clear-network-review-$safeCase" -Arguments @("network", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
$reviewGoto = Invoke-BrowseCommand -Name "browse-goto-review-$safeCase" -Arguments @("goto", $reviewUrl) -LogDir $artifactDir -AllowFailure
|
|
Start-Sleep -Milliseconds 1200
|
|
$reviewFinalUrl = (Invoke-BrowseCommand -Name "browse-url-review-$safeCase" -Arguments @("url") -LogDir $artifactDir -AllowFailure).output.Trim()
|
|
$reviewTextOutput = (Invoke-BrowseCommand -Name "browse-text-review-$safeCase" -Arguments @("text") -LogDir $artifactDir -AllowFailure).output
|
|
$reviewConsoleOutput = (Invoke-BrowseCommand -Name "browse-console-review-$safeCase" -Arguments @("console", "--errors") -LogDir $artifactDir -AllowFailure).output
|
|
$reviewPassed = (
|
|
($reviewFinalUrl -eq $reviewUrl) -and
|
|
(Test-PageTextHealthy -Text $reviewTextOutput) -and
|
|
($reviewConsoleOutput -match "\(no console errors\)")
|
|
)
|
|
$browserResults["review_detail_pages"].Add([ordered]@{
|
|
case_key = $case.case_key
|
|
review_id = $reviewId
|
|
status = if ($reviewPassed) { "pass" } else { "fail" }
|
|
requested_url = $reviewUrl
|
|
final_url = $reviewFinalUrl
|
|
goto_exit_code = $reviewGoto.exit_code
|
|
console_clean = ($reviewConsoleOutput -match "\(no console errors\)")
|
|
}) | Out-Null
|
|
}
|
|
}
|
|
|
|
$firstChatCase = @($generatedCases | Where-Object { -not [string]::IsNullOrWhiteSpace([string]$_.auto_review.review_id) } | Select-Object -First 1)
|
|
if ($firstChatCase.Count -gt 0) {
|
|
$case = $firstChatCase[0]
|
|
$safeCase = ([string]$case.case_key) -replace "[^0-9A-Za-z_-]", "_"
|
|
$reviewPayload = [ordered]@{
|
|
reviewId = [string]$case.auto_review.review_id
|
|
prompt = "Please continue the review and list the next revision priorities for this generated contract."
|
|
metadata = [ordered]@{
|
|
reviewTitle = [string]$case.template_name
|
|
reviewStatus = [string]$case.auto_review.status
|
|
contractType = [string]$case.document_type_code
|
|
overallRisk = [string]$case.auto_review.risk_level
|
|
focusLabel = [string]$case.file_name
|
|
actionLabel = "continue review follow-up"
|
|
focusType = "generated_contract"
|
|
}
|
|
}
|
|
$reviewPayloadJson = $reviewPayload | ConvertTo-Json -Compress -Depth 20
|
|
$chatUrl = $FrontendBaseUrl.TrimEnd("/") + "/chat?review=" + [uri]::EscapeDataString($reviewPayloadJson)
|
|
Invoke-BrowseCommand -Name "browse-clear-console-followup-$safeCase" -Arguments @("console", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-clear-network-followup-$safeCase" -Arguments @("network", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
$followupGoto = Invoke-BrowseCommand -Name "browse-goto-followup-$safeCase" -Arguments @("goto", $chatUrl) -LogDir $artifactDir -AllowFailure
|
|
Start-Sleep -Seconds $ChatWaitSeconds
|
|
$followupFinalUrl = (Invoke-BrowseCommand -Name "browse-url-followup-$safeCase" -Arguments @("url") -LogDir $artifactDir -AllowFailure).output.Trim()
|
|
$followupTextOutput = (Invoke-BrowseCommand -Name "browse-text-followup-$safeCase" -Arguments @("text") -LogDir $artifactDir -AllowFailure).output
|
|
$followupConsoleOutput = (Invoke-BrowseCommand -Name "browse-console-followup-$safeCase" -Arguments @("console", "--errors") -LogDir $artifactDir -AllowFailure).output
|
|
$followupNetworkOutput = (Invoke-BrowseCommand -Name "browse-network-followup-$safeCase" -Arguments @("network") -LogDir $artifactDir -AllowFailure).output
|
|
$followupPassed = (
|
|
($followupFinalUrl -like ($FrontendBaseUrl.TrimEnd("/") + "/chat*")) -and
|
|
($followupConsoleOutput -match "\(no console errors\)") -and
|
|
(Test-ChatTextHealthy -Text $followupTextOutput -NetworkText $followupNetworkOutput)
|
|
)
|
|
$browserResults["continue_followup_paths"].Add([ordered]@{
|
|
case_key = $case.case_key
|
|
status = if ($followupPassed) { "pass" } else { "blocked" }
|
|
requested_url = $chatUrl
|
|
final_url = $followupFinalUrl
|
|
goto_exit_code = $followupGoto.exit_code
|
|
console_clean = ($followupConsoleOutput -match "\(no console errors\)")
|
|
network_pending = ($followupNetworkOutput -match "pending")
|
|
}) | Out-Null
|
|
}
|
|
|
|
foreach ($item in $businessAuxiliaryCases) {
|
|
$safeCase = ([string]$item.case_key) -replace "[^0-9A-Za-z_-]", "_"
|
|
$chatUrl = $FrontendBaseUrl.TrimEnd("/") + "/chat"
|
|
Invoke-BrowseCommand -Name "browse-clear-console-business-$safeCase" -Arguments @("console", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-clear-network-business-$safeCase" -Arguments @("network", "--clear") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
$goto = Invoke-BrowseCommand -Name "browse-goto-business-$safeCase" -Arguments @("goto", $chatUrl) -LogDir $artifactDir -AllowFailure
|
|
Invoke-BrowseCommand -Name "browse-wait-textarea-business-$safeCase" -Arguments @("wait", "textarea") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-fill-business-$safeCase" -Arguments @("fill", "textarea", ([string]$item.message)) -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Invoke-BrowseCommand -Name "browse-submit-business-$safeCase" -Arguments @("press", "Control+Enter") -LogDir $artifactDir -AllowFailure | Out-Null
|
|
Start-Sleep -Seconds $ChatWaitSeconds
|
|
$finalUrl = (Invoke-BrowseCommand -Name "browse-url-business-$safeCase" -Arguments @("url") -LogDir $artifactDir -AllowFailure).output.Trim()
|
|
$textOutput = (Invoke-BrowseCommand -Name "browse-text-business-$safeCase" -Arguments @("text") -LogDir $artifactDir -AllowFailure).output
|
|
$consoleOutput = (Invoke-BrowseCommand -Name "browse-console-business-$safeCase" -Arguments @("console", "--errors") -LogDir $artifactDir -AllowFailure).output
|
|
$networkOutput = (Invoke-BrowseCommand -Name "browse-network-business-$safeCase" -Arguments @("network") -LogDir $artifactDir -AllowFailure).output
|
|
$chatPassed = (
|
|
($finalUrl -eq $chatUrl) -and
|
|
($textOutput -match [regex]::Escape([string]$item.message)) -and
|
|
($consoleOutput -match "\(no console errors\)") -and
|
|
(Test-ChatTextHealthy -Text $textOutput -NetworkText $networkOutput)
|
|
)
|
|
$browserResults["business_auxiliary_chat_paths"].Add([ordered]@{
|
|
case_key = $item.case_key
|
|
status = if ($chatPassed) { "pass" } else { "blocked" }
|
|
message = $item.message
|
|
requested_url = $chatUrl
|
|
final_url = $finalUrl
|
|
goto_exit_code = $goto.exit_code
|
|
console_clean = ($consoleOutput -match "\(no console errors\)")
|
|
network_pending = ($networkOutput -match "pending")
|
|
}) | Out-Null
|
|
}
|
|
} catch {
|
|
Add-Check -Checks $checks -Key "browser_product_e2e" -Label "browser product E2E" -Status "fail" -Detail $_.Exception.Message
|
|
}
|
|
}
|
|
|
|
$contractPageItems = $browserResults["contract_detail_pages"].ToArray()
|
|
$reviewPageItems = $browserResults["review_detail_pages"].ToArray()
|
|
$continueFollowupItems = $browserResults["continue_followup_paths"].ToArray()
|
|
$businessChatItems = $browserResults["business_auxiliary_chat_paths"].ToArray()
|
|
$contractPagesPassed = @($contractPageItems | Where-Object { $_.status -eq "pass" }).Count
|
|
$reviewPagesPassed = @($reviewPageItems | Where-Object { $_.status -eq "pass" }).Count
|
|
$followupPassed = @($continueFollowupItems | Where-Object { $_.status -eq "pass" }).Count
|
|
$businessChatPassed = @($businessChatItems | Where-Object { $_.status -eq "pass" }).Count
|
|
$continueFollowupTotal = [int](($browserResults["continue_followup_paths"]).Count)
|
|
$businessAuxiliaryTotal = [int]$businessAuxiliaryCases.Count
|
|
|
|
if ($null -ne $apiSummary) {
|
|
$expectedCaseTotal = [int]$apiSummary.metrics.document_cases_total
|
|
} else {
|
|
$expectedCaseTotal = 4
|
|
}
|
|
|
|
if ($contractPagesPassed -eq $expectedCaseTotal) {
|
|
Add-Check -Checks $checks -Key "frontend_contract_detail_pages" -Label "frontend contract detail pages" -Status "pass" -Detail "contract_detail_pages=$contractPagesPassed/$expectedCaseTotal"
|
|
} else {
|
|
Add-Check -Checks $checks -Key "frontend_contract_detail_pages" -Label "frontend contract detail pages" -Status "fail" -Detail "contract_detail_pages=$contractPagesPassed/$expectedCaseTotal"
|
|
}
|
|
|
|
if ($reviewPagesPassed -eq $expectedCaseTotal) {
|
|
Add-Check -Checks $checks -Key "frontend_review_detail_pages" -Label "frontend review detail pages" -Status "pass" -Detail "review_detail_pages=$reviewPagesPassed/$expectedCaseTotal"
|
|
} else {
|
|
Add-Check -Checks $checks -Key "frontend_review_detail_pages" -Label "frontend review detail pages" -Status "fail" -Detail "review_detail_pages=$reviewPagesPassed/$expectedCaseTotal"
|
|
}
|
|
|
|
if (($continueFollowupTotal -gt 0) -and ($followupPassed -eq $continueFollowupTotal)) {
|
|
Add-Check -Checks $checks -Key "frontend_continue_followup_chat" -Label "frontend continue followup chat" -Status "pass" -Detail "continue_followup=$followupPassed/$continueFollowupTotal"
|
|
} else {
|
|
Add-Check -Checks $checks -Key "frontend_continue_followup_chat" -Label "frontend continue followup chat" -Status "blocked" -Required:$false -Detail "continue_followup=$followupPassed/$continueFollowupTotal"
|
|
}
|
|
|
|
if ($businessChatPassed -eq $businessAuxiliaryTotal) {
|
|
Add-Check -Checks $checks -Key "frontend_business_auxiliary_chat" -Label "frontend business auxiliary chat" -Status "pass" -Detail "business_auxiliary_chat=$businessChatPassed/$businessAuxiliaryTotal"
|
|
} else {
|
|
Add-Check -Checks $checks -Key "frontend_business_auxiliary_chat" -Label "frontend business auxiliary chat" -Status "blocked" -Required:$false -Detail "business_auxiliary_chat=$businessChatPassed/$businessAuxiliaryTotal"
|
|
}
|
|
|
|
Save-Json -Path $browserSummaryPath -Value $browserResults
|
|
|
|
$chatBlocked = ($businessChatPassed -lt $businessAuxiliaryTotal) -or ($followupPassed -lt $continueFollowupTotal)
|
|
if ($chatBlocked) {
|
|
try {
|
|
$backendLogRun = Invoke-LoggedCommand -Name "remote-backend-logs-on-chat-block" -FilePath $sshFile -Arguments ($sshPrefix + $sshCommon + @($target, "docker logs --tail 160 $BackendContainer")) -LogDir $artifactDir -AllowFailure
|
|
$sharedLogRun = Invoke-LoggedCommand -Name "remote-shared-ai-logs-on-chat-block" -FilePath $sshFile -Arguments ($sshPrefix + $sshCommon + @($target, "docker logs --tail 160 $SharedAIContainer")) -LogDir $artifactDir -AllowFailure
|
|
$combinedLogs = "$($backendLogRun.output)`n$($sharedLogRun.output)"
|
|
if ($combinedLogs -match "prompt template version not found") {
|
|
Add-Check -Checks $checks -Key "chat_blocker_root_cause" -Label "chat blocker root cause" -Status "blocked" -Required:$false -Detail "shared-ai-platform prompt center reports prompt template version not found" -Meta @{ backend_log = $backendLogRun.log; shared_ai_log = $sharedLogRun.log }
|
|
} elseif ($combinedLogs -match "incomplete chunked read|RemoteProtocolError") {
|
|
Add-Check -Checks $checks -Key "chat_blocker_root_cause" -Label "chat blocker root cause" -Status "blocked" -Required:$false -Detail "backend stream reports incomplete chunked read from shared AI" -Meta @{ backend_log = $backendLogRun.log; shared_ai_log = $sharedLogRun.log }
|
|
} else {
|
|
Add-Check -Checks $checks -Key "chat_blocker_root_cause" -Label "chat blocker root cause" -Status "warn" -Required:$false -Detail "chat blocked; logs captured but known prompt-center marker was not found" -Meta @{ backend_log = $backendLogRun.log; shared_ai_log = $sharedLogRun.log }
|
|
}
|
|
} catch {
|
|
Add-Check -Checks $checks -Key "chat_blocker_root_cause" -Label "chat blocker root cause" -Status "warn" -Required:$false -Detail "failed to capture remote chat logs: $($_.Exception.Message)"
|
|
}
|
|
}
|
|
|
|
$legalRefCasesPassed = if ($null -ne $apiSummary) { [int]$apiSummary.metrics.legal_article_reference_cases_passed } else { 0 }
|
|
if ($legalRefCasesPassed -eq $expectedCaseTotal) {
|
|
Add-Check -Checks $checks -Key "legal_article_reference_coverage" -Label "legal article reference coverage" -Status "pass" -Required:$false -Detail "legal_article_refs=$legalRefCasesPassed/$expectedCaseTotal"
|
|
} else {
|
|
Add-Check -Checks $checks -Key "legal_article_reference_coverage" -Label "legal article reference coverage" -Status "blocked" -Required:$false -Detail "legal_article_refs=$legalRefCasesPassed/$expectedCaseTotal"
|
|
}
|
|
|
|
$requiredFailures = @($checks | Where-Object { $_.required -and $_.status -eq "fail" })
|
|
$blockedChecks = @($checks | Where-Object { $_.status -eq "blocked" })
|
|
$status = if ($requiredFailures.Count -gt 0) {
|
|
"FAIL"
|
|
} elseif ($blockedChecks.Count -gt 0) {
|
|
"BLOCKED"
|
|
} else {
|
|
"PASS"
|
|
}
|
|
|
|
$summary = [ordered]@{
|
|
status = $status
|
|
generated_at = (Get-Date).ToUniversalTime().ToString("o")
|
|
artifact_dir = $artifactDir
|
|
endpoints = [ordered]@{
|
|
backend_base_url = $BackendBaseUrl
|
|
frontend_base_url = $FrontendBaseUrl
|
|
backend_container = $BackendContainer
|
|
shared_ai_container = $SharedAIContainer
|
|
}
|
|
metrics = [ordered]@{
|
|
document_cases_total = $expectedCaseTotal
|
|
document_cases_passed = if ($null -ne $apiSummary) { [int]$apiSummary.metrics.document_cases_passed } else { 0 }
|
|
auto_review_completed = if ($null -ne $apiSummary) { [int]$apiSummary.metrics.auto_review_completed } else { 0 }
|
|
legal_article_reference_cases_passed = $legalRefCasesPassed
|
|
contract_export_download_passed = if ($null -ne $apiSummary) { [int]$apiSummary.metrics.contract_export_download_passed } else { 0 }
|
|
review_report_download_passed = if ($null -ne $apiSummary) { [int]$apiSummary.metrics.review_report_download_passed } else { 0 }
|
|
frontend_contract_detail_pages_passed = $contractPagesPassed
|
|
frontend_review_detail_pages_passed = $reviewPagesPassed
|
|
frontend_continue_followup_passed = $followupPassed
|
|
frontend_continue_followup_total = $continueFollowupTotal
|
|
frontend_business_auxiliary_chat_passed = $businessChatPassed
|
|
frontend_business_auxiliary_chat_total = $businessAuxiliaryTotal
|
|
required_failures = $requiredFailures.Count
|
|
blocked_checks = $blockedChecks.Count
|
|
}
|
|
checks = $checks
|
|
artifacts = [ordered]@{
|
|
api = if (Test-Path -LiteralPath $apiSummaryPath) { $apiSummaryPath } else { $null }
|
|
browser = if (Test-Path -LiteralPath $browserSummaryPath) { $browserSummaryPath } else { $null }
|
|
seed = if (Test-Path -LiteralPath (Join-Path $artifactDir "phase15-product-e2e-seed.json")) { (Join-Path $artifactDir "phase15-product-e2e-seed.json") } else { $null }
|
|
}
|
|
}
|
|
Save-Json -Path $summaryJsonPath -Value $summary
|
|
|
|
$markdown = New-Object System.Collections.Generic.List[string]
|
|
$markdown.Add("# Phase15 Product E2E Acceptance Summary") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("- Status: $status") | Out-Null
|
|
$markdown.Add("- Artifact dir: $artifactDir") | Out-Null
|
|
$markdown.Add("- Backend: $BackendBaseUrl") | Out-Null
|
|
$markdown.Add("- Frontend: $FrontendBaseUrl") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("## Metrics") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("- Draft generation: $($summary.metrics.document_cases_passed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Auto review completed: $($summary.metrics.auto_review_completed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Legal article references: $($summary.metrics.legal_article_reference_cases_passed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Contract export downloads: $($summary.metrics.contract_export_download_passed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Review report downloads: $($summary.metrics.review_report_download_passed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Frontend contract detail pages: $($summary.metrics.frontend_contract_detail_pages_passed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Frontend review detail pages: $($summary.metrics.frontend_review_detail_pages_passed)/$($summary.metrics.document_cases_total)") | Out-Null
|
|
$markdown.Add("- Frontend continue follow-up chat: $($summary.metrics.frontend_continue_followup_passed)/$($summary.metrics.frontend_continue_followup_total)") | Out-Null
|
|
$markdown.Add("- Frontend business auxiliary chat: $($summary.metrics.frontend_business_auxiliary_chat_passed)/$($summary.metrics.frontend_business_auxiliary_chat_total)") | Out-Null
|
|
$markdown.Add("- Required failures: $($summary.metrics.required_failures)") | Out-Null
|
|
$markdown.Add("- Blocked checks: $($summary.metrics.blocked_checks)") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
$markdown.Add("## Checks") | Out-Null
|
|
$markdown.Add("") | Out-Null
|
|
foreach ($check in $checks) {
|
|
$markdown.Add("- [$($check.status)] $($check.key): $($check.detail)") | Out-Null
|
|
}
|
|
$markdown | Set-Content -Encoding UTF8 -Path $summaryMdPath
|
|
|
|
if ($status -eq "PASS") {
|
|
Write-Host "PHASE15_PRODUCT_E2E_PASS $artifactDir"
|
|
exit 0
|
|
}
|
|
if ($status -eq "BLOCKED") {
|
|
Write-Host "PHASE15_PRODUCT_E2E_BLOCKED $artifactDir"
|
|
exit 2
|
|
}
|
|
|
|
Write-Host "PHASE15_PRODUCT_E2E_FAIL $artifactDir"
|
|
exit 1
|