diff --git a/.gitignore b/.gitignore index c8d8df5..e444354 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ memory/ # Local tool state and transient logs .hermes/ geckodriver.log +tmp_*.jsonl +*_capture.jsonl # Large artifacts and local docs copies *.pdf diff --git a/README.md b/README.md index ed29b04..0464cdc 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,7 @@ must-run before every market validation or deploy: .venv/bin/python -m unittest discover -s tests/unit -p "test_*.py" .venv/bin/python scripts/run_offline_checks.py .venv/bin/python scripts/check_local_demo_readiness.py +.venv/bin/python scripts/run_secret_static_checks.py .venv/bin/python scripts/check_trading_delivery_readiness.py ``` @@ -197,7 +198,7 @@ Rollback trigger: git rev-parse HEAD docker compose -p ai-ict -f /root/deploy/ai-ict/docker-compose.yml logs --tail=100 ai-ict git worktree add /tmp/ai-ict-rollback -cd /tmp/ai-ict-rollback && ./scripts/deploy_server.exp +cd /tmp/ai-ict-rollback && AI_ICT_DEPLOY_PASSWORD= AI_ICT_DEPLOY_USE_SUDO=1 .venv/bin/python scripts/deploy_with_password.py --host 192.168.3.90 --user yuqei --port 22 --remote-path /home/yuqei/project/ai-exchange --remote-archive /home/yuqei/ai-ict-deploy.tgz rm -rf /tmp/ai-ict-rollback ``` @@ -239,7 +240,8 @@ Record the current commit as the last-known-good SHA before deploy. If any gate - `.venv/bin/python scripts/run_historical_compatibility_repair.py --apply-safe-repairs` - apply the safe historical compatibility repairs for signal invalidations, MSS/FVG meta backfills, and liquidity safe merges limited to unreferenced supersets plus exact duplicate rows with identical sweep events; `needs_sweep_reference_migration` and `keep_version_history` groups still require manual review - `.venv/bin/python scripts/run_historical_compatibility_repair.py --apply-liquidity-reference-migrations` - apply executable liquidity reference migrations for exact duplicate pools with nonidentical sweep events; duplicate sweep timestamp conflicts are blocked instead of auto-merged - `python3 scripts/run_migration_static_checks.py` - check migration order and required core schema without connecting to the database -- `python3 scripts/run_script_static_checks.py` - check automation scripts for unsafe ambient environment fallback patterns +- `python3 scripts/run_script_static_checks.py` - check automation scripts for unsafe ambient environment fallback patterns and repo-tracked secret literals +- `python3 scripts/run_secret_static_checks.py` - scan git-tracked text files for high-confidence secret material without printing secret values - `python3 scripts/run_replay_file.py ` - run offline no-lookahead candle replay without database access - `python3 scripts/run_reports.py` - build sample reports from current services - `python3 scripts/run_api_server.py` - start a lightweight HTTP API server on `127.0.0.1:8000`; `/` serves the trading workbench and `/training` serves the training console diff --git a/RUNBOOK.md b/RUNBOOK.md index cb055f2..d6761f7 100644 --- a/RUNBOOK.md +++ b/RUNBOOK.md @@ -118,6 +118,7 @@ must-run before every market validation or deploy: .venv/bin/python -m unittest discover -s tests/unit -p "test_*.py" .venv/bin/python scripts/run_offline_checks.py .venv/bin/python scripts/check_local_demo_readiness.py +.venv/bin/python scripts/run_secret_static_checks.py .venv/bin/python scripts/check_trading_delivery_readiness.py ``` @@ -163,7 +164,7 @@ Rollback trigger: git rev-parse HEAD docker compose -p ai-ict -f /root/deploy/ai-ict/docker-compose.yml logs --tail=100 ai-ict git worktree add /tmp/ai-ict-rollback -cd /tmp/ai-ict-rollback && ./scripts/deploy_server.exp +cd /tmp/ai-ict-rollback && AI_ICT_DEPLOY_PASSWORD= AI_ICT_DEPLOY_USE_SUDO=1 .venv/bin/python scripts/deploy_with_password.py --host 192.168.3.90 --user yuqei --port 22 --remote-path /home/yuqei/project/ai-exchange --remote-archive /home/yuqei/ai-ict-deploy.tgz rm -rf /tmp/ai-ict-rollback ``` diff --git a/SESSION.md b/SESSION.md index fb45508..8536030 100644 --- a/SESSION.md +++ b/SESSION.md @@ -796,6 +796,7 @@ Automation execution-layer PXX remains closed and its known scope stays frozen. - Remote API responsiveness recovery passed on 2026-05-18: no-proxy `/health` returned `200` in milliseconds, standard `scripts/run_remote_ops_smoke.py --base-url http://192.168.3.90:8000 --attempts 2 --timeout-seconds 30` passed all 9 checks including `unattended_harness`, and SSH command execution recovered. Triggering read-only `POST /trading/runtime/run-once` succeeded, but `/trading/launch-readiness` still reports `blocked_by_runtime` because OKX market/broker read connectivity is refused through the configured proxy path: remote host and container both fail to connect to `192.168.3.73:10808`; container env has `MARKET_TRUTH_PROXY_URL=http://192.168.3.73:10808` and empty `BROKER_READ_PROXY_URL`. Independent `/trading/shadow-paper-validation` remains at `collecting_paper_samples` / `paper_ticket_sample_gate`, but live launch readiness requires `runtime_market_broker_ready` first. Latest `/trading/candidates?instrument_id=1&limit=5` returned candidate `4244` as `blocked`, `executable=false`, missing `fvg_retrace_confirmed`, rejected by `fvg_direction_mismatch`; no shadow paper ticket was created. - Added `docs/AI_ICT_实盘辅助分析决策助手上线必做清单_2026-05-18_v1.md` for the urgent launch target: read-only live analysis assistant, no order submission, no trading, no direct buy/sell instruction. The must-do launch queue is P0-1 read-only market data recovery, P0-2 `live_analysis_assistant_ready` narrow readiness, P0-3 forced boundary and de-traded UI, P0-4 one-screen cockpit health, P0-6 read-only launch smoke, and P0-5 minimal observation/note/weekly-review operations loop. - Trader Mode P1/P2 live gate recovered on 2026-05-18: used `yuqei` SSH + sudo with the user-provided temporary password, copied the complete existing root deploy `.env` into `/home/yuqei/project/ai-exchange/.env` without exposing secret values, redeployed via `scripts/deploy_with_password.py --host 192.168.3.90 --user yuqei --port 22 --remote-path /home/yuqei/project/ai-exchange --remote-archive /home/yuqei/ai-ict-deploy.tgz`, deploy smoke passed 11 endpoints, standard `run_remote_ops_smoke.py` passed all checks including `unattended_harness`, and `run_readonly_live_analysis_smoke.py --base-url http://192.168.3.90:8000 --attempts 2 --timeout-seconds 30 --output-json` passed with `ready_for_readonly_live_analysis=true`, `must_not_trade=true`, `trade_permission_required=false`, and `submit_cancel_or_live_execution_called=false`. Remote readiness now reports `market_stream_connected`, `market_data_gap_clear`, `readonly_account_context_status=available`, `product_operating_mode=restricted_decision_support`, and `real_live_ready=false`. +- Low-risk sensitive credential cleanup passed on 2026-05-18 without rotating or changing any live credentials: removed tracked legacy expect scripts that embedded password sends, replaced rollback docs/checker with `deploy_with_password.py` plus runtime env injection, added redacted `scripts/run_secret_static_checks.py`, wired it into `run_script_static_checks.py` and the Trading Delivery Checklist, ignored `tmp_*.jsonl` / `*_capture.jsonl`, and documented the boundary in `docs/AI_ICT_敏感凭据低风险清理记录_2026-05-18_v1.md`. Validation passed secret static checks, script static checks, 35 focused tests, `git diff --check` with only LF/CRLF warnings, and full unit discovery (`779` tests). Current tree secret scan passed; Git history was not rewritten, so any historical exposure should be handled by a future explicit rotation window after backup login and smoke paths are confirmed. - Current assisted-analysis next gate: keep R1/R2 focused on `decision_support_ready` trust/sample/quality evidence and shadow/paper/operator feedback collection. OKX trade-permission sandbox smoke is deferred until the project explicitly re-enters execution adapter or real-trading preparation. ## Exact Next Step diff --git a/TASKS.md b/TASKS.md index 4464bad..fd2ee0d 100644 --- a/TASKS.md +++ b/TASKS.md @@ -159,6 +159,7 @@ - [x] Trader Mode P1 local intraday usability closure: profile templates, A/D/N/R/Space hotkeys, plain-language focus quality explanation, Do Not Use recovery actions, local readonly smoke, full unit validation, and browser screenshot QA passed without changing execution/readiness gates - [x] Trader Mode P2 local enhancement closure: chart context bridge starter, BTC/ETH/SOL fast queue, after-close review draft, and ops duty panel implemented and locally validated without connecting real TradingView MCP or changing execution/readiness gates - [x] Trader Mode P1/P2 remote deploy/smoke closure: used `yuqei` SSH + sudo to migrate the complete read-only `.env` into `/home/yuqei/project/ai-exchange/.env`, redeployed current worktree, and passed deploy smoke, standard remote ops smoke, `unattended_harness`, and readonly live analysis smoke +- [x] Low-risk sensitive credential cleanup complete: removed tracked expect scripts with embedded password sends, switched rollback docs/checker to `deploy_with_password.py` + runtime env, added redacted `run_secret_static_checks.py`, wired it into script static checks and Trading Delivery Checklist, and kept all real remote/runtime credentials unchanged - [ ] R2 next collection action after readonly launch: wait for or surface a real executable candidate, then build shadow paper ticket / confirm paper execution / record operator feedback / add weekly review evidence without changing execution/live gates - [x] R3 OKX sandbox preflight starter implemented and deployed: signed dry-run request previews, idempotency key preview, error normalization catalog, `/trading/okx-sandbox-preflight`, live-readiness summary, cockpit surface, deploy smoke, remote ops smoke, and remote field/UI smoke - [x] R3 OKX sandbox isolated smoke tooling implemented and deployed: explicit `OKX_SANDBOX_SMOKE_EXECUTE` gate, submit/query/cancel/query/fills script, JSON artifact, proxy reuse, permission-error normalization, focused tests, full unit, deploy smoke, and remote ops smoke diff --git a/docs/AI_ICT_敏感凭据低风险清理记录_2026-05-18_v1.md b/docs/AI_ICT_敏感凭据低风险清理记录_2026-05-18_v1.md new file mode 100644 index 0000000..46c18bb --- /dev/null +++ b/docs/AI_ICT_敏感凭据低风险清理记录_2026-05-18_v1.md @@ -0,0 +1,53 @@ +# AI_ICT 敏感凭据低风险清理记录 2026-05-18 v1 + +## 本轮边界 + +本轮只做仓库侧敏感信息清理和防回归检查，不做任何真实凭据轮换。 + +明确不改： +- 远端 SSH / sudo 密码。 +- OKX key / secret / passphrase。 +- 数据库密码。 +- webhook URL。 +- proxy / broker / market truth 运行配置。 +- 远端正在运行的 `.env`。 +- `execution_allowed`、`real_live_ready`、risk governor、trust bucket、operator confirmation protocol。 + +## 已清理内容 + +- 删除旧 `expect` 明文运维脚本： + - `scripts/deploy_server.exp` + - `scripts/deploy_with_expect.exp` + - `scripts/fix_pg_hba.exp` + - `scripts/ssh_diagnose.exp` +- rollback 文档和检查器从 `deploy_server.exp` 切换到 `scripts/deploy_with_password.py`，密码只通过运行时环境变量注入。 +- `.gitignore` 新增 `tmp_*.jsonl` 和 `*_capture.jsonl`，避免 webhook capture / 临时事件日志被误提交。 +- 新增 `scripts/run_secret_static_checks.py`，只输出文件、行号、规则名和原因，不打印 secret 值。 +- `scripts/run_script_static_checks.py` 已接入 secret scan，现有脚本静态检查会一起挡住高置信明文凭据。 +- Trading Delivery Checklist 已加入 `run_secret_static_checks.py`，后续 market validation / deploy 前会先过敏感信息门。 + +## 当前风险说明 + +- 当前工作树不再追踪旧明文 expect 脚本。 +- `.env.local` 仍保持 gitignored，不进入仓库。 +- 未进行 Git 历史重写；`git log --all -- scripts/deploy_server.exp scripts/deploy_with_expect.exp scripts/fix_pg_hba.exp scripts/ssh_diagnose.exp` 显示历史中仍有 4 个提交触达过这些旧脚本。 +- 因本轮不做轮换，历史泄露风险没有被“密码失效”彻底关闭；只是避免继续扩散和再次误提交。 + +## 验证证据 + +- `python scripts/run_secret_static_checks.py`：passed，检查 git-tracked 文件且不打印 secret 值。 +- `python scripts/run_script_static_checks.py`：passed，已包含 secret scan。 +- `python -m unittest tests.unit.test_secret_static_checks tests.unit.test_script_static_checks tests.unit.test_trading_delivery_readiness tests.unit.test_deploy_with_password tests.unit.test_documentation_consistency`：35 tests OK。 +- `python -m unittest discover -s tests/unit`：779 tests OK。 +- `git diff --check`：仅 LF/CRLF 提示，无 whitespace error。 + +## 后续轮换建议 + +只有在确认备用登录方式、远端 `.env` 备份、部署 smoke 和回滚路径都可用之后，才逐项轮换： +- SSH / sudo 密码。 +- OKX read-only / future sandbox key。 +- 数据库密码。 +- webhook URL。 +- proxy auth。 + +轮换时应一次只换一类，换完立即运行对应 smoke，不要批量同时替换。 diff --git a/scripts/check_trading_delivery_readiness.py b/scripts/check_trading_delivery_readiness.py index bf00664..f6d5cf4 100644 --- a/scripts/check_trading_delivery_readiness.py +++ b/scripts/check_trading_delivery_readiness.py @@ -12,6 +12,13 @@ TRADING_DELIVERY_SECTION_HEADING = "Trading Delivery Checklist" TRADING_DELIVERY_COMPOSE_PREFIX = ( "docker compose -p ai-ict -f /root/deploy/ai-ict/docker-compose.yml" ) +TRADING_DELIVERY_ROLLBACK_DEPLOY_COMMAND = ( + "cd /tmp/ai-ict-rollback && AI_ICT_DEPLOY_PASSWORD= " + "AI_ICT_DEPLOY_USE_SUDO=1 .venv/bin/python scripts/deploy_with_password.py " + "--host 192.168.3.90 --user yuqei --port 22 " + "--remote-path /home/yuqei/project/ai-exchange " + "--remote-archive /home/yuqei/ai-ict-deploy.tgz" +) TRADING_DELIVERY_COMMAND_GROUPS = ( ( "pre_deploy_offline", @@ -19,6 +26,7 @@ TRADING_DELIVERY_COMMAND_GROUPS = ( '.venv/bin/python -m unittest discover -s tests/unit -p "test_*.py"', ".venv/bin/python scripts/run_offline_checks.py", ".venv/bin/python scripts/check_local_demo_readiness.py", + ".venv/bin/python scripts/run_secret_static_checks.py", ".venv/bin/python scripts/check_trading_delivery_readiness.py", ), ), @@ -58,7 +66,7 @@ TRADING_DELIVERY_COMMAND_GROUPS = ( "git rev-parse HEAD", f"{TRADING_DELIVERY_COMPOSE_PREFIX} logs --tail=100 ai-ict", "git worktree add /tmp/ai-ict-rollback ", - "cd /tmp/ai-ict-rollback && ./scripts/deploy_server.exp", + TRADING_DELIVERY_ROLLBACK_DEPLOY_COMMAND, "rm -rf /tmp/ai-ict-rollback", ), ), @@ -79,7 +87,8 @@ TRADING_DELIVERY_REQUIRED_SNIPPETS = ( ) TRADING_DELIVERY_REQUIRED_LOCAL_PATHS = ( ROOT / "scripts" / "check_local_demo_readiness.py", - ROOT / "scripts" / "deploy_server.exp", + ROOT / "scripts" / "deploy_with_password.py", + ROOT / "scripts" / "run_secret_static_checks.py", ROOT / "scripts" / "run_database_readiness.py", ROOT / "scripts" / "run_historical_compatibility_repair.py", ROOT / "scripts" / "run_offline_checks.py", @@ -97,6 +106,7 @@ TRADING_DELIVERY_COMMAND_PREFIXES = ( "curl ", "git ", "cd /tmp/ai-ict-rollback &&", + "AI_ICT_DEPLOY_PASSWORD=", "rm -rf /tmp/ai-ict-rollback", ) diff --git a/scripts/deploy_server.exp b/scripts/deploy_server.exp deleted file mode 100755 index 8840e0a..0000000 --- a/scripts/deploy_server.exp +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/expect -f -set timeout 300 -set host "192.168.3.90" -set user "root" -set password "816301" -set script_dir [file dirname [file normalize [info script]]] -set project_root [file normalize [file join $script_dir ".."]] -set archive "/tmp/ai-ict-deploy.tgz" -set remote_archive "/root/ai-ict-deploy.tgz" -set remote_path "/root/deploy/ai-ict" - -proc auth {} { - expect { - "*yes/no*" { send "yes\r"; exp_continue } - "*assword:*" { send "$::password\r"; exp_continue } - eof - } -} - -exec /bin/sh -c "cd '$project_root' && tar -czf '$archive' Dockerfile docker-compose.yml pyproject.toml README.md .env.example src scripts migrations" - -spawn ssh -o StrictHostKeyChecking=accept-new $user@$host "hostname && docker --version && ((docker compose version) || (docker-compose --version)) && mkdir -p $remote_path" -auth - -spawn scp -o StrictHostKeyChecking=accept-new $archive $user@$host:$remote_archive -auth - -spawn ssh -o StrictHostKeyChecking=accept-new $user@$host "if [ -f $remote_path/.env ]; then cp $remote_path/.env /root/ai-ict-deploy.env.bak; fi && rm -rf $remote_path/* && tar -xzf $remote_archive -C $remote_path && if [ -f /root/ai-ict-deploy.env.bak ]; then mv /root/ai-ict-deploy.env.bak $remote_path/.env; fi && cd $remote_path && ((docker compose down || docker-compose down) >/dev/null 2>&1 || true) && ((docker compose up -d --build) || (docker-compose up -d --build)) && ((docker compose ps) || (docker-compose ps))" -auth - -file delete -force $archive - -puts "deploy_complete" diff --git a/scripts/deploy_with_expect.exp b/scripts/deploy_with_expect.exp deleted file mode 100755 index b03d879..0000000 --- a/scripts/deploy_with_expect.exp +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/expect -f -set timeout 120 -set host "192.168.3.90" -set user "root" -set password "123456" -set local_path "/Users/yuqei/AI Exchange" -set remote_path "/root/deploy/ai-ict" - -spawn ssh -o StrictHostKeyChecking=accept-new $user@$host "mkdir -p $remote_path" -expect { - "*assword:*" { send "$password\r" } - eof -} -expect eof - -spawn rsync -az --delete --exclude ".git" --exclude ".venv" --exclude "__pycache__" --exclude "tests" -e "ssh -o StrictHostKeyChecking=accept-new" "$local_path/" "$user@$host:$remote_path/" -expect { - "*assword:*" { send "$password\r"; exp_continue } - eof -} -expect eof - -spawn ssh -o StrictHostKeyChecking=accept-new $user@$host "cd $remote_path && docker compose down && docker compose up -d --build" -expect { - "*assword:*" { send "$password\r" } - eof -} -expect eof diff --git a/scripts/fix_pg_hba.exp b/scripts/fix_pg_hba.exp deleted file mode 100755 index e6c6b53..0000000 --- a/scripts/fix_pg_hba.exp +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/expect -f -set timeout 600 -spawn ssh -o StrictHostKeyChecking=accept-new root@192.168.3.90 {python3 - <<'PY' -from pathlib import Path -hba = Path('/var/lib/pgsql/16/data/pg_hba.conf') -lines = [ - 'host ai_ict yuqei 172.18.0.0/16 scram-sha-256\n', - 'host ai_ict yuqei 172.19.0.0/16 scram-sha-256\n', - 'host ai_ict yuqei 172.20.0.0/16 scram-sha-256\n', - 'host ai_ict yuqei 172.21.0.0/16 scram-sha-256\n', - 'host ai_ict yuqei 172.22.0.0/16 scram-sha-256\n', - 'host ai_ict yuqei 172.23.0.0/16 scram-sha-256\n', -] -text = hba.read_text() -added = [] -for line in lines: - if line not in text: - with hba.open('a') as f: - f.write(line) - added.append(line.strip()) -print('ADDED', added) -print('TAIL') -for row in hba.read_text().splitlines()[-12:]: - print(row) -PY -su - postgres -c "psql -d postgres -c 'select pg_reload_conf();'" && echo RELOADED} -expect "*assword:*" { send "816301\r" } -expect eof diff --git a/scripts/run_script_static_checks.py b/scripts/run_script_static_checks.py index b069927..1d4273f 100644 --- a/scripts/run_script_static_checks.py +++ b/scripts/run_script_static_checks.py @@ -2,6 +2,12 @@ from pathlib import Path import ast import json import os +import sys + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from scripts.run_secret_static_checks import run_secret_static_checks ROOT = Path(__file__).resolve().parents[1] SCRIPT_DIR = ROOT / "scripts" @@ -17,6 +23,7 @@ def run_script_static_checks(env=None, script_dir: Path = SCRIPT_DIR) -> dict: findings = [] for path in sorted(script_dir.glob("*.py")): findings.extend(check_script_file(path)) + findings.extend(run_secret_static_checks(env=env)["findings"]) return { "passed": len(findings) == 0, "script_count": len(list(script_dir.glob("*.py"))), diff --git a/scripts/run_secret_static_checks.py b/scripts/run_secret_static_checks.py new file mode 100644 index 0000000..64006e9 --- /dev/null +++ b/scripts/run_secret_static_checks.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Iterable, Optional +import json +import os +import re +import subprocess + + +ROOT = Path(__file__).resolve().parents[1] + +PLACEHOLDER_PATTERN = re.compile( + r"^(<[^>]+>|\$\{[^}]+\}|your[_-].+|change[_-]?me|changeme|placeholder|example|dummy|test|xxx+)$", + re.IGNORECASE, +) +KNOWN_SECRET_PREFIX_PATTERN = re.compile( + r"(? dict: + env = os.environ if env is None else env + root = Path(root) + findings = [] + checked_files = list(files) if files is not None else _tracked_files(root) + for path in checked_files: + path = Path(path) + absolute = path if path.is_absolute() else root / path + if not absolute.is_file() or _is_binary_or_large(absolute): + continue + try: + text = absolute.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + findings.extend(_scan_text(path=absolute, root=root, text=text)) + return { + "passed": len(findings) == 0, + "checked_file_count": len(checked_files), + "findings": findings, + } + + +def _scan_text(*, path: Path, root: Path, text: str) -> list[dict]: + findings = [] + relative = _display_path(path, root) + normalized = relative.replace("\\", "/") + if _is_allowed_example_file(normalized): + return findings + if _is_tracked_runtime_env(normalized): + findings.append(_finding(relative, 1, "tracked_runtime_env", "runtime_env_files_must_not_be_tracked")) + + for line_number, line in enumerate(text.splitlines(), start=1): + rule = _line_rule(normalized, line) + if rule: + findings.append(_finding(relative, line_number, rule[0], rule[1])) + return findings + + +def _line_rule(normalized_path: str, line: str) -> Optional[tuple[str, str]]: + stripped = line.strip() + if not stripped or stripped.startswith("#"): + return None + if PRIVATE_KEY_PATTERN.search(line): + return ("private_key_material", "private_key_material_must_not_be_committed") + if KNOWN_SECRET_PREFIX_PATTERN.search(line): + return ("known_secret_prefix", "known_secret_prefix_must_not_be_committed") + if WEBHOOK_SECRET_URL_PATTERN.search(line) and not _contains_placeholder(line): + return ("webhook_secret_url", "webhook_secret_url_must_use_environment_variable") + if normalized_path.endswith(".exp"): + password_match = EXPECT_PASSWORD_ASSIGNMENT_PATTERN.search(line) + if password_match and not _is_placeholder(password_match.group(1)): + return ("expect_password_literal", "expect_scripts_must_not_embed_passwords") + send_match = EXPECT_SEND_LITERAL_PATTERN.search(line) + if send_match and _looks_like_expect_secret_send(send_match.group(1)): + return ("expect_send_secret_literal", "expect_scripts_must_not_send_secret_literals") + assignment_match = INLINE_SECRET_ASSIGNMENT_PATTERN.search(line) + if ( + assignment_match + and not normalized_path.startswith("tests/") + and not _is_placeholder(assignment_match.group(2)) + and not _is_dynamic_reference(assignment_match.group(2)) + ): + return ("inline_secret_assignment", "secrets_must_be_supplied_by_runtime_environment") + return None + + +def _tracked_files(root: Path) -> list[Path]: + process = subprocess.run( + ["git", "ls-files", "-z"], + cwd=root, + capture_output=True, + check=False, + ) + if process.returncode != 0: + return [path.relative_to(root) for path in root.rglob("*") if path.is_file()] + raw = process.stdout.decode("utf-8", errors="replace") + return [Path(item) for item in raw.split("\0") if item] + + +def _is_binary_or_large(path: Path) -> bool: + try: + return path.stat().st_size > 2_000_000 + except OSError: + return True + + +def _is_allowed_example_file(normalized_path: str) -> bool: + name = normalized_path.rsplit("/", 1)[-1].lower() + return name.endswith(".example") or name.endswith(".sample") or name.endswith(".template") + + +def _is_tracked_runtime_env(normalized_path: str) -> bool: + name = normalized_path.rsplit("/", 1)[-1].lower() + return name == ".env" or (name.startswith(".env.") and not _is_allowed_example_file(normalized_path)) + + +def _looks_like_expect_secret_send(value: str) -> bool: + if value in {"yes\\r", "y\\r", "$password\\r", "$::password\\r"}: + return False + if value.startswith("$") or _is_placeholder(value): + return False + return value.endswith("\\r") and bool(re.search(r"[A-Za-z0-9]", value)) + + +def _contains_placeholder(value: str) -> bool: + return any(token in value.lower() for token in ("<", "your_", "changeme", "placeholder", "example")) + + +def _is_dynamic_reference(value: str) -> bool: + cleaned = value.strip().strip("\"'") + return ( + cleaned.startswith("$") + or cleaned.startswith("os.") + or cleaned.startswith("env.") + or cleaned.startswith("env[") + or cleaned.startswith("settings.") + or "." in cleaned + ) + + +def _is_placeholder(value: str) -> bool: + cleaned = value.strip().strip("\"'").removesuffix("\\r") + return bool(PLACEHOLDER_PATTERN.match(cleaned)) + + +def _finding(file_path: str, line: int, pattern: str, reason: str) -> dict: + return { + "file": file_path, + "line": line, + "pattern": pattern, + "reason": reason, + } + + +def _display_path(path: Path, root: Path) -> str: + try: + return str(path.relative_to(root)) + except ValueError: + return path.name + + +def main(env=None) -> int: + env = os.environ if env is None else env + result = run_secret_static_checks(env=env) + if _get_bool(env, "SECRET_CHECKS_OUTPUT_JSON", False): + print(json.dumps(result, ensure_ascii=False, sort_keys=True)) + else: + print(f"Secret static checks passed: {result['passed']}") + print(f"Files checked: {result['checked_file_count']}") + for finding in result["findings"]: + print( + f"{finding['file']}:{finding['line']} " + f"{finding['reason']} pattern={finding['pattern']}" + ) + return 0 if result["passed"] else 2 + + +def _get_bool(env, key: str, default: bool) -> bool: + value = env.get(key) + if value is None or value == "": + return default + return str(value).strip().lower() in {"1", "true", "yes", "on"} + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ssh_diagnose.exp b/scripts/ssh_diagnose.exp deleted file mode 100755 index 506abaf..0000000 --- a/scripts/ssh_diagnose.exp +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/expect -f -log_user 1 -exp_internal 0 -set timeout 30 -spawn ssh -v -o PreferredAuthentications=password -o PubkeyAuthentication=no -o StrictHostKeyChecking=accept-new root@192.168.3.90 "exit" -expect { - "*assword:*" { send "123456\r"; exp_continue } - eof -} diff --git a/tests/unit/test_documentation_consistency.py b/tests/unit/test_documentation_consistency.py index f0c54e9..eb21f7b 100644 --- a/tests/unit/test_documentation_consistency.py +++ b/tests/unit/test_documentation_consistency.py @@ -20,6 +20,7 @@ NONEXISTENT_VISUAL_PLAN_NAME = "AI_ICT_" + "视觉盯盘与主动告警增强方 ACTIVE_TOP_LEVEL_DOCS = { "AI_ICT_实盘辅助分析决策助手上线必做清单_2026-05-18_v1.md", "AI_ICT_实盘辅助分析决策助手使用说明_2026-05-18_v1.md", + "AI_ICT_敏感凭据低风险清理记录_2026-05-18_v1.md", "AI_ICT_分析助手主线增强开发计划_2026-05-14_v1.md", "AI_ICT_实盘辅助分析决策助手可用性评估与补齐方案_2026-05-17_v1.md", "AI_ICT_全自动开发执行流程_v1.md", diff --git a/tests/unit/test_secret_static_checks.py b/tests/unit/test_secret_static_checks.py new file mode 100644 index 0000000..29d2646 --- /dev/null +++ b/tests/unit/test_secret_static_checks.py @@ -0,0 +1,58 @@ +import contextlib +import io +import json +import tempfile +import unittest +from pathlib import Path + +from scripts import run_secret_static_checks + + +class SecretStaticChecksTests(unittest.TestCase): + def test_detects_expect_password_without_printing_value(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + script = root / "deploy.exp" + script.write_text('set password "123456"\n', encoding="utf-8") + + result = run_secret_static_checks.run_secret_static_checks(root=root, files=(script,)) + + self.assertFalse(result["passed"]) + self.assertEqual(result["findings"][0]["pattern"], "expect_password_literal") + self.assertNotIn("123456", json.dumps(result, ensure_ascii=False)) + + def test_allows_placeholder_env_examples(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + example = root / ".env.example" + example.write_text("OKX_API_SECRET=\n", encoding="utf-8") + + result = run_secret_static_checks.run_secret_static_checks(root=root, files=(example,)) + + self.assertTrue(result["passed"]) + + def test_detects_tracked_runtime_env(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + env_file = root / ".env" + env_file.write_text("DATABASE_URL=postgresql://user:pass@localhost/db\n", encoding="utf-8") + + result = run_secret_static_checks.run_secret_static_checks(root=root, files=(env_file,)) + + self.assertFalse(result["passed"]) + self.assertEqual(result["findings"][0]["pattern"], "tracked_runtime_env") + + def test_json_output_is_machine_readable_and_redacted(self) -> None: + output = io.StringIO() + + with contextlib.redirect_stdout(output): + exit_code = run_secret_static_checks.main(env={"SECRET_CHECKS_OUTPUT_JSON": "1"}) + + summary = json.loads(output.getvalue()) + self.assertEqual(exit_code, 0) + self.assertTrue(summary["passed"]) + self.assertIn("checked_file_count", summary) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_trading_delivery_readiness.py b/tests/unit/test_trading_delivery_readiness.py index e0b3ab3..1314b54 100644 --- a/tests/unit/test_trading_delivery_readiness.py +++ b/tests/unit/test_trading_delivery_readiness.py @@ -21,6 +21,7 @@ EXPECTED_COMMAND_GROUPS = ( '.venv/bin/python -m unittest discover -s tests/unit -p "test_*.py"', ".venv/bin/python scripts/run_offline_checks.py", ".venv/bin/python scripts/check_local_demo_readiness.py", + ".venv/bin/python scripts/run_secret_static_checks.py", ".venv/bin/python scripts/check_trading_delivery_readiness.py", ), ), @@ -60,7 +61,7 @@ EXPECTED_COMMAND_GROUPS = ( "git rev-parse HEAD", "docker compose -p ai-ict -f /root/deploy/ai-ict/docker-compose.yml logs --tail=100 ai-ict", "git worktree add /tmp/ai-ict-rollback ", - "cd /tmp/ai-ict-rollback && ./scripts/deploy_server.exp", + "cd /tmp/ai-ict-rollback && AI_ICT_DEPLOY_PASSWORD= AI_ICT_DEPLOY_USE_SUDO=1 .venv/bin/python scripts/deploy_with_password.py --host 192.168.3.90 --user yuqei --port 22 --remote-path /home/yuqei/project/ai-exchange --remote-archive /home/yuqei/ai-ict-deploy.tgz", "rm -rf /tmp/ai-ict-rollback", ), ),