在前后端开发联调前 的提交20260223
This commit is contained in:
@@ -845,7 +845,10 @@ def main():
|
||||
md_content = generate_report(report, coupling)
|
||||
|
||||
# 确定输出路径
|
||||
reports_dir = root / "docs" / "reports"
|
||||
_report_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not _report_root:
|
||||
raise KeyError("环境变量 ETL_REPORT_ROOT 未定义。请在根 .env 中配置。")
|
||||
reports_dir = Path(_report_root)
|
||||
reports_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
if args.output:
|
||||
|
||||
@@ -31,7 +31,10 @@ from typing import Any
|
||||
_SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
_FEIQIU_ROOT = _SCRIPT_DIR.parent.parent # apps/etl/connectors/feiqiu
|
||||
_OUTPUT_DIR = _SCRIPT_DIR / "output"
|
||||
_REPORTS_DIR = _FEIQIU_ROOT / "docs" / "reports"
|
||||
_etl_report_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not _etl_report_root:
|
||||
raise KeyError("环境变量 ETL_REPORT_ROOT 未定义。请在根 .env 中配置。")
|
||||
_REPORTS_DIR = Path(_etl_report_root)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -17,6 +17,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
@@ -1148,7 +1149,10 @@ def run_blackbox_check(
|
||||
logger.info("JSON 报告: %s", json_path)
|
||||
|
||||
# 输出 Markdown
|
||||
reports_dir = _FEIQIU_ROOT / "docs" / "reports"
|
||||
_report_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not _report_root:
|
||||
raise KeyError("环境变量 ETL_REPORT_ROOT 未定义。请在根 .env 中配置。")
|
||||
reports_dir = Path(_report_root)
|
||||
reports_dir.mkdir(parents=True, exist_ok=True)
|
||||
md_path = reports_dir / f"blackbox_report_{ts}.md"
|
||||
md_content = _generate_markdown_report(report)
|
||||
|
||||
@@ -170,7 +170,7 @@ def check_invalid_flow_rejection() -> DebugResult:
|
||||
config=_stub_config, task_executor=None, task_registry=None,
|
||||
db_conn=None, api_client=None, logger=logging.getLogger("test"),
|
||||
)
|
||||
runner.run(pipeline=name)
|
||||
runner.run(flow=name)
|
||||
errors_missed.append(name)
|
||||
except ValueError as exc:
|
||||
errors_raised.append({"name": name, "error": str(exc)})
|
||||
@@ -738,8 +738,8 @@ def check_cli_mode_detection() -> DebugResult:
|
||||
"""验证 CLI 的 Flow 模式 vs 传统模式判断逻辑。
|
||||
|
||||
通过检查 main() 源码确认:
|
||||
- 有 --pipeline 参数 → Flow 模式(使用 FlowRunner)
|
||||
- 无 --pipeline 参数 → 传统模式(使用 TaskExecutor.run_tasks)
|
||||
- 有 --flow 参数 → Flow 模式(使用 FlowRunner)
|
||||
- 无 --flow 参数 → 传统模式(使用 TaskExecutor.run_tasks)
|
||||
"""
|
||||
import inspect
|
||||
from cli.main import main as cli_main
|
||||
@@ -753,14 +753,14 @@ def check_cli_mode_detection() -> DebugResult:
|
||||
source = inspect.getsource(cli_main)
|
||||
|
||||
# 检查 Flow 模式分支
|
||||
if "args.pipeline" in source and "FlowRunner" in source:
|
||||
checks.append("✓ 有 --pipeline 参数时使用 FlowRunner(Flow 模式)")
|
||||
if "args.flow" in source and "FlowRunner" in source:
|
||||
checks.append("✓ 有 --flow 参数时使用 FlowRunner(Flow 模式)")
|
||||
else:
|
||||
issues.append("未找到 Flow 模式分支(args.pipeline + FlowRunner)")
|
||||
issues.append("未找到 Flow 模式分支(args.flow + FlowRunner)")
|
||||
|
||||
# 检查传统模式分支
|
||||
if "run_tasks" in source:
|
||||
checks.append("✓ 无 --pipeline 参数时使用 run_tasks(传统模式)")
|
||||
checks.append("✓ 无 --flow 参数时使用 run_tasks(传统模式)")
|
||||
else:
|
||||
issues.append("未找到传统模式分支(run_tasks)")
|
||||
|
||||
@@ -814,13 +814,13 @@ def check_cli_flow_choices() -> DebugResult:
|
||||
cli_choices = set(FlowRunner.FLOW_LAYERS.keys())
|
||||
issues.append("无法导入 FLOW_CHOICES,使用 FLOW_LAYERS 键集合")
|
||||
|
||||
pipeline_keys = set(FlowRunner.FLOW_LAYERS.keys())
|
||||
flow_keys = set(FlowRunner.FLOW_LAYERS.keys())
|
||||
|
||||
missing_in_cli = pipeline_keys - cli_choices
|
||||
extra_in_cli = cli_choices - pipeline_keys
|
||||
missing_in_cli = flow_keys - cli_choices
|
||||
extra_in_cli = cli_choices - flow_keys
|
||||
|
||||
result.details = {
|
||||
"flow_layers_keys": sorted(pipeline_keys),
|
||||
"flow_layers_keys": sorted(flow_keys),
|
||||
"cli_choices": sorted(cli_choices),
|
||||
"missing_in_cli": sorted(missing_in_cli),
|
||||
"extra_in_cli": sorted(extra_in_cli),
|
||||
@@ -836,7 +836,7 @@ def check_cli_flow_choices() -> DebugResult:
|
||||
result.message = "; ".join(issues)
|
||||
else:
|
||||
result.status = "PASS"
|
||||
result.message = f"CLI --flow 可选值与 FLOW_LAYERS 完全一致 ({len(pipeline_keys)} 种)"
|
||||
result.message = f"CLI --flow 可选值与 FLOW_LAYERS 完全一致 ({len(flow_keys)} 种)"
|
||||
|
||||
result.duration_sec = round(time.monotonic() - t0, 4)
|
||||
return result
|
||||
@@ -847,7 +847,7 @@ def check_cli_flow_choices() -> DebugResult:
|
||||
# ══════════════════════════════════════════════════════════════
|
||||
|
||||
def check_processing_modes() -> DebugResult:
|
||||
"""验证 FlowRunner.run() 对三种处理模式的分支逻辑。
|
||||
"""验证 FlowRunner.run() 对四种处理模式的分支逻辑。
|
||||
|
||||
- increment_only: 仅执行增量 ETL
|
||||
- verify_only: 跳过增量 ETL,直接执行校验
|
||||
@@ -899,7 +899,7 @@ def check_processing_modes() -> DebugResult:
|
||||
result.message = f"处理模式验证有 {len(issues)} 个问题"
|
||||
else:
|
||||
result.status = "PASS"
|
||||
result.message = "三种处理模式(increment_only/verify_only/increment_verify)逻辑正确"
|
||||
result.message = "四种处理模式(increment_only/verify_only/increment_verify/full_window)逻辑正确"
|
||||
|
||||
result.duration_sec = round(time.monotonic() - t0, 4)
|
||||
return result
|
||||
|
||||
@@ -6,11 +6,11 @@ Debug 报告生成脚本 —— 汇总所有阶段的调试结果,生成结构
|
||||
- 阶段1: 属性测试结果(pytest 执行)
|
||||
- 阶段2: 全量刷新 JSON(scripts/debug/output/full_refresh_*.json)
|
||||
- 阶段3: 黑盒校验 JSON(scripts/debug/output/blackbox_*.json)
|
||||
- 阶段4: 架构分析报告(docs/reports/architecture_report_*.md)
|
||||
- 阶段5: 性能分析报告(docs/reports/performance_report_*.md)
|
||||
- 阶段4: 架构分析报告($ETL_REPORT_ROOT/architecture_report_*.md)
|
||||
- 阶段5: 性能分析报告($ETL_REPORT_ROOT/performance_report_*.md)
|
||||
|
||||
输出:
|
||||
docs/reports/debug_report_YYYYMMDD.md
|
||||
$ETL_REPORT_ROOT/debug_report_YYYYMMDD.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -18,6 +18,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
@@ -25,13 +26,26 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# 加载根 .env
|
||||
load_dotenv(Path(__file__).resolve().parents[5] / ".env", override=False)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 路径常量
|
||||
# ---------------------------------------------------------------------------
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
ETL_ROOT = SCRIPT_DIR.parent.parent # apps/etl/connectors/feiqiu
|
||||
OUTPUT_DIR = SCRIPT_DIR / "output"
|
||||
REPORTS_DIR = ETL_ROOT / "docs" / "reports"
|
||||
|
||||
_report_root = os.environ.get("ETL_REPORT_ROOT")
|
||||
if not _report_root:
|
||||
raise KeyError(
|
||||
"环境变量 ETL_REPORT_ROOT 未定义。"
|
||||
"请在根 .env 中配置,参考 .env.template 和 docs/deployment/EXPORT-PATHS.md"
|
||||
)
|
||||
REPORTS_DIR = Path(_report_root)
|
||||
|
||||
TESTS_DIR = ETL_ROOT / "tests" / "unit"
|
||||
|
||||
# 属性测试文件
|
||||
|
||||
Reference in New Issue
Block a user