root/Neo-ZQYY
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

在准备环境前提交次全部更改。

Browse Source
This commit is contained in:
Neo
2026-02-19 08:35:13 +08:00
parent ded6dfb9d8
commit 4eac07da47
1387 changed files with 6107191 additions and 33002 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1945
apps/etl/connectors/feiqiu/scripts/debug/output/blackbox_20260216_023922.json Normal file
View File

File diff suppressed because it is too large Load Diff

608
apps/etl/connectors/feiqiu/scripts/debug/output/debug_orchestration_20260216_014001.json Normal file
View File

@@ -0,0 +1,608 @@
[
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_DEFINITIONS",
"status": "PASS",
"message": "全部 7 种 Flow 定义完整",
"details": {
"expected": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"actual": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"missing": [],
"extra": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_LAYER_MAPPING",
"status": "PASS",
"message": "所有 Flow 层映射正确",
"details": {
"total_flows": 7,
"mismatches": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "INVALID_FLOW_REJECTION",
"status": "FAIL",
"message": "以下无效 Flow 未被拒绝: ['nonexistent', 'API_ODS', 'full', '', 'api_full_extra']",
"details": {
"tested": [
"nonexistent",
"API_ODS",
"full",
"",
"api_full_extra"
],
"correctly_rejected": [],
"missed": [
"nonexistent",
"API_ODS",
"full",
"",
"api_full_extra"
]
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_TASK_RESOLUTION",
"status": "PASS",
"message": "所有 7 种 Flow 任务解析正确",
"details": {
"flow_tasks": {
"api_ods": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET"
],
"api_ods_dwd": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"DWD_LOAD_FROM_ODS"
],
"api_full": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"DWD_LOAD_FROM_ODS",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
],
"ods_dwd": [
"DWD_LOAD_FROM_ODS"
],
"dwd_dws": [
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY"
],
"dwd_dws_index": [
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
],
"dwd_index": [
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
]
},
"issues": []
},
"duration_sec": 0.0016,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "TASK_REGISTRY_LAYERS",
"status": "PASS",
"message": "各层任务数量正确 (ODS=23, DWD=2, DWS=15, INDEX=4)",
"details": {
"ODS": {
"expected": 23,
"actual": 23,
"tasks": [
"ODS_ASSISTANT_ABOLISH",
"ODS_ASSISTANT_ACCOUNT",
"ODS_ASSISTANT_LEDGER",
"ODS_GOODS_CATEGORY",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_GROUP_PACKAGE",
"ODS_INVENTORY_CHANGE",
"ODS_INVENTORY_STOCK",
"ODS_MEMBER",
"ODS_MEMBER_BALANCE",
"ODS_MEMBER_CARD",
"ODS_PAYMENT",
"ODS_PLATFORM_COUPON",
"ODS_RECHARGE_SETTLE",
"ODS_REFUND",
"ODS_SETTLEMENT_RECORDS",
"ODS_SETTLEMENT_TICKET",
"ODS_STORE_GOODS",
"ODS_STORE_GOODS_SALES",
"ODS_TABLES",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TABLE_USE",
"ODS_TENANT_GOODS"
]
},
"DWD": {
"expected": 2,
"actual": 2,
"tasks": [
"DWD_LOAD_FROM_ODS",
"DWD_QUALITY_CHECK"
]
},
"DWS": {
"expected": 15,
"actual": 15,
"tasks": [
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_FINANCE",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_SALARY",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_RECHARGE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_RETENTION_CLEANUP"
]
},
"INDEX": {
"expected": 4,
"actual": 4,
"tasks": [
"DWS_ML_MANUAL_IMPORT",
"DWS_NEWCONV_INDEX",
"DWS_RELATION_INDEX",
"DWS_WINBACK_INDEX"
]
},
"TOTAL": {
"actual": 52
}
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "UTILITY_TASK_IDENTIFICATION",
"status": "PASS",
"message": "工具类任务识别正确 (6 个工具类, 6 个 ETL 类)",
"details": {
"utility_tasks": {
"MANUAL_INGEST": true,
"INIT_ODS_SCHEMA": true,
"INIT_DWD_SCHEMA": true,
"INIT_DWS_SCHEMA": true,
"ODS_JSON_ARCHIVE": true,
"CHECK_CUTOFF": true
},
"etl_tasks": {
"ODS_MEMBER": false,
"ODS_ORDER": false,
"ODS_PAYMENT": false,
"DWD_LOAD_FROM_ODS": false,
"DWS_ASSISTANT_DAILY": false,
"DWS_FINANCE_DAILY": false
},
"index_tasks_utility_status": {
"DWS_WINBACK_INDEX": true,
"DWS_NEWCONV_INDEX": true,
"DWS_ML_MANUAL_IMPORT": true,
"DWS_RELATION_INDEX": true
},
"issues": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "TASK_DISPATCH_PATHS",
"status": "PASS",
"message": "任务分发路径正确 (utility=13, ods=23, standard=16)",
"details": {
"path_counts": {
"utility": 13,
"standard": 16,
"ods": 23
},
"issues": [],
"sample_dispatch": {
"CHECK_CUTOFF": {
"layer": null,
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DATA_INTEGRITY_CHECK": {
"layer": null,
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DWD_LOAD_FROM_ODS": {
"layer": "DWD",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWD_QUALITY_CHECK": {
"layer": "DWD",
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DWS_ASSISTANT_CUSTOMER": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_DAILY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_FINANCE": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_MONTHLY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_SALARY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_BUILD_ORDER_SUMMARY": {
"layer": "DWS",
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
}
}
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_MANAGER_INTERFACE",
"status": "PASS",
"message": "CursorManager 接口签名正确 (get_or_create, advance)",
"details": {
"method_signatures": {
"get_or_create": [
"self",
"task_id",
"store_id"
],
"advance": [
"self",
"task_id",
"store_id",
"window_start",
"window_end",
"run_id",
"last_id"
]
},
"issues": []
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_ADVANCE_SQL",
"status": "PASS",
"message": "游标推进 SQL 逻辑正确",
"details": {
"checks": [
"✓ 使用 UPDATE meta.etl_cursor",
"✓ 使用 GREATEST 保护 last_id 不回退",
"✓ 调用 commit() 持久化",
"✓ last_id 参数可选（有 None 分支）",
"✓ 更新 updated_at 时间戳"
],
"issues": []
},
"duration_sec": 0.0006,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_SKIP_UTILITY",
"status": "FAIL",
"message": "工具类任务游标跳过逻辑有问题: _run_utility_task 中出现了 cursor 相关调用",
"details": {
"checks": [
"✓ run_single_task 检查 is_utility_task 并分发到 _run_utility_task",
"✓ _run_utility_task 不调用 run_tracker.create_run"
],
"issues": [
"_run_utility_task 中出现了 cursor 相关调用"
]
},
"duration_sec": 0.0018,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_DATA_SOURCE",
"status": "PASS",
"message": "全部 7 个 data_source 解析用例通过",
"details": {
"test_cases": [
{
"case": "默认值",
"expected": "hybrid",
"actual": "hybrid"
},
{
"case": "--data-source online",
"expected": "online",
"actual": "online"
},
{
"case": "--data-source offline",
"expected": "offline",
"actual": "offline"
},
{
"case": "--pipeline-flow FULL",
"expected": "hybrid",
"actual": "hybrid",
"deprecation_warning": true
},
{
"case": "--pipeline-flow FETCH_ONLY",
"expected": "online",
"actual": "online"
},
{
"case": "--pipeline-flow INGEST_ONLY",
"expected": "offline",
"actual": "offline"
},
{
"case": "--data-source online + --pipeline-flow INGEST_ONLY",
"expected": "online",
"actual": "online"
}
],
"issues": []
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_MODE_DETECTION",
"status": "PASS",
"message": "CLI Flow/传统模式检测逻辑正确",
"details": {
"checks": [
"✓ 有 --pipeline 参数时使用 PipelineRunner（Flow 模式）",
"✓ 无 --pipeline 参数时使用 run_tasks（传统模式）",
"✓ 调用 resolve_data_source 解析数据源模式",
"✓ 支持 --lookback-hours 回溯窗口",
"✓ 设置 window_override 确保任务使用指定窗口"
],
"issues": []
},
"duration_sec": 0.0015,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_PIPELINE_CHOICES",
"status": "PASS",
"message": "CLI --pipeline 可选值与 PIPELINE_LAYERS 完全一致 (7 种)",
"details": {
"pipeline_layers_keys": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"cli_choices": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"missing_in_cli": [],
"extra_in_cli": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "PROCESSING_MODES",
"status": "PASS",
"message": "三种处理模式（increment_only/verify_only/increment_verify）逻辑正确",
"details": {
"checks": [
"✓ 支持 verify_only 模式",
"✓ verify_only 调用 _run_verification",
"✓ 支持 increment_verify 模式",
"✓ 支持 fetch_before_verify 参数（校验前先获取 API 数据）",
"✓ _run_verification 方法存在"
],
"issues": []
},
"duration_sec": 0.0012,
"error_detail": null,
"fix_applied": null
}
]

607
apps/etl/connectors/feiqiu/scripts/debug/output/debug_orchestration_20260216_014157.json Normal file
View File

@@ -0,0 +1,607 @@
[
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_DEFINITIONS",
"status": "PASS",
"message": "全部 7 种 Flow 定义完整",
"details": {
"expected": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"actual": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"missing": [],
"extra": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_LAYER_MAPPING",
"status": "PASS",
"message": "所有 Flow 层映射正确",
"details": {
"total_flows": 7,
"mismatches": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "INVALID_FLOW_REJECTION",
"status": "PASS",
"message": "全部 5 个无效 Flow 名称被正确拒绝",
"details": {
"tested": [
"nonexistent",
"API_ODS",
"full",
"",
"api_full_extra"
],
"correctly_rejected": [
"nonexistent",
"API_ODS",
"full",
"",
"api_full_extra"
],
"missed": []
},
"duration_sec": 0.0008,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_TASK_RESOLUTION",
"status": "PASS",
"message": "所有 7 种 Flow 任务解析正确",
"details": {
"flow_tasks": {
"api_ods": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET"
],
"api_ods_dwd": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"DWD_LOAD_FROM_ODS"
],
"api_full": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"DWD_LOAD_FROM_ODS",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
],
"ods_dwd": [
"DWD_LOAD_FROM_ODS"
],
"dwd_dws": [
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY"
],
"dwd_dws_index": [
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
],
"dwd_index": [
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
]
},
"issues": []
},
"duration_sec": 0.002,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "TASK_REGISTRY_LAYERS",
"status": "PASS",
"message": "各层任务数量正确 (ODS=23, DWD=2, DWS=15, INDEX=4)",
"details": {
"ODS": {
"expected": 23,
"actual": 23,
"tasks": [
"ODS_ASSISTANT_ABOLISH",
"ODS_ASSISTANT_ACCOUNT",
"ODS_ASSISTANT_LEDGER",
"ODS_GOODS_CATEGORY",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_GROUP_PACKAGE",
"ODS_INVENTORY_CHANGE",
"ODS_INVENTORY_STOCK",
"ODS_MEMBER",
"ODS_MEMBER_BALANCE",
"ODS_MEMBER_CARD",
"ODS_PAYMENT",
"ODS_PLATFORM_COUPON",
"ODS_RECHARGE_SETTLE",
"ODS_REFUND",
"ODS_SETTLEMENT_RECORDS",
"ODS_SETTLEMENT_TICKET",
"ODS_STORE_GOODS",
"ODS_STORE_GOODS_SALES",
"ODS_TABLES",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TABLE_USE",
"ODS_TENANT_GOODS"
]
},
"DWD": {
"expected": 2,
"actual": 2,
"tasks": [
"DWD_LOAD_FROM_ODS",
"DWD_QUALITY_CHECK"
]
},
"DWS": {
"expected": 15,
"actual": 15,
"tasks": [
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_FINANCE",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_SALARY",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_RECHARGE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_RETENTION_CLEANUP"
]
},
"INDEX": {
"expected": 4,
"actual": 4,
"tasks": [
"DWS_ML_MANUAL_IMPORT",
"DWS_NEWCONV_INDEX",
"DWS_RELATION_INDEX",
"DWS_WINBACK_INDEX"
]
},
"TOTAL": {
"actual": 52
}
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "UTILITY_TASK_IDENTIFICATION",
"status": "PASS",
"message": "工具类任务识别正确 (6 个工具类, 6 个 ETL 类)",
"details": {
"utility_tasks": {
"MANUAL_INGEST": true,
"INIT_ODS_SCHEMA": true,
"INIT_DWD_SCHEMA": true,
"INIT_DWS_SCHEMA": true,
"ODS_JSON_ARCHIVE": true,
"CHECK_CUTOFF": true
},
"etl_tasks": {
"ODS_MEMBER": false,
"ODS_ORDER": false,
"ODS_PAYMENT": false,
"DWD_LOAD_FROM_ODS": false,
"DWS_ASSISTANT_DAILY": false,
"DWS_FINANCE_DAILY": false
},
"index_tasks_utility_status": {
"DWS_WINBACK_INDEX": true,
"DWS_NEWCONV_INDEX": true,
"DWS_ML_MANUAL_IMPORT": true,
"DWS_RELATION_INDEX": true
},
"issues": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "TASK_DISPATCH_PATHS",
"status": "PASS",
"message": "任务分发路径正确 (utility=13, ods=23, standard=16)",
"details": {
"path_counts": {
"utility": 13,
"standard": 16,
"ods": 23
},
"issues": [],
"sample_dispatch": {
"CHECK_CUTOFF": {
"layer": null,
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DATA_INTEGRITY_CHECK": {
"layer": null,
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DWD_LOAD_FROM_ODS": {
"layer": "DWD",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWD_QUALITY_CHECK": {
"layer": "DWD",
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DWS_ASSISTANT_CUSTOMER": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_DAILY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_FINANCE": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_MONTHLY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_SALARY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_BUILD_ORDER_SUMMARY": {
"layer": "DWS",
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
}
}
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_MANAGER_INTERFACE",
"status": "PASS",
"message": "CursorManager 接口签名正确 (get_or_create, advance)",
"details": {
"method_signatures": {
"get_or_create": [
"self",
"task_id",
"store_id"
],
"advance": [
"self",
"task_id",
"store_id",
"window_start",
"window_end",
"run_id",
"last_id"
]
},
"issues": []
},
"duration_sec": 0.0002,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_ADVANCE_SQL",
"status": "PASS",
"message": "游标推进 SQL 逻辑正确",
"details": {
"checks": [
"✓ 使用 UPDATE meta.etl_cursor",
"✓ 使用 GREATEST 保护 last_id 不回退",
"✓ 调用 commit() 持久化",
"✓ last_id 参数可选（有 None 分支）",
"✓ 更新 updated_at 时间戳"
],
"issues": []
},
"duration_sec": 0.0005,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_SKIP_UTILITY",
"status": "PASS",
"message": "工具类任务正确跳过游标管理和运行记录",
"details": {
"checks": [
"✓ run_single_task 检查 is_utility_task 并分发到 _run_utility_task",
"✓ _run_utility_task 不调用 cursor_mgr",
"✓ _run_utility_task 不调用 run_tracker.create_run"
],
"issues": []
},
"duration_sec": 0.0019,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_DATA_SOURCE",
"status": "PASS",
"message": "全部 7 个 data_source 解析用例通过",
"details": {
"test_cases": [
{
"case": "默认值",
"expected": "hybrid",
"actual": "hybrid"
},
{
"case": "--data-source online",
"expected": "online",
"actual": "online"
},
{
"case": "--data-source offline",
"expected": "offline",
"actual": "offline"
},
{
"case": "--pipeline-flow FULL",
"expected": "hybrid",
"actual": "hybrid",
"deprecation_warning": true
},
{
"case": "--pipeline-flow FETCH_ONLY",
"expected": "online",
"actual": "online"
},
{
"case": "--pipeline-flow INGEST_ONLY",
"expected": "offline",
"actual": "offline"
},
{
"case": "--data-source online + --pipeline-flow INGEST_ONLY",
"expected": "online",
"actual": "online"
}
],
"issues": []
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_MODE_DETECTION",
"status": "PASS",
"message": "CLI Flow/传统模式检测逻辑正确",
"details": {
"checks": [
"✓ 有 --pipeline 参数时使用 PipelineRunner（Flow 模式）",
"✓ 无 --pipeline 参数时使用 run_tasks（传统模式）",
"✓ 调用 resolve_data_source 解析数据源模式",
"✓ 支持 --lookback-hours 回溯窗口",
"✓ 设置 window_override 确保任务使用指定窗口"
],
"issues": []
},
"duration_sec": 0.0011,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_PIPELINE_CHOICES",
"status": "PASS",
"message": "CLI --pipeline 可选值与 PIPELINE_LAYERS 完全一致 (7 种)",
"details": {
"pipeline_layers_keys": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"cli_choices": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"missing_in_cli": [],
"extra_in_cli": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "PROCESSING_MODES",
"status": "PASS",
"message": "三种处理模式（increment_only/verify_only/increment_verify）逻辑正确",
"details": {
"checks": [
"✓ 支持 verify_only 模式",
"✓ verify_only 调用 _run_verification",
"✓ 支持 increment_verify 模式",
"✓ 支持 fetch_before_verify 参数（校验前先获取 API 数据）",
"✓ _run_verification 方法存在"
],
"issues": []
},
"duration_sec": 0.0019,
"error_detail": null,
"fix_applied": null
}
]

607
apps/etl/connectors/feiqiu/scripts/debug/output/debug_orchestration_20260216_014221.json Normal file
View File

@@ -0,0 +1,607 @@
[
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_DEFINITIONS",
"status": "PASS",
"message": "全部 7 种 Flow 定义完整",
"details": {
"expected": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"actual": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"missing": [],
"extra": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_LAYER_MAPPING",
"status": "PASS",
"message": "所有 Flow 层映射正确",
"details": {
"total_flows": 7,
"mismatches": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "INVALID_FLOW_REJECTION",
"status": "PASS",
"message": "全部 5 个无效 Flow 名称被正确拒绝",
"details": {
"tested": [
"nonexistent",
"API_ODS",
"full",
"",
"api_full_extra"
],
"correctly_rejected": [
"nonexistent",
"API_ODS",
"full",
"",
"api_full_extra"
],
"missed": []
},
"duration_sec": 0.0008,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "FLOW_TASK_RESOLUTION",
"status": "PASS",
"message": "所有 7 种 Flow 任务解析正确",
"details": {
"flow_tasks": {
"api_ods": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET"
],
"api_ods_dwd": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"DWD_LOAD_FROM_ODS"
],
"api_full": [
"ODS_ASSISTANT_ACCOUNT",
"ODS_SETTLEMENT_RECORDS",
"ODS_TABLE_USE",
"ODS_ASSISTANT_LEDGER",
"ODS_ASSISTANT_ABOLISH",
"ODS_STORE_GOODS_SALES",
"ODS_PAYMENT",
"ODS_REFUND",
"ODS_PLATFORM_COUPON",
"ODS_MEMBER",
"ODS_MEMBER_CARD",
"ODS_MEMBER_BALANCE",
"ODS_RECHARGE_SETTLE",
"ODS_GROUP_PACKAGE",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_INVENTORY_STOCK",
"ODS_INVENTORY_CHANGE",
"ODS_TABLES",
"ODS_GOODS_CATEGORY",
"ODS_STORE_GOODS",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TENANT_GOODS",
"ODS_SETTLEMENT_TICKET",
"DWD_LOAD_FROM_ODS",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
],
"ods_dwd": [
"DWD_LOAD_FROM_ODS"
],
"dwd_dws": [
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY"
],
"dwd_dws_index": [
"DWS_BUILD_ORDER_SUMMARY",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_SALARY",
"DWS_ASSISTANT_FINANCE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_RECHARGE",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_RETENTION_CLEANUP",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
],
"dwd_index": [
"DWS_WINBACK_INDEX",
"DWS_NEWCONV_INDEX",
"DWS_ML_MANUAL_IMPORT",
"DWS_RELATION_INDEX"
]
},
"issues": []
},
"duration_sec": 0.0012,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "TASK_REGISTRY_LAYERS",
"status": "PASS",
"message": "各层任务数量正确 (ODS=23, DWD=2, DWS=15, INDEX=4)",
"details": {
"ODS": {
"expected": 23,
"actual": 23,
"tasks": [
"ODS_ASSISTANT_ABOLISH",
"ODS_ASSISTANT_ACCOUNT",
"ODS_ASSISTANT_LEDGER",
"ODS_GOODS_CATEGORY",
"ODS_GROUP_BUY_REDEMPTION",
"ODS_GROUP_PACKAGE",
"ODS_INVENTORY_CHANGE",
"ODS_INVENTORY_STOCK",
"ODS_MEMBER",
"ODS_MEMBER_BALANCE",
"ODS_MEMBER_CARD",
"ODS_PAYMENT",
"ODS_PLATFORM_COUPON",
"ODS_RECHARGE_SETTLE",
"ODS_REFUND",
"ODS_SETTLEMENT_RECORDS",
"ODS_SETTLEMENT_TICKET",
"ODS_STORE_GOODS",
"ODS_STORE_GOODS_SALES",
"ODS_TABLES",
"ODS_TABLE_FEE_DISCOUNT",
"ODS_TABLE_USE",
"ODS_TENANT_GOODS"
]
},
"DWD": {
"expected": 2,
"actual": 2,
"tasks": [
"DWD_LOAD_FROM_ODS",
"DWD_QUALITY_CHECK"
]
},
"DWS": {
"expected": 15,
"actual": 15,
"tasks": [
"DWS_ASSISTANT_CUSTOMER",
"DWS_ASSISTANT_DAILY",
"DWS_ASSISTANT_FINANCE",
"DWS_ASSISTANT_MONTHLY",
"DWS_ASSISTANT_SALARY",
"DWS_BUILD_ORDER_SUMMARY",
"DWS_FINANCE_DAILY",
"DWS_FINANCE_DISCOUNT_DETAIL",
"DWS_FINANCE_INCOME_STRUCTURE",
"DWS_FINANCE_RECHARGE",
"DWS_MEMBER_CONSUMPTION",
"DWS_MEMBER_VISIT",
"DWS_MV_REFRESH_ASSISTANT_DAILY",
"DWS_MV_REFRESH_FINANCE_DAILY",
"DWS_RETENTION_CLEANUP"
]
},
"INDEX": {
"expected": 4,
"actual": 4,
"tasks": [
"DWS_ML_MANUAL_IMPORT",
"DWS_NEWCONV_INDEX",
"DWS_RELATION_INDEX",
"DWS_WINBACK_INDEX"
]
},
"TOTAL": {
"actual": 52
}
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "UTILITY_TASK_IDENTIFICATION",
"status": "PASS",
"message": "工具类任务识别正确 (6 个工具类, 6 个 ETL 类)",
"details": {
"utility_tasks": {
"MANUAL_INGEST": true,
"INIT_ODS_SCHEMA": true,
"INIT_DWD_SCHEMA": true,
"INIT_DWS_SCHEMA": true,
"ODS_JSON_ARCHIVE": true,
"CHECK_CUTOFF": true
},
"etl_tasks": {
"ODS_MEMBER": false,
"ODS_ORDER": false,
"ODS_PAYMENT": false,
"DWD_LOAD_FROM_ODS": false,
"DWS_ASSISTANT_DAILY": false,
"DWS_FINANCE_DAILY": false
},
"index_tasks_utility_status": {
"DWS_WINBACK_INDEX": true,
"DWS_NEWCONV_INDEX": true,
"DWS_ML_MANUAL_IMPORT": true,
"DWS_RELATION_INDEX": true
},
"issues": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "TASK_DISPATCH_PATHS",
"status": "PASS",
"message": "任务分发路径正确 (utility=13, ods=23, standard=16)",
"details": {
"path_counts": {
"utility": 13,
"standard": 16,
"ods": 23
},
"issues": [],
"sample_dispatch": {
"CHECK_CUTOFF": {
"layer": null,
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DATA_INTEGRITY_CHECK": {
"layer": null,
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DWD_LOAD_FROM_ODS": {
"layer": "DWD",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWD_QUALITY_CHECK": {
"layer": "DWD",
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
},
"DWS_ASSISTANT_CUSTOMER": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_DAILY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_FINANCE": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_MONTHLY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_ASSISTANT_SALARY": {
"layer": "DWS",
"is_utility": false,
"is_ods": false,
"dispatch_path": "standard"
},
"DWS_BUILD_ORDER_SUMMARY": {
"layer": "DWS",
"is_utility": true,
"is_ods": false,
"dispatch_path": "utility"
}
}
},
"duration_sec": 0.0001,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_MANAGER_INTERFACE",
"status": "PASS",
"message": "CursorManager 接口签名正确 (get_or_create, advance)",
"details": {
"method_signatures": {
"get_or_create": [
"self",
"task_id",
"store_id"
],
"advance": [
"self",
"task_id",
"store_id",
"window_start",
"window_end",
"run_id",
"last_id"
]
},
"issues": []
},
"duration_sec": 0.0002,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_ADVANCE_SQL",
"status": "PASS",
"message": "游标推进 SQL 逻辑正确",
"details": {
"checks": [
"✓ 使用 UPDATE meta.etl_cursor",
"✓ 使用 GREATEST 保护 last_id 不回退",
"✓ 调用 commit() 持久化",
"✓ last_id 参数可选（有 None 分支）",
"✓ 更新 updated_at 时间戳"
],
"issues": []
},
"duration_sec": 0.0005,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CURSOR_SKIP_UTILITY",
"status": "PASS",
"message": "工具类任务正确跳过游标管理和运行记录",
"details": {
"checks": [
"✓ run_single_task 检查 is_utility_task 并分发到 _run_utility_task",
"✓ _run_utility_task 不调用 cursor_mgr",
"✓ _run_utility_task 不调用 run_tracker.create_run"
],
"issues": []
},
"duration_sec": 0.002,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_DATA_SOURCE",
"status": "PASS",
"message": "全部 7 个 data_source 解析用例通过",
"details": {
"test_cases": [
{
"case": "默认值",
"expected": "hybrid",
"actual": "hybrid"
},
{
"case": "--data-source online",
"expected": "online",
"actual": "online"
},
{
"case": "--data-source offline",
"expected": "offline",
"actual": "offline"
},
{
"case": "--pipeline-flow FULL",
"expected": "hybrid",
"actual": "hybrid",
"deprecation_warning": true
},
{
"case": "--pipeline-flow FETCH_ONLY",
"expected": "online",
"actual": "online"
},
{
"case": "--pipeline-flow INGEST_ONLY",
"expected": "offline",
"actual": "offline"
},
{
"case": "--data-source online + --pipeline-flow INGEST_ONLY",
"expected": "online",
"actual": "online"
}
],
"issues": []
},
"duration_sec": 0.0002,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_MODE_DETECTION",
"status": "PASS",
"message": "CLI Flow/传统模式检测逻辑正确",
"details": {
"checks": [
"✓ 有 --pipeline 参数时使用 PipelineRunner（Flow 模式）",
"✓ 无 --pipeline 参数时使用 run_tasks（传统模式）",
"✓ 调用 resolve_data_source 解析数据源模式",
"✓ 支持 --lookback-hours 回溯窗口",
"✓ 设置 window_override 确保任务使用指定窗口"
],
"issues": []
},
"duration_sec": 0.0017,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "CLI_PIPELINE_CHOICES",
"status": "PASS",
"message": "CLI --pipeline 可选值与 PIPELINE_LAYERS 完全一致 (7 种)",
"details": {
"pipeline_layers_keys": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"cli_choices": [
"api_full",
"api_ods",
"api_ods_dwd",
"dwd_dws",
"dwd_dws_index",
"dwd_index",
"ods_dwd"
],
"missing_in_cli": [],
"extra_in_cli": []
},
"duration_sec": 0.0,
"error_detail": null,
"fix_applied": null
},
{
"layer": "ORCHESTRATION",
"task_code": "PROCESSING_MODES",
"status": "PASS",
"message": "三种处理模式（increment_only/verify_only/increment_verify）逻辑正确",
"details": {
"checks": [
"✓ 支持 verify_only 模式",
"✓ verify_only 调用 _run_verification",
"✓ 支持 increment_verify 模式",
"✓ 支持 fetch_before_verify 参数（校验前先获取 API 数据）",
"✓ _run_verification 方法存在"
],
"issues": []
},
"duration_sec": 0.0014,
"error_detail": null,
"fix_applied": null
}
]

954
apps/etl/connectors/feiqiu/scripts/debug/output/full_refresh_20260216_020131.json Normal file
View File

@@ -0,0 +1,954 @@
{
"flow": "api_full",
"window_start": "2026-01-01T00:00:00",
"window_end": "2026-02-16T00:00:00",
"overall_start": "2026-02-16T02:00:29.172416+08:00",
"overall_end": "2026-02-16T02:01:31.080429+08:00",
"overall_duration_sec": 60.681,
"overall_status": "PARTIAL",
"layers": [
{
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.399132+08:00",
"end_time": "2026-02-16T02:00:32.176952+08:00",
"duration_sec": 1.778,
"status": "ERROR",
"task_count": 23,
"success_count": 0,
"fail_count": 23,
"skip_count": 0,
"total_fetched": 0,
"total_inserted": 0,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "ODS_ASSISTANT_ABOLISH",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.404857+08:00",
"end_time": "2026-02-16T02:00:30.626099+08:00",
"duration_sec": 0.221,
"status": "ERROR",
"counts": {},
"error": "错误: 重复键违反唯一约束\"etl_run_pkey\"\nDETAIL: 键值\"(run_id)=(1)\" 已经存在\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_ASSISTANT_ACCOUNT",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.631572+08:00",
"end_time": "2026-02-16T02:00:30.692791+08:00",
"duration_sec": 0.061,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_ASSISTANT_LEDGER",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.696161+08:00",
"end_time": "2026-02-16T02:00:30.749470+08:00",
"duration_sec": 0.053,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_GOODS_CATEGORY",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.768981+08:00",
"end_time": "2026-02-16T02:00:30.820272+08:00",
"duration_sec": 0.051,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_GROUP_BUY_REDEMPTION",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.823809+08:00",
"end_time": "2026-02-16T02:00:30.886627+08:00",
"duration_sec": 0.063,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_GROUP_PACKAGE",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.889777+08:00",
"end_time": "2026-02-16T02:00:30.949457+08:00",
"duration_sec": 0.06,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_INVENTORY_CHANGE",
"layer": "ODS",
"start_time": "2026-02-16T02:00:30.951549+08:00",
"end_time": "2026-02-16T02:00:31.001887+08:00",
"duration_sec": 0.05,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_INVENTORY_STOCK",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.004960+08:00",
"end_time": "2026-02-16T02:00:31.154070+08:00",
"duration_sec": 0.149,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_MEMBER",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.196785+08:00",
"end_time": "2026-02-16T02:00:31.277524+08:00",
"duration_sec": 0.081,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_MEMBER_BALANCE",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.282884+08:00",
"end_time": "2026-02-16T02:00:31.389490+08:00",
"duration_sec": 0.107,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_MEMBER_CARD",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.391553+08:00",
"end_time": "2026-02-16T02:00:31.443622+08:00",
"duration_sec": 0.052,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_PAYMENT",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.455259+08:00",
"end_time": "2026-02-16T02:00:31.516596+08:00",
"duration_sec": 0.061,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_PLATFORM_COUPON",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.519180+08:00",
"end_time": "2026-02-16T02:00:31.571746+08:00",
"duration_sec": 0.053,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_RECHARGE_SETTLE",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.587139+08:00",
"end_time": "2026-02-16T02:00:31.645916+08:00",
"duration_sec": 0.059,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_REFUND",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.649692+08:00",
"end_time": "2026-02-16T02:00:31.707211+08:00",
"duration_sec": 0.057,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_SETTLEMENT_RECORDS",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.709718+08:00",
"end_time": "2026-02-16T02:00:31.762221+08:00",
"duration_sec": 0.052,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_SETTLEMENT_TICKET",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.772867+08:00",
"end_time": "2026-02-16T02:00:31.831914+08:00",
"duration_sec": 0.059,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_STORE_GOODS",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.835277+08:00",
"end_time": "2026-02-16T02:00:31.890692+08:00",
"duration_sec": 0.055,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_STORE_GOODS_SALES",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.900307+08:00",
"end_time": "2026-02-16T02:00:31.950987+08:00",
"duration_sec": 0.051,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TABLES",
"layer": "ODS",
"start_time": "2026-02-16T02:00:31.953280+08:00",
"end_time": "2026-02-16T02:00:32.003567+08:00",
"duration_sec": 0.05,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TABLE_FEE_DISCOUNT",
"layer": "ODS",
"start_time": "2026-02-16T02:00:32.013166+08:00",
"end_time": "2026-02-16T02:00:32.064346+08:00",
"duration_sec": 0.051,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TABLE_USE",
"layer": "ODS",
"start_time": "2026-02-16T02:00:32.066702+08:00",
"end_time": "2026-02-16T02:00:32.120437+08:00",
"duration_sec": 0.054,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TENANT_GOODS",
"layer": "ODS",
"start_time": "2026-02-16T02:00:32.124485+08:00",
"end_time": "2026-02-16T02:00:32.174414+08:00",
"duration_sec": 0.05,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
}
]
},
{
"layer": "DWD",
"start_time": "2026-02-16T02:00:32.180137+08:00",
"end_time": "2026-02-16T02:00:32.288850+08:00",
"duration_sec": 0.109,
"status": "ERROR",
"task_count": 1,
"success_count": 0,
"fail_count": 1,
"skip_count": 0,
"total_fetched": 0,
"total_inserted": 0,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "DWD_LOAD_FROM_ODS",
"layer": "DWD",
"start_time": "2026-02-16T02:00:32.187417+08:00",
"end_time": "2026-02-16T02:00:32.270397+08:00",
"duration_sec": 0.083,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
}
]
},
{
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.434597+08:00",
"end_time": "2026-02-16T02:00:33.733537+08:00",
"duration_sec": 1.299,
"status": "ERROR",
"task_count": 15,
"success_count": 0,
"fail_count": 6,
"skip_count": 9,
"total_fetched": 0,
"total_inserted": 0,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "DWS_ASSISTANT_CUSTOMER",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.438912+08:00",
"end_time": "2026-02-16T02:00:32.524164+08:00",
"duration_sec": 0.085,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.531949+08:00",
"end_time": "2026-02-16T02:00:32.584970+08:00",
"duration_sec": 0.053,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_FINANCE",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.587080+08:00",
"end_time": "2026-02-16T02:00:32.640944+08:00",
"duration_sec": 0.054,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_MONTHLY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.651498+08:00",
"end_time": "2026-02-16T02:00:32.702636+08:00",
"duration_sec": 0.051,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_SALARY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.704713+08:00",
"end_time": "2026-02-16T02:00:32.765519+08:00",
"duration_sec": 0.061,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_BUILD_ORDER_SUMMARY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.778507+08:00",
"end_time": "2026-02-16T02:00:32.891324+08:00",
"duration_sec": 0.113,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:32.892786+08:00",
"end_time": "2026-02-16T02:00:33.011285+08:00",
"duration_sec": 0.118,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_DISCOUNT_DETAIL",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.015489+08:00",
"end_time": "2026-02-16T02:00:33.070572+08:00",
"duration_sec": 0.055,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_INCOME_STRUCTURE",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.072599+08:00",
"end_time": "2026-02-16T02:00:33.133539+08:00",
"duration_sec": 0.061,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_RECHARGE",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.135092+08:00",
"end_time": "2026-02-16T02:00:33.185416+08:00",
"duration_sec": 0.05,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MEMBER_CONSUMPTION",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.189842+08:00",
"end_time": "2026-02-16T02:00:33.256656+08:00",
"duration_sec": 0.067,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MEMBER_VISIT",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.258323+08:00",
"end_time": "2026-02-16T02:00:33.471349+08:00",
"duration_sec": 0.213,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MV_REFRESH_ASSISTANT_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.524696+08:00",
"end_time": "2026-02-16T02:00:33.614408+08:00",
"duration_sec": 0.09,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MV_REFRESH_FINANCE_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.621684+08:00",
"end_time": "2026-02-16T02:00:33.675725+08:00",
"duration_sec": 0.054,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_RETENTION_CLEANUP",
"layer": "DWS",
"start_time": "2026-02-16T02:00:33.677408+08:00",
"end_time": "2026-02-16T02:00:33.732228+08:00",
"duration_sec": 0.055,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
}
]
},
{
"layer": "INDEX",
"start_time": "2026-02-16T02:00:33.738360+08:00",
"end_time": "2026-02-16T02:00:37.081635+08:00",
"duration_sec": 3.343,
"status": "ERROR",
"task_count": 4,
"success_count": 0,
"fail_count": 4,
"skip_count": 0,
"total_fetched": 0,
"total_inserted": 0,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "DWS_ML_MANUAL_IMPORT",
"layer": "INDEX",
"start_time": "2026-02-16T02:00:33.740154+08:00",
"end_time": "2026-02-16T02:00:33.743497+08:00",
"duration_sec": 0.003,
"status": "ERROR",
"counts": {},
"error": "未找到 ML 台账文件，请通过环境变量 ML_MANUAL_LEDGER_FILE 或配置 run.ml_manual_ledger_file 指定",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_NEWCONV_INDEX",
"layer": "INDEX",
"start_time": "2026-02-16T02:00:33.744864+08:00",
"end_time": "2026-02-16T02:00:36.742473+08:00",
"duration_sec": 2.998,
"status": "ERROR",
"counts": {},
"error": "错误: 重复键违反唯一约束\"dws_index_percentile_history_pkey\"\nDETAIL: 键值\"(history_id)=(1)\" 已经存在\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_RELATION_INDEX",
"layer": "INDEX",
"start_time": "2026-02-16T02:00:36.747281+08:00",
"end_time": "2026-02-16T02:00:36.820768+08:00",
"duration_sec": 0.073,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_WINBACK_INDEX",
"layer": "INDEX",
"start_time": "2026-02-16T02:00:36.823642+08:00",
"end_time": "2026-02-16T02:00:37.079836+08:00",
"duration_sec": 0.256,
"status": "ERROR",
"counts": {},
"error": "错误: 当前事务被终止, 事务块结束之前的查询被忽略\n",
"api_calls": 0,
"api_total_sec": 0.0
}
]
}
],
"verification": {
"status": "COMPLETED",
"start_time": "2026-02-16T02:00:37.094515+08:00",
"end_time": "2026-02-16T02:01:31.080036+08:00",
"duration_sec": 53.985,
"total_tables": 15,
"consistent_tables": 7,
"total_backfilled": 163,
"error_tables": 3,
"layers": {
"ODS": {
"layer": "ODS",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"total_tables": 1,
"consistent_tables": 0,
"inconsistent_tables": 1,
"total_source_count": 0,
"total_target_count": 0,
"total_missing": 0,
"total_mismatch": 0,
"total_backfilled": 0,
"total_backfilled_missing": 0,
"total_backfilled_mismatch": 0,
"error_tables": 1,
"elapsed_seconds": 0.1302633285522461,
"status": "ERROR",
"results": [
{
"layer": "ODS",
"table": "assistant_accounts_master",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 0,
"target_count": 0,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "ERROR",
"elapsed_seconds": 0.1302633285522461,
"error_message": "获取 ODS hash 失败: assistant_accounts_master",
"details": {
"fatal": true
}
}
]
},
"DWD": {
"layer": "DWD",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"total_tables": 6,
"consistent_tables": 0,
"inconsistent_tables": 6,
"total_source_count": 163,
"total_target_count": 219,
"total_missing": 0,
"total_mismatch": 163,
"total_backfilled": 163,
"total_backfilled_missing": 0,
"total_backfilled_mismatch": 163,
"error_tables": 1,
"elapsed_seconds": 49.87700796127319,
"status": "ERROR",
"results": [
{
"layer": "DWD",
"table": "dim_site",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 1,
"target_count": 1,
"missing_count": 0,
"mismatch_count": 1,
"backfilled_count": 1,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 1,
"status": "BACKFILLED",
"elapsed_seconds": 14.668256521224976,
"error_message": null,
"details": {}
},
{
"layer": "DWD",
"table": "dim_site_ex",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 1,
"target_count": 1,
"missing_count": 0,
"mismatch_count": 1,
"backfilled_count": 1,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 1,
"status": "BACKFILLED",
"elapsed_seconds": 14.868768453598022,
"error_message": null,
"details": {}
},
{
"layer": "DWD",
"table": "dim_table",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 66,
"target_count": 74,
"missing_count": 0,
"mismatch_count": 66,
"backfilled_count": 66,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 66,
"status": "BACKFILLED",
"elapsed_seconds": 9.084474802017212,
"error_message": null,
"details": {}
},
{
"layer": "DWD",
"table": "dim_table_ex",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 66,
"target_count": 74,
"missing_count": 0,
"mismatch_count": 66,
"backfilled_count": 66,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 66,
"status": "BACKFILLED",
"elapsed_seconds": 7.42323637008667,
"error_message": null,
"details": {}
},
{
"layer": "DWD",
"table": "dim_assistant",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 29,
"target_count": 69,
"missing_count": 0,
"mismatch_count": 29,
"backfilled_count": 29,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 29,
"status": "BACKFILLED",
"elapsed_seconds": 3.5641441345214844,
"error_message": null,
"details": {}
},
{
"layer": "DWD",
"table": "dim_assistant_ex",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 0,
"target_count": 0,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "ERROR",
"elapsed_seconds": 0.2681276798248291,
"error_message": "获取 DWD hash 失败: dim_assistant_ex",
"details": {
"fatal": true
}
}
]
},
"DWS": {
"layer": "DWS",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"total_tables": 6,
"consistent_tables": 6,
"inconsistent_tables": 0,
"total_source_count": 1712,
"total_target_count": 1712,
"total_missing": 0,
"total_mismatch": 0,
"total_backfilled": 0,
"total_backfilled_missing": 0,
"total_backfilled_mismatch": 0,
"error_tables": 0,
"elapsed_seconds": 0.8101677894592285,
"status": "OK",
"results": [
{
"layer": "DWS",
"table": "dws_finance_daily_summary",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 31,
"target_count": 31,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.17444229125976562,
"error_message": null,
"details": {}
},
{
"layer": "DWS",
"table": "dws_assistant_daily_detail",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 854,
"target_count": 854,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.1419987678527832,
"error_message": null,
"details": {}
},
{
"layer": "DWS",
"table": "dws_member_visit_detail",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 564,
"target_count": 564,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.1277296543121338,
"error_message": null,
"details": {}
},
{
"layer": "DWS",
"table": "dws_finance_daily_summary",
"window_start": "2026-02-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"source_count": 10,
"target_count": 10,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.15016722679138184,
"error_message": null,
"details": {}
},
{
"layer": "DWS",
"table": "dws_assistant_daily_detail",
"window_start": "2026-02-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"source_count": 125,
"target_count": 125,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.10792803764343262,
"error_message": null,
"details": {}
},
{
"layer": "DWS",
"table": "dws_member_visit_detail",
"window_start": "2026-02-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"source_count": 128,
"target_count": 128,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.10790181159973145,
"error_message": null,
"details": {}
}
]
},
"INDEX": {
"layer": "INDEX",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-16T00:00:00+08:00",
"total_tables": 2,
"consistent_tables": 1,
"inconsistent_tables": 1,
"total_source_count": 94,
"total_target_count": 171,
"total_missing": 0,
"total_mismatch": 0,
"total_backfilled": 0,
"total_backfilled_missing": 0,
"total_backfilled_mismatch": 0,
"error_tables": 1,
"elapsed_seconds": 0.2954070568084717,
"status": "ERROR",
"results": [
{
"layer": "INDEX",
"table": "v_member_recall_priority",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 94,
"target_count": 171,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "OK",
"elapsed_seconds": 0.18698692321777344,
"error_message": null,
"details": {}
},
{
"layer": "INDEX",
"table": "dws_member_assistant_relation_index",
"window_start": "2026-01-01T00:00:00+08:00",
"window_end": "2026-02-01T00:00:00+08:00",
"source_count": 0,
"target_count": 0,
"missing_count": 0,
"mismatch_count": 0,
"backfilled_count": 0,
"backfilled_missing_count": 0,
"backfilled_mismatch_count": 0,
"status": "ERROR",
"elapsed_seconds": 0.10842013359069824,
"error_message": "获取源实体失败: dws_member_assistant_relation_index",
"details": {
"fatal": true
}
}
]
}
}
},
"environment": {
"store_id": 2790685415443269,
"db_name": "",
"api_base_url": "https://pc.ficoo.vip/apiprod/admin/v1/",
"timezone": "Asia/Shanghai"
}
}

1889
apps/etl/connectors/feiqiu/scripts/debug/output/full_refresh_20260216_022203.json Normal file
View File

File diff suppressed because it is too large Load Diff

5
apps/etl/connectors/feiqiu/scripts/debug/output/full_refresh_checkpoint.json Normal file
View File

@@ -0,0 +1,5 @@
{
"last_completed_layer": "INDEX",
"last_completed_task": "DWS_WINBACK_INDEX",
"timestamp": "2026-02-16T02:21:00.006150+08:00"
}

774
apps/etl/connectors/feiqiu/scripts/debug/output/full_refresh_intermediate.json Normal file
View File

@@ -0,0 +1,774 @@
{
"flow": "api_full",
"window_start": "2026-01-01T00:00:00",
"window_end": "2026-02-16T00:00:00",
"overall_start": "2026-02-16T02:05:42.502194+08:00",
"overall_end": "",
"overall_duration_sec": 0.0,
"overall_status": "",
"layers": [
{
"layer": "ODS",
"start_time": "2026-02-16T02:05:43.228274+08:00",
"end_time": "2026-02-16T02:20:46.966053+08:00",
"duration_sec": 903.738,
"status": "SUCCESS",
"task_count": 23,
"success_count": 23,
"fail_count": 0,
"skip_count": 0,
"total_fetched": 280391,
"total_inserted": 2421,
"total_updated": 303,
"total_errors": 0,
"tasks": [
{
"task_code": "ODS_ASSISTANT_ABOLISH",
"layer": "ODS",
"start_time": "2026-02-16T02:05:43.229896+08:00",
"end_time": "2026-02-16T02:05:46.094082+08:00",
"duration_sec": 2.864,
"status": "SUCCESS",
"counts": {
"fetched": 37,
"inserted": 0,
"updated": 0,
"skipped": 37,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_ASSISTANT_ACCOUNT",
"layer": "ODS",
"start_time": "2026-02-16T02:05:46.095823+08:00",
"end_time": "2026-02-16T02:05:53.344708+08:00",
"duration_sec": 7.249,
"status": "SUCCESS",
"counts": {
"fetched": 345,
"inserted": 3,
"updated": 0,
"skipped": 342,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_ASSISTANT_LEDGER",
"layer": "ODS",
"start_time": "2026-02-16T02:05:53.350637+08:00",
"end_time": "2026-02-16T02:06:01.035404+08:00",
"duration_sec": 7.685,
"status": "SUCCESS",
"counts": {
"fetched": 998,
"inserted": 16,
"updated": 0,
"skipped": 982,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_GOODS_CATEGORY",
"layer": "ODS",
"start_time": "2026-02-16T02:06:01.038609+08:00",
"end_time": "2026-02-16T02:06:03.197268+08:00",
"duration_sec": 2.159,
"status": "SUCCESS",
"counts": {
"fetched": 45,
"inserted": 0,
"updated": 0,
"skipped": 45,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_GROUP_BUY_REDEMPTION",
"layer": "ODS",
"start_time": "2026-02-16T02:06:03.243603+08:00",
"end_time": "2026-02-16T02:08:51.258331+08:00",
"duration_sec": 168.015,
"status": "SUCCESS",
"counts": {
"fetched": 38860,
"inserted": 242,
"updated": 0,
"skipped": 38618,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_GROUP_PACKAGE",
"layer": "ODS",
"start_time": "2026-02-16T02:08:51.261789+08:00",
"end_time": "2026-02-16T02:08:54.850306+08:00",
"duration_sec": 3.588,
"status": "SUCCESS",
"counts": {
"fetched": 90,
"inserted": 0,
"updated": 0,
"skipped": 90,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_INVENTORY_CHANGE",
"layer": "ODS",
"start_time": "2026-02-16T02:08:54.854769+08:00",
"end_time": "2026-02-16T02:09:14.467632+08:00",
"duration_sec": 19.613,
"status": "SUCCESS",
"counts": {
"fetched": 7044,
"inserted": 601,
"updated": 0,
"skipped": 6443,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_INVENTORY_STOCK",
"layer": "ODS",
"start_time": "2026-02-16T02:09:14.469443+08:00",
"end_time": "2026-02-16T02:09:18.538862+08:00",
"duration_sec": 4.069,
"status": "SUCCESS",
"counts": {
"fetched": 865,
"inserted": 48,
"updated": 0,
"skipped": 817,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_MEMBER",
"layer": "ODS",
"start_time": "2026-02-16T02:09:18.540713+08:00",
"end_time": "2026-02-16T02:09:30.301232+08:00",
"duration_sec": 11.76,
"status": "SUCCESS",
"counts": {
"fetched": 2785,
"inserted": 14,
"updated": 0,
"skipped": 2771,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_MEMBER_BALANCE",
"layer": "ODS",
"start_time": "2026-02-16T02:09:30.302995+08:00",
"end_time": "2026-02-16T02:11:45.226394+08:00",
"duration_sec": 134.924,
"status": "SUCCESS",
"counts": {
"fetched": 11725,
"inserted": 39,
"updated": 0,
"skipped": 11686,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_MEMBER_CARD",
"layer": "ODS",
"start_time": "2026-02-16T02:11:45.228393+08:00",
"end_time": "2026-02-16T02:11:58.353257+08:00",
"duration_sec": 13.125,
"status": "SUCCESS",
"counts": {
"fetched": 4730,
"inserted": 19,
"updated": 0,
"skipped": 4711,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_PAYMENT",
"layer": "ODS",
"start_time": "2026-02-16T02:11:58.354979+08:00",
"end_time": "2026-02-16T02:13:57.385086+08:00",
"duration_sec": 119.03,
"status": "SUCCESS",
"counts": {
"fetched": 56795,
"inserted": 325,
"updated": 0,
"skipped": 56470,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_PLATFORM_COUPON",
"layer": "ODS",
"start_time": "2026-02-16T02:13:57.387334+08:00",
"end_time": "2026-02-16T02:17:35.403217+08:00",
"duration_sec": 218.016,
"status": "SUCCESS",
"counts": {
"fetched": 91555,
"inserted": 242,
"updated": 0,
"skipped": 91313,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_RECHARGE_SETTLE",
"layer": "ODS",
"start_time": "2026-02-16T02:17:35.405051+08:00",
"end_time": "2026-02-16T02:17:37.892719+08:00",
"duration_sec": 2.488,
"status": "SUCCESS",
"counts": {
"fetched": 90,
"inserted": 0,
"updated": 0,
"skipped": 90,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_REFUND",
"layer": "ODS",
"start_time": "2026-02-16T02:17:37.894298+08:00",
"end_time": "2026-02-16T02:17:40.855120+08:00",
"duration_sec": 2.961,
"status": "SUCCESS",
"counts": {
"fetched": 180,
"inserted": 1,
"updated": 0,
"skipped": 179,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_SETTLEMENT_RECORDS",
"layer": "ODS",
"start_time": "2026-02-16T02:17:40.857110+08:00",
"end_time": "2026-02-16T02:18:10.883995+08:00",
"duration_sec": 30.027,
"status": "SUCCESS",
"counts": {
"fetched": 4917,
"inserted": 320,
"updated": 303,
"skipped": 4294,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_SETTLEMENT_TICKET",
"layer": "ODS",
"start_time": "2026-02-16T02:18:10.885741+08:00",
"end_time": "2026-02-16T02:18:20.540209+08:00",
"duration_sec": 9.654,
"status": "SUCCESS",
"counts": {
"fetched": 0,
"inserted": 0,
"updated": 0,
"skipped": 0,
"errors": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_STORE_GOODS",
"layer": "ODS",
"start_time": "2026-02-16T02:18:20.541830+08:00",
"end_time": "2026-02-16T02:18:23.785491+08:00",
"duration_sec": 3.244,
"status": "SUCCESS",
"counts": {
"fetched": 865,
"inserted": 173,
"updated": 0,
"skipped": 692,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_STORE_GOODS_SALES",
"layer": "ODS",
"start_time": "2026-02-16T02:18:23.787736+08:00",
"end_time": "2026-02-16T02:18:25.142325+08:00",
"duration_sec": 1.355,
"status": "SUCCESS",
"counts": {
"fetched": 0,
"inserted": 0,
"updated": 0,
"skipped": 0,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TABLES",
"layer": "ODS",
"start_time": "2026-02-16T02:18:25.144553+08:00",
"end_time": "2026-02-16T02:18:28.900234+08:00",
"duration_sec": 3.756,
"status": "SUCCESS",
"counts": {
"fetched": 370,
"inserted": 51,
"updated": 0,
"skipped": 319,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TABLE_FEE_DISCOUNT",
"layer": "ODS",
"start_time": "2026-02-16T02:18:28.901962+08:00",
"end_time": "2026-02-16T02:19:04.877900+08:00",
"duration_sec": 35.976,
"status": "SUCCESS",
"counts": {
"fetched": 8680,
"inserted": 39,
"updated": 0,
"skipped": 8641,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TABLE_USE",
"layer": "ODS",
"start_time": "2026-02-16T02:19:04.879677+08:00",
"end_time": "2026-02-16T02:20:43.722924+08:00",
"duration_sec": 98.844,
"status": "SUCCESS",
"counts": {
"fetched": 48545,
"inserted": 285,
"updated": 0,
"skipped": 48260,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "ODS_TENANT_GOODS",
"layer": "ODS",
"start_time": "2026-02-16T02:20:43.725501+08:00",
"end_time": "2026-02-16T02:20:46.964569+08:00",
"duration_sec": 3.239,
"status": "SUCCESS",
"counts": {
"fetched": 870,
"inserted": 3,
"updated": 0,
"skipped": 867,
"errors": 0,
"deleted": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
}
]
},
{
"layer": "DWD",
"start_time": "2026-02-16T02:20:46.967739+08:00",
"end_time": "2026-02-16T02:20:47.394765+08:00",
"duration_sec": 0.427,
"status": "SUCCESS",
"task_count": 1,
"success_count": 1,
"fail_count": 0,
"skip_count": 0,
"total_fetched": 0,
"total_inserted": 0,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "DWD_LOAD_FROM_ODS",
"layer": "DWD",
"start_time": "2026-02-16T02:20:46.968453+08:00",
"end_time": "2026-02-16T02:20:47.392262+08:00",
"duration_sec": 0.424,
"status": "SUCCESS",
"counts": {
"fetched": 0,
"inserted": 0,
"updated": 0,
"skipped": 0,
"errors": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
}
]
},
{
"layer": "DWS",
"start_time": "2026-02-16T02:20:47.397305+08:00",
"end_time": "2026-02-16T02:20:50.260818+08:00",
"duration_sec": 2.863,
"status": "SUCCESS",
"task_count": 15,
"success_count": 1,
"fail_count": 0,
"skip_count": 14,
"total_fetched": 0,
"total_inserted": 5117,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "DWS_ASSISTANT_CUSTOMER",
"layer": "DWS",
"start_time": "2026-02-16T02:20:47.478185+08:00",
"end_time": "2026-02-16T02:20:47.912593+08:00",
"duration_sec": 0.434,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:47.916343+08:00",
"end_time": "2026-02-16T02:20:48.058411+08:00",
"duration_sec": 0.142,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_FINANCE",
"layer": "DWS",
"start_time": "2026-02-16T02:20:48.060291+08:00",
"end_time": "2026-02-16T02:20:48.109598+08:00",
"duration_sec": 0.049,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_MONTHLY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:48.111303+08:00",
"end_time": "2026-02-16T02:20:48.164240+08:00",
"duration_sec": 0.053,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_ASSISTANT_SALARY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:48.165763+08:00",
"end_time": "2026-02-16T02:20:48.216816+08:00",
"duration_sec": 0.051,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_BUILD_ORDER_SUMMARY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:48.218588+08:00",
"end_time": "2026-02-16T02:20:49.720095+08:00",
"duration_sec": 1.501,
"status": "SUCCESS",
"counts": {
"fetched": 0,
"inserted": 5117,
"updated": 0,
"skipped": 0,
"errors": 0
},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:49.721608+08:00",
"end_time": "2026-02-16T02:20:49.827953+08:00",
"duration_sec": 0.106,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_DISCOUNT_DETAIL",
"layer": "DWS",
"start_time": "2026-02-16T02:20:49.830310+08:00",
"end_time": "2026-02-16T02:20:49.882148+08:00",
"duration_sec": 0.052,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_INCOME_STRUCTURE",
"layer": "DWS",
"start_time": "2026-02-16T02:20:49.884147+08:00",
"end_time": "2026-02-16T02:20:49.937621+08:00",
"duration_sec": 0.053,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_FINANCE_RECHARGE",
"layer": "DWS",
"start_time": "2026-02-16T02:20:49.939594+08:00",
"end_time": "2026-02-16T02:20:49.990880+08:00",
"duration_sec": 0.051,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MEMBER_CONSUMPTION",
"layer": "DWS",
"start_time": "2026-02-16T02:20:49.993066+08:00",
"end_time": "2026-02-16T02:20:50.050887+08:00",
"duration_sec": 0.058,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MEMBER_VISIT",
"layer": "DWS",
"start_time": "2026-02-16T02:20:50.052695+08:00",
"end_time": "2026-02-16T02:20:50.102870+08:00",
"duration_sec": 0.05,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MV_REFRESH_ASSISTANT_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:50.104174+08:00",
"end_time": "2026-02-16T02:20:50.153937+08:00",
"duration_sec": 0.05,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_MV_REFRESH_FINANCE_DAILY",
"layer": "DWS",
"start_time": "2026-02-16T02:20:50.155430+08:00",
"end_time": "2026-02-16T02:20:50.205405+08:00",
"duration_sec": 0.05,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_RETENTION_CLEANUP",
"layer": "DWS",
"start_time": "2026-02-16T02:20:50.207165+08:00",
"end_time": "2026-02-16T02:20:50.259470+08:00",
"duration_sec": 0.052,
"status": "SKIP",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
}
]
},
{
"layer": "INDEX",
"start_time": "2026-02-16T02:20:50.263599+08:00",
"end_time": "2026-02-16T02:21:00.007411+08:00",
"duration_sec": 9.744,
"status": "PARTIAL",
"task_count": 4,
"success_count": 2,
"fail_count": 2,
"skip_count": 0,
"total_fetched": 0,
"total_inserted": 0,
"total_updated": 0,
"total_errors": 0,
"tasks": [
{
"task_code": "DWS_ML_MANUAL_IMPORT",
"layer": "INDEX",
"start_time": "2026-02-16T02:20:50.264449+08:00",
"end_time": "2026-02-16T02:20:50.266491+08:00",
"duration_sec": 0.002,
"status": "ERROR",
"counts": {},
"error": "未找到 ML 台账文件，请通过环境变量 ML_MANUAL_LEDGER_FILE 或配置 run.ml_manual_ledger_file 指定",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_NEWCONV_INDEX",
"layer": "INDEX",
"start_time": "2026-02-16T02:20:50.316501+08:00",
"end_time": "2026-02-16T02:20:53.334095+08:00",
"duration_sec": 3.018,
"status": "SUCCESS",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_RELATION_INDEX",
"layer": "INDEX",
"start_time": "2026-02-16T02:20:53.335483+08:00",
"end_time": "2026-02-16T02:20:53.696046+08:00",
"duration_sec": 0.361,
"status": "ERROR",
"counts": {},
"error": "错误: 字段 d.is_delete 不存在\nLINE 13: AND COALESCE(d.is_delete, 0) = 0\n ^\nHINT: 也许您想要引用列\"s.is_delete\"。\n",
"api_calls": 0,
"api_total_sec": 0.0
},
{
"task_code": "DWS_WINBACK_INDEX",
"layer": "INDEX",
"start_time": "2026-02-16T02:20:53.749149+08:00",
"end_time": "2026-02-16T02:21:00.005571+08:00",
"duration_sec": 6.256,
"status": "SUCCESS",
"counts": {},
"error": null,
"api_calls": 0,
"api_total_sec": 0.0
}
]
}
],
"verification": {},
"environment": {
"store_id": 2790685415443269,
"db_name": "",
"api_base_url": "https://pc.ficoo.vip/apiprod/admin/v1/",
"timezone": "Asia/Shanghai"
}
}