在前后端开发联调前 的提交20260223
This commit is contained in:
123
scripts/ops/export_v8_report.py
Normal file
123
scripts/ops/export_v8_report.py
Normal file
@@ -0,0 +1,123 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""导出 v8 执行报告。"""
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv(Path(__file__).resolve().parents[2] / ".env")
|
||||
from _env_paths import get_output_path
|
||||
|
||||
out = get_output_path("SYSTEM_LOG_ROOT")
|
||||
|
||||
report = """# ETL 第八次执行报告 (v8)
|
||||
|
||||
- execution_id: `f943bac6-23be-45c5-8b8c-a864e85a1916`
|
||||
- 时间: 2026-02-21 21:33:37 ~ 21:35:01 (1分24秒)
|
||||
- 整体状态: success, exit_code=0
|
||||
|
||||
## 本次修复验证
|
||||
|
||||
| BUG | 修复内容 | 验证结果 |
|
||||
|-----|---------|---------|
|
||||
| BUG 8 | `finance_base_task.py` + `finance_recharge_task.py`: pay_money→pay_amount, gift_money→point_amount | ✅ DWS_FINANCE_DAILY + DWS_FINANCE_RECHARGE 均完成 |
|
||||
| BUG 9 | `dwd_load_task.py`: 添加 `_pick_snapshot_order_column` 方法 | ✅ 所有 dim 表 SCD2 装载成功 |
|
||||
| BUG 10 | `dwd_load_task.py`: FACT_MAPPINGS 驼峰字段名→小写 | ✅ dwd_goods_stock_summary(716条) + dwd_goods_stock_movement(14306条) 装载成功 |
|
||||
| BUG 11 | `flow_runner.py`: sum() 类型安全处理 | ✅ 不再出现 TypeError |
|
||||
|
||||
## DWD_LOAD_FROM_ODS 详情
|
||||
|
||||
### 维度表 (SCD2) — 全部成功
|
||||
| 表 | processed | inserted | updated |
|
||||
|----|-----------|----------|---------|
|
||||
| dim_site | 1 | 0 | 1 |
|
||||
| dim_site_ex | 1 | 0 | 1 |
|
||||
| dim_table | 74 | 0 | 74 |
|
||||
| dim_table_ex | 74 | 0 | 74 |
|
||||
| dim_assistant | 69 | 0 | 69 |
|
||||
| dim_member | 557 | 0 | 557 |
|
||||
| dim_member_ex | 557 | 0 | 557 |
|
||||
| dim_member_card_account | 946 | 0 | 946 |
|
||||
| dim_tenant_goods | 174 | 1 | 173 |
|
||||
| dim_tenant_goods_ex | 174 | 1 | 173 |
|
||||
| dim_store_goods | 173 | 1 | 172 |
|
||||
| dim_store_goods_ex | 173 | 1 | 172 |
|
||||
| dim_goods_category | 26 | 0 | 26 |
|
||||
| dim_groupbuy_package | 34 | 0 | 34 |
|
||||
| dim_groupbuy_package_ex | 34 | 0 | 34 |
|
||||
|
||||
### 事实表 (INCREMENT) — 全部成功
|
||||
| 表 | processed | inserted | updated |
|
||||
|----|-----------|----------|---------|
|
||||
| dwd_settlement_head | 10366 | 0 | 10366 |
|
||||
| dwd_settlement_head_ex | 10366 | 0 | 10366 |
|
||||
| dwd_table_fee_log | 9103 | 0 | 9103 |
|
||||
| dwd_table_fee_log_ex | 9103 | 0 | 9103 |
|
||||
| dwd_table_fee_adjust | 1616 | 0 | 1616 |
|
||||
| dwd_table_fee_adjust_ex | 1616 | 0 | 1616 |
|
||||
| dwd_assistant_service_log | 2619 | 0 | 2619 |
|
||||
| dwd_assistant_service_log_ex | 2619 | 0 | 2619 |
|
||||
| dwd_assistant_trash_event | 78 | 0 | 78 |
|
||||
| dwd_assistant_trash_event_ex | 78 | 0 | 78 |
|
||||
| dwd_member_balance_change | 2185 | 0 | 2185 |
|
||||
| dwd_member_balance_change_ex | 2185 | 0 | 2185 |
|
||||
| dwd_groupbuy_redemption | 7267 | 0 | 7267 |
|
||||
| dwd_groupbuy_redemption_ex | 7267 | 0 | 7267 |
|
||||
| dwd_platform_coupon_redemption | 18311 | 0 | 18311 |
|
||||
| dwd_platform_coupon_redemption_ex | 18311 | 0 | 18311 |
|
||||
| dwd_recharge_order | 191 | 0 | 191 |
|
||||
| dwd_recharge_order_ex | 191 | 0 | 191 |
|
||||
| dwd_payment | 10625 | 0 | 10625 |
|
||||
| dwd_refund | 29 | 0 | 29 |
|
||||
| dwd_refund_ex | 29 | 0 | 29 |
|
||||
| dwd_goods_stock_summary | 716 | 716 | 0 |
|
||||
| dwd_goods_stock_movement | 14306 | 14306 | 0 |
|
||||
|
||||
### DWD 装载错误 (2个，数据质量问题，非代码 BUG)
|
||||
| 表 | 错误 |
|
||||
|----|------|
|
||||
| dim_assistant_ex | year -1 is out of range |
|
||||
| dim_member_card_account_ex | year -1 is out of range |
|
||||
|
||||
## DWS 任务状态
|
||||
|
||||
| 任务 | 状态 | 备注 |
|
||||
|------|------|------|
|
||||
| ODS_FETCH | ✅ 完成 | |
|
||||
| DWD_LOAD_FROM_ODS | ✅ 完成 | 39表成功，2表数据质量错误 |
|
||||
| DWS_ASSISTANT_DAILY | ✅ 完成 | |
|
||||
| DWS_ASSISTANT_MONTHLY | ✅ 完成 | 删除9行，插入9行 |
|
||||
| DWS_ASSISTANT_CUSTOMER | ✅ 完成 | 删除285行，插入285行 |
|
||||
| DWS_ASSISTANT_SALARY | ✅ 完成 | |
|
||||
| DWS_ASSISTANT_FINANCE | ✅ 完成 | |
|
||||
| DWS_MEMBER_CONSUMPTION | ✅ 完成 | 删除198行，插入198行 |
|
||||
| DWS_MEMBER_VISIT | ✅ 完成 | |
|
||||
| DWS_GOODS_STOCK_DAILY | ✅ 完成 | |
|
||||
| DWS_GOODS_STOCK_WEEKLY | ✅ 完成 | |
|
||||
| DWS_GOODS_STOCK_MONTHLY | ✅ 完成 | |
|
||||
| DWS_FINANCE_DAILY | ✅ 完成 | |
|
||||
| DWS_FINANCE_RECHARGE | ✅ 完成 | |
|
||||
| DWS_FINANCE_INCOME_STRUCTURE | ❌ 级联失败 | InFailedSqlTransaction |
|
||||
| DWS_FINANCE_DISCOUNT_DETAIL | ❌ 级联失败 | InFailedSqlTransaction |
|
||||
| DWS_WINBACK_INDEX | ❌ 级联失败 | InFailedSqlTransaction |
|
||||
| DWS_NEWCONV_INDEX | ❌ 级联失败 | InFailedSqlTransaction |
|
||||
| DWS_RELATION_INDEX | ❌ 级联失败 | InFailedSqlTransaction |
|
||||
|
||||
## 总结
|
||||
|
||||
- 14/19 任务成功完成
|
||||
- 5/19 任务因 InFailedSqlTransaction 级联失败
|
||||
- 级联失败根因: `dim_assistant_ex` 和 `dim_member_card_account_ex` 中存在非法日期值 (year=-1)，导致事务进入失败状态
|
||||
- 这是数据质量问题，不是代码 BUG — 需要在 DWD 装载时对日期字段做容错处理
|
||||
|
||||
## 与 v6（上次最好成绩）对比
|
||||
|
||||
| 指标 | v6 | v8 |
|
||||
|------|----|----|
|
||||
| 耗时 | 29m26s | 1m24s |
|
||||
| 成功任务 | 11/19 | 14/19 |
|
||||
| 失败任务 | 8/19 | 5/19 |
|
||||
| DWD 装载 | 部分 dim 失败 | 39/41 表成功 |
|
||||
| 新增成功 | — | DWS_FINANCE_DAILY, DWS_FINANCE_RECHARGE, DWS_GOODS_STOCK_* |
|
||||
"""
|
||||
|
||||
(out / "2026-02-21__etl_run_result_v8.md").write_text(report, encoding="utf-8")
|
||||
print(f"报告已导出: {out / '2026-02-21__etl_run_result_v8.md'}")
|
||||
Reference in New Issue
Block a user