From a3f4d043350b81872b90b4b87f38aa27eef4c1f0 Mon Sep 17 00:00:00 2001 From: Neo Date: Wed, 4 Feb 2026 21:39:01 +0800 Subject: [PATCH] Updata2 --- .config/dotnet-tools.json | 13 + README.md | 157 +- etl_billiards/.env | 2 +- etl_billiards/database/schema_ODS_doc.sql | 59 + etl_billiards/database/schema_dwd_doc.sql | 91 +- etl_billiards/database/schema_dws.sql | 1334 +++- .../database/seed_index_parameters.sql | 118 + etl_billiards/docs/DWS 数据库处理需求.md | 82 +- etl_billiards/docs/DWS_任务计划_v1.md | 1122 ++++ etl_billiards/docs/DWS_任务计划_v2.md | 1353 +++++ .../Ex/BD_manual_dim_assistant_ex.md | 0 .../Ex/BD_manual_dim_groupbuy_package_ex.md | 9 +- .../BD_manual_dim_member_card_account_ex.md | 13 +- .../{ => DWD}/Ex/BD_manual_dim_member_ex.md | 11 +- .../{ => DWD}/Ex/BD_manual_dim_site_ex.md | 0 .../Ex/BD_manual_dim_store_goods_ex.md | 0 .../{ => DWD}/Ex/BD_manual_dim_table_ex.md | 0 .../Ex/BD_manual_dim_tenant_goods_ex.md | 0 .../BD_manual_dwd_assistant_service_log_ex.md | 1 + .../BD_manual_dwd_assistant_trash_event_ex.md | 0 .../BD_manual_dwd_groupbuy_redemption_ex.md | 7 + .../BD_manual_dwd_member_balance_change_ex.md | 1 + ...anual_dwd_platform_coupon_redemption_ex.md | 0 .../Ex/BD_manual_dwd_recharge_order_ex.md | 0 .../{ => DWD}/Ex/BD_manual_dwd_refund_ex.md | 0 .../Ex/BD_manual_dwd_settlement_head_ex.md | 1 + .../Ex/BD_manual_dwd_store_goods_sale_ex.md | 0 .../Ex/BD_manual_dwd_table_fee_adjust_ex.md | 5 + .../Ex/BD_manual_dwd_table_fee_log_ex.md | 1 + .../{ => DWD}/main/BD_manual_billiards_dwd.md | 0 .../{ => DWD}/main/BD_manual_dim_assistant.md | 0 .../main/BD_manual_dim_goods_category.md | 6 +- .../main/BD_manual_dim_groupbuy_package.md | 18 +- .../{ => DWD}/main/BD_manual_dim_member.md | 10 +- .../main/BD_manual_dim_member_card_account.md | 10 +- .../{ => DWD}/main/BD_manual_dim_site.md | 0 .../main/BD_manual_dim_store_goods.md | 10 +- .../{ => DWD}/main/BD_manual_dim_table.md | 9 +- .../main/BD_manual_dim_tenant_goods.md | 17 +- .../BD_manual_dwd_assistant_service_log.md | 9 +- .../BD_manual_dwd_assistant_trash_event.md | 7 +- .../main/BD_manual_dwd_groupbuy_redemption.md | 6 +- .../BD_manual_dwd_member_balance_change.md | 11 +- .../{ => DWD}/main/BD_manual_dwd_payment.md | 1 + ...D_manual_dwd_platform_coupon_redemption.md | 8 +- .../main/BD_manual_dwd_recharge_order.md | 0 .../{ => DWD}/main/BD_manual_dwd_refund.md | 0 .../main/BD_manual_dwd_settlement_head.md | 17 +- .../main/BD_manual_dwd_store_goods_sale.md | 5 +- .../main/BD_manual_dwd_table_fee_adjust.md | 6 +- .../main/BD_manual_dwd_table_fee_log.md | 6 +- .../dws/BD_manual_cfg_area_category.md | 74 + .../BD_manual_cfg_assistant_level_price.md | 56 + .../dws/BD_manual_cfg_bonus_rules.md | 73 + .../dws/BD_manual_cfg_performance_tier.md | 71 + .../bd_manual/dws/BD_manual_cfg_skill_type.md | 62 + .../BD_manual_dws_assistant_customer_stats.md | 98 + .../BD_manual_dws_assistant_daily_detail.md | 109 + ...D_manual_dws_assistant_finance_analysis.md | 88 + ...BD_manual_dws_assistant_monthly_summary.md | 110 + ...anual_dws_assistant_recharge_commission.md | 84 + .../BD_manual_dws_assistant_salary_calc.md | 98 + .../BD_manual_dws_finance_daily_summary.md | 125 + .../BD_manual_dws_finance_discount_detail.md | 90 + .../BD_manual_dws_finance_expense_summary.md | 87 + .../BD_manual_dws_finance_income_structure.md | 88 + .../BD_manual_dws_finance_recharge_summary.md | 95 + ...D_manual_dws_member_consumption_summary.md | 102 + .../dws/BD_manual_dws_member_visit_detail.md | 119 + .../dws/BD_manual_dws_platform_settlement.md | 100 + etl_billiards/docs/dws_tables_dictionary.md | 585 ++ etl_billiards/docs/index_tables.md | 494 ++ etl_billiards/docs/index_tables_output.txt | Bin 0 -> 46056 bytes etl_billiards/docs/补充更多信息.md | 167 + etl_billiards/docs/财务页面需求.md | 198 + etl_billiards/gui/widgets/task_panel.py | 7 + etl_billiards/orchestration/task_registry.py | 40 + etl_billiards/scheduled_tasks.json | 318 +- .../scripts/analyze_discount_patterns.py | 636 ++ .../scripts/analyze_member_discount_usage.py | 287 + etl_billiards/scripts/check_assistant_dim.py | 74 + etl_billiards/scripts/check_dwd_service.py | 82 + etl_billiards/scripts/check_intimacy_stats.py | 57 + etl_billiards/scripts/check_ods_gaps.py | 3 +- etl_billiards/scripts/create_index_tables.py | 185 + etl_billiards/scripts/import_dws_excel.py | 602 ++ etl_billiards/scripts/run_seed_dws_config.py | 35 + etl_billiards/scripts/show_area_category.py | 43 + .../scripts/show_performance_tier.py | 48 + etl_billiards/scripts/test_index_tasks.py | 222 + etl_billiards/scripts/verify_dws_config.py | 34 + etl_billiards/tasks/dwd_load_task.py | 79 +- etl_billiards/tasks/dws/__init__.py | 55 + .../tasks/dws/assistant_customer_task.py | 333 + .../tasks/dws/assistant_daily_task.py | 344 ++ .../tasks/dws/assistant_finance_task.py | 199 + .../tasks/dws/assistant_monthly_task.py | 444 ++ .../tasks/dws/assistant_salary_task.py | 403 ++ etl_billiards/tasks/dws/base_dws_task.py | 1223 ++++ etl_billiards/tasks/dws/finance_daily_task.py | 574 ++ .../tasks/dws/finance_discount_task.py | 410 ++ .../tasks/dws/finance_income_task.py | 437 ++ .../tasks/dws/finance_recharge_task.py | 172 + etl_billiards/tasks/dws/index/__init__.py | 16 + .../tasks/dws/index/base_index_task.py | 518 ++ .../tasks/dws/index/intimacy_index_task.py | 688 +++ .../tasks/dws/index/recall_index_task.py | 564 ++ .../tasks/dws/member_consumption_task.py | 368 ++ etl_billiards/tasks/dws/member_visit_task.py | 386 ++ .../tasks/dws/retention_cleanup_task.py | 161 + etl_billiards/tasks/ods_tasks.py | 61 +- etl_billiards/tasks/seed_dws_config_task.py | 63 + etl_billiards/tests/test_dws_tasks.py | 472 ++ scripts/backfill_202507_to_now.bat | 45 + tmp/Untitled | 94 + tmp/add_missing_dwd_columns.py | 226 + tmp/add_missing_ods_columns.py | 162 + tmp/add_remaining_dwd_columns.py | 37 + tmp/api_ods_comparison.json | 2305 +++++++ tmp/api_ods_issue_report.json | 355 ++ tmp/backfill_dwd_from_ods.py | 283 + tmp/backfill_ods_from_payload.py | 208 + tmp/bd_manual_diff.json | 57 + tmp/check_api_ods_issues.py | 295 + tmp/check_ddl_vs_db.py | 231 + tmp/check_field_variants.py | 181 + tmp/check_new_fields_data.py | 95 + tmp/check_scd2_tables.py | 90 + tmp/check_seq.py | 26 + tmp/compare_api_ods_fields.py | 510 ++ tmp/detailed_field_compare.py | 181 + tmp/dwd_schema.json | 5404 +++++++++++++++++ tmp/field_coverage_report.json | 38 + tmp/fix_bd_manual.py | 180 + tmp/fix_not_sale_type.py | 55 + tmp/fix_remaining_issues.py | 63 + tmp/full_reload_validation.py | 237 + tmp/get_dwd_schema.py | 48 + tmp/list_dwd_tables.py | 9 + tmp/query_schema.py | 19 + tmp/schema_output.txt | 994 +++ tmp/sync_api_to_ods_columns.py | 238 + tmp/sync_bd_manual.py | 181 + tmp/sync_dwd_columns_log.json | 10 + tmp/sync_ods_columns_log.json | 13 + tmp/sync_ods_to_dwd_columns.py | 259 + tmp/test_backfill_feature.py | 86 + tmp/test_conflict_modes.py | 70 + 148 files changed, 31455 insertions(+), 182 deletions(-) create mode 100644 .config/dotnet-tools.json create mode 100644 etl_billiards/database/seed_index_parameters.sql create mode 100644 etl_billiards/docs/DWS_任务计划_v1.md create mode 100644 etl_billiards/docs/DWS_任务计划_v2.md rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_assistant_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_groupbuy_package_ex.md (91%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_member_card_account_ex.md (90%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_member_ex.md (81%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_site_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_store_goods_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_table_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dim_tenant_goods_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_assistant_service_log_ex.md (98%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_assistant_trash_event_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_groupbuy_redemption_ex.md (85%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_member_balance_change_ex.md (97%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_platform_coupon_redemption_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_recharge_order_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_refund_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_settlement_head_ex.md (98%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_store_goods_sale_ex.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_table_fee_adjust_ex.md (86%) rename etl_billiards/docs/bd_manual/{ => DWD}/Ex/BD_manual_dwd_table_fee_log_ex.md (97%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_billiards_dwd.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_assistant.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_goods_category.md (89%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_groupbuy_package.md (72%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_member.md (83%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_member_card_account.md (89%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_site.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_store_goods.md (89%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_table.md (89%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dim_tenant_goods.md (70%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_assistant_service_log.md (87%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_assistant_trash_event.md (82%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_groupbuy_redemption.md (89%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_member_balance_change.md (82%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_payment.md (97%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_platform_coupon_redemption.md (86%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_recharge_order.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_refund.md (100%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_settlement_head.md (78%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_store_goods_sale.md (88%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_table_fee_adjust.md (84%) rename etl_billiards/docs/bd_manual/{ => DWD}/main/BD_manual_dwd_table_fee_log.md (87%) create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_cfg_area_category.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_cfg_assistant_level_price.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_cfg_bonus_rules.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_cfg_performance_tier.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_cfg_skill_type.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_customer_stats.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_daily_detail.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_finance_analysis.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_monthly_summary.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_recharge_commission.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_salary_calc.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_daily_summary.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_discount_detail.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_expense_summary.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_income_structure.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_recharge_summary.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_consumption_summary.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_visit_detail.md create mode 100644 etl_billiards/docs/bd_manual/dws/BD_manual_dws_platform_settlement.md create mode 100644 etl_billiards/docs/dws_tables_dictionary.md create mode 100644 etl_billiards/docs/index_tables.md create mode 100644 etl_billiards/docs/index_tables_output.txt create mode 100644 etl_billiards/docs/补充更多信息.md create mode 100644 etl_billiards/docs/财务页面需求.md create mode 100644 etl_billiards/scripts/analyze_discount_patterns.py create mode 100644 etl_billiards/scripts/analyze_member_discount_usage.py create mode 100644 etl_billiards/scripts/check_assistant_dim.py create mode 100644 etl_billiards/scripts/check_dwd_service.py create mode 100644 etl_billiards/scripts/check_intimacy_stats.py create mode 100644 etl_billiards/scripts/create_index_tables.py create mode 100644 etl_billiards/scripts/import_dws_excel.py create mode 100644 etl_billiards/scripts/run_seed_dws_config.py create mode 100644 etl_billiards/scripts/show_area_category.py create mode 100644 etl_billiards/scripts/show_performance_tier.py create mode 100644 etl_billiards/scripts/test_index_tasks.py create mode 100644 etl_billiards/scripts/verify_dws_config.py create mode 100644 etl_billiards/tasks/dws/__init__.py create mode 100644 etl_billiards/tasks/dws/assistant_customer_task.py create mode 100644 etl_billiards/tasks/dws/assistant_daily_task.py create mode 100644 etl_billiards/tasks/dws/assistant_finance_task.py create mode 100644 etl_billiards/tasks/dws/assistant_monthly_task.py create mode 100644 etl_billiards/tasks/dws/assistant_salary_task.py create mode 100644 etl_billiards/tasks/dws/base_dws_task.py create mode 100644 etl_billiards/tasks/dws/finance_daily_task.py create mode 100644 etl_billiards/tasks/dws/finance_discount_task.py create mode 100644 etl_billiards/tasks/dws/finance_income_task.py create mode 100644 etl_billiards/tasks/dws/finance_recharge_task.py create mode 100644 etl_billiards/tasks/dws/index/__init__.py create mode 100644 etl_billiards/tasks/dws/index/base_index_task.py create mode 100644 etl_billiards/tasks/dws/index/intimacy_index_task.py create mode 100644 etl_billiards/tasks/dws/index/recall_index_task.py create mode 100644 etl_billiards/tasks/dws/member_consumption_task.py create mode 100644 etl_billiards/tasks/dws/member_visit_task.py create mode 100644 etl_billiards/tasks/dws/retention_cleanup_task.py create mode 100644 etl_billiards/tasks/seed_dws_config_task.py create mode 100644 etl_billiards/tests/test_dws_tasks.py create mode 100644 scripts/backfill_202507_to_now.bat create mode 100644 tmp/Untitled create mode 100644 tmp/add_missing_dwd_columns.py create mode 100644 tmp/add_missing_ods_columns.py create mode 100644 tmp/add_remaining_dwd_columns.py create mode 100644 tmp/api_ods_comparison.json create mode 100644 tmp/api_ods_issue_report.json create mode 100644 tmp/backfill_dwd_from_ods.py create mode 100644 tmp/backfill_ods_from_payload.py create mode 100644 tmp/bd_manual_diff.json create mode 100644 tmp/check_api_ods_issues.py create mode 100644 tmp/check_ddl_vs_db.py create mode 100644 tmp/check_field_variants.py create mode 100644 tmp/check_new_fields_data.py create mode 100644 tmp/check_scd2_tables.py create mode 100644 tmp/check_seq.py create mode 100644 tmp/compare_api_ods_fields.py create mode 100644 tmp/detailed_field_compare.py create mode 100644 tmp/dwd_schema.json create mode 100644 tmp/field_coverage_report.json create mode 100644 tmp/fix_bd_manual.py create mode 100644 tmp/fix_not_sale_type.py create mode 100644 tmp/fix_remaining_issues.py create mode 100644 tmp/full_reload_validation.py create mode 100644 tmp/get_dwd_schema.py create mode 100644 tmp/list_dwd_tables.py create mode 100644 tmp/query_schema.py create mode 100644 tmp/schema_output.txt create mode 100644 tmp/sync_api_to_ods_columns.py create mode 100644 tmp/sync_bd_manual.py create mode 100644 tmp/sync_dwd_columns_log.json create mode 100644 tmp/sync_ods_columns_log.json create mode 100644 tmp/sync_ods_to_dwd_columns.py create mode 100644 tmp/test_backfill_feature.py create mode 100644 tmp/test_conflict_modes.py diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 0000000..cdb3c8d --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "csharpier": { + "version": "1.2.5", + "commands": [ + "csharpier" + ], + "rollForward": false + } + } +} \ No newline at end of file diff --git a/README.md b/README.md index 0667c86..74c8156 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,46 @@ python -m cli.main \ - `FETCH_ONLY`：仅在线抓取落盘，不入库 - `INGEST_ONLY`：仅从本地 JSON 回放入库（适合离线回放/补跑） +## DWS 层（汇总/财务） + +### 建表与初始化 +- 建表：`INIT_DWS_SCHEMA` +- 配置：`SEED_DWS_CONFIG` +- 订单汇总（可选）：`DWS_BUILD_ORDER_SUMMARY` + +### 任务与调度建议 +- **每小时**：`DWS_ASSISTANT_DAILY`、`DWS_FINANCE_DAILY`、`DWS_FINANCE_INCOME_STRUCTURE` +- **每日**：`DWS_ASSISTANT_MONTHLY`、`DWS_ASSISTANT_CUSTOMER`、`DWS_MEMBER_CONSUMPTION`、`DWS_MEMBER_VISIT`、`DWS_FINANCE_DISCOUNT_DETAIL`、`DWS_FINANCE_RECHARGE`、`DWS_ASSISTANT_FINANCE` +- **每月（月初）**：`DWS_ASSISTANT_SALARY` +- **维护清理（按需）**：`DWS_RETENTION_CLEANUP` + +调度配置默认保存在 `etl_billiards/scheduled_tasks.json`，GUI 调度器会读取该文件。 + +### 时间口径 +- 周起始日：周一 +- 月/季度起始：第一天 0 点 +- 环比：对比上一个等长区间 +- 窗口类型：本周/上周/本月/上月/前3个月不含本月/前3个月含本月/本季度/上季度/最近半年不含本月 + +### Excel 导入（支出/平台回款/充值提成） +脚本：`etl_billiards/scripts/import_dws_excel.py` +- 支出结构：`--type expense`，按月导入（房租/水电/物业/工资/报销/平台服务费等） +- 平台回款：`--type platform`，按回款日期导入（回款金额、佣金、服务费、订单号等） +- 充值提成：`--type commission`，按月份导入（助教ID、提成金额、充值订单金额等） + +### 大客户优惠拆分（可选） +用于将手动调整拆分为“大客户优惠/其他优惠”，可在配置中指定： +- `dws.discount.big_customer_member_ids`：会员ID列表（逗号分隔） +- `dws.discount.big_customer_order_ids`：订单ID列表（逗号分隔） +未配置时，大客户金额为 0，手动调整全部计入“其他优惠”。 + +### 时间分层清理（可选） +任务：`DWS_RETENTION_CLEANUP`，按配置清理历史数据 +- `dws.retention.enabled`：是否启用 +- `dws.retention.layer`：分层（如 `LAST_3_MONTHS`） +- `dws.retention.tables`：需要清理的表列表（逗号分隔） +- `dws.retention.table_layers`：表级分层覆盖（JSON 字符串） + ## 目录结构与关键文件 - 仓库根目录：`etl_billiards/` 主代码；`app/` 示例 runner；`开发笔记/` 项目笔记；`tmp/` 草稿/调试归档；`requirements.txt`（仓库根）依赖；`run_etl.sh/.bat` 启动脚本。 - 注意：根目录的 `run_etl.sh/.bat` 运行时要求当前目录为 `etl_billiards/`（因为入口是 `python -m cli.main`）。 @@ -367,6 +407,95 @@ python -m cli.main \ 6) 去嵌套：数组展开为子表/子行，重复 profile 提炼为维度。 7) 长期演进：优先加列/加表，减少对已有表结构的破坏。 +## DWS 数据层（汇总层） + +DWS（Data Warehouse Service）层基于 DWD 明细层数据构建，提供预聚合的数据服务。 + +### 表结构概览 + +| 分类 | 表数量 | 说明 | +|------|--------|------| +| 配置表 | 5 | 绩效档位、等级定价、奖金规则、区域分类、技能映射 | +| 助教维度 | 5 | 日度/月度业绩、客户统计、工资计算、充值提成 | +| 客户维度 | 2 | 消费汇总、来店明细 | +| 财务维度 | 7 | 日度汇总、收入结构、优惠明细、充值统计、支出、助教收支、平台结算 | +| 订单汇总 | 1 | 订单级别聚合 | + +### 核心特性 + +- **时间分层**：支持近2天/近1月/近3月/全量的时间窗口筛选 +- **滚动窗口**：支持7/10/15/30/60/90天的滚动统计 +- **SCD2 as-of**：维度取值支持按时间点获取历史值（如助教等级） +- **幂等更新**：采用 delete-before-insert 策略，支持重复执行 +- **Excel导入**：支出/平台结算/充值提成支持手动导入 + +### 助教工资计算 + +**绩效档位（6档 + 新入职）** + +| 档位 | 业绩阈值 | 专业课抽成 | 打赏课抽成 | 休假 | +|------|----------|-----------|-----------|------| +| T0 | H < 100 | 28元/时 | 50% | 3天 | +| T1 | 100 ≤ H < 130 | 18元/时 | 40% | 4天 | +| T2 | 130 ≤ H < 160 | 15元/时 | 38% | 4天 | +| T3 | 160 ≤ H < 190 | 13元/时 | 35% | 5天 | +| T4 | 190 ≤ H < 220 | 10元/时 | 33% | 6天 | +| T5 | H ≥ 220 | 8元/时 | 30% | 休假自由 | + +**工资计算公式** + +``` +基础课收入 = 基础课小时数 × (客户支付价格 - 专业课抽成) +附加课收入 = 附加课小时数 × 190 × (1 - 打赏课抽成比例) +应发工资 = 课时收入 + 奖金 +``` + +**计算示例（中级助教185小时，3档）** +- 基础课170小时: 170 × (108 - 13) = 16,150元 +- 附加课15小时: 15 × 190 × (1 - 0.35) = 1,852.5元 +- 课时收入: 18,002.5元 + +**等级定价（客户支付价格）** + +| 等级 | 基础课价格 | 附加课价格 | +|------|-----------|-----------| +| 初级 | 98元/时 | 190元/时 | +| 中级 | 108元/时 | 190元/时 | +| 高级 | 118元/时 | 190元/时 | +| 星级 | 138元/时 | 190元/时 | + +### 运行 DWS 任务 + +```bash +# 初始化 DWS Schema +python -m cli.main --tasks INIT_DWS_SCHEMA + +# 执行配置数据初始化 +psql -f etl_billiards/database/seed_dws_config.sql + +# 执行 DWS 订单汇总构建 +python -m cli.main --tasks DWS_BUILD_ORDER_SUMMARY +``` + +### Excel 数据导入 + +```bash +# 导入支出数据 +python etl_billiards/scripts/import_dws_excel.py --type expense --file expenses.xlsx + +# 导入平台结算 +python etl_billiards/scripts/import_dws_excel.py --type platform --file platform.xlsx + +# 导入充值提成 +python etl_billiards/scripts/import_dws_excel.py --type commission --file commission.xlsx +``` + +### 相关文档 + +- `etl_billiards/docs/dws_tables_dictionary.md`：DWS 数据字典 +- `etl_billiards/database/schema_dws.sql`：DWS DDL +- `etl_billiards/database/seed_dws_config.sql`：配置初始数据 + ## 常用 CLI ```bash cd etl_billiards @@ -474,11 +603,37 @@ python scripts/test_db_connection.py --dsn "postgresql://user:pwd@host:5432/db" > 完整字段级映射见 `etl_billiards/docs/` 与 ODS/DWD DDL。 -## 当前状态（2025-12-09） +## 当前状态（2026-02-02） - 示例 JSON 已全量灌入，DWD 行数与 ODS 对齐。 - 分类维度已展平大类+子类：`dim_goods_category` 26 行（category_level/leaf 已赋值）。 - 部分空字段源数据即为空，如需补值请先确认上游。 +### 2026-02-02 更新：字段补全 + +本次更新完成了 API → ODS → DWD 全链路字段补全： + +**ODS 新增字段**（16 张表，共 50+ 字段）： +- `settlement_records`/`recharge_settlements`：电费相关字段（`electricitymoney`、`realelectricitymoney`、`electricityadjustmoney`）、券销售额、结算明细列表 +- `table_fee_transactions`：活动折扣金额、订单消费类型、实际服务费 +- `assistant_service_records`：助教团队名称、实际服务费 +- `group_buy_redemption_records`：会员折扣、各类分摊金额（台费/商品/助教/充值） +- `table_fee_discount_records`：台区信息、台桌名称/价格、免费标记 +- `member_stored_value_cards`：本金余额、会员等级、电费相关配置 +- `member_profiles`：累计支付/充值金额、注册来源 +- `member_balance_changes`：本金变动（前/后/数据） +- `group_buy_packages`：排序、首单限制、租户券销售订单项ID +- 其他：商品编码/停售、租户ID等 + +**DWD 新增字段**： +- 主表新增核心业务字段（金额、ID、状态） +- 扩展表新增配置/明细字段 + +**数据补全脚本**： +- `scripts/backfill_202507_to_now.bat`：从 2025-07-01 重新抓取并装载数据 + +**文档更新**： +- `etl_billiards/docs/bd_manual/` 下所有相关表文档已同步更新 + ## 可精简/归档 - `tmp/`、`tmp/etl_billiards_misc/` 中草稿、旧备份、调试脚本仅供参考，不影响运行。 - 根级保留必要文件（README、requirements、run_etl.*），其余临时文件按需归档至 `tmp/`。 diff --git a/etl_billiards/.env b/etl_billiards/.env index 351a96e..68ca98b 100644 --- a/etl_billiards/.env +++ b/etl_billiards/.env @@ -32,7 +32,7 @@ SCHEMA_ETL=etl_admin # API 配置 # ------------------------------------------------------------------------------ API_BASE=https://pc.ficoo.vip/apiprod/admin/v1/ -API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IktlbTVsdHRqZ2tSUExOcVA2ajhNakdQYnFrNW5mRzBQNzRvMHE0b295VVE9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzIvOCDkuIvljYg2OjU3OjA1IiwibmVlZENoZWNrVG9rZW4iOiJmYWxzZSIsImV4cCI6MTc3MDU0ODIyNSwiaXNzIjoidGVzdCIsImF1ZCI6IlVzZXIifQ.wJlm7pTqUzp769nUGdxx0e1bVMy4x9Prp9U_UMWQvlk +API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IjlES1lWcEVkYWw1bEc5cTMrdFptMkJXeTlyMkVMeEY5MHZuUWRyRnNYVFU9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzIvOSDkuIrljYgyOjQzOjU0IiwibmVlZENoZWNrVG9rZW4iOiJmYWxzZSIsImV4cCI6MTc3MDU3NjIzNCwiaXNzIjoidGVzdCIsImF1ZCI6IlVzZXIifQ._1gnWcJHw8O26pcfiT1x8tgQRGn3g56vv2IZP8shgGU # API 请求超时（秒） API_TIMEOUT=20 diff --git a/etl_billiards/database/schema_ODS_doc.sql b/etl_billiards/database/schema_ODS_doc.sql index 77db312..13db67a 100644 --- a/etl_billiards/database/schema_ODS_doc.sql +++ b/etl_billiards/database/schema_ODS_doc.sql @@ -19,6 +19,11 @@ CREATE TABLE IF NOT EXISTS billiards_ods.member_profiles ( status INT, user_status INT, create_time TIMESTAMP, + pay_money_sum NUMERIC(18,2), + person_tenant_org_id BIGINT, + person_tenant_org_name TEXT, + recharge_money_sum NUMERIC(18,2), + register_source TEXT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -75,6 +80,9 @@ CREATE TABLE IF NOT EXISTS billiards_ods.member_balance_changes ( operator_name TEXT, is_delete INT, create_time TIMESTAMP, + principal_after NUMERIC(18,2), + principal_before NUMERIC(18,2), + principal_data TEXT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -185,6 +193,13 @@ CREATE TABLE IF NOT EXISTS billiards_ods.member_stored_value_cards ( tenantName TEXT, pdAssisnatLevel TEXT, cxAssisnatLevel TEXT, + able_share_member_discount BOOLEAN, + electricity_deduct_radio NUMERIC(18,4), + electricity_discount NUMERIC(18,4), + electricitycarddeduct BOOLEAN, + member_grade BIGINT, + principal_balance NUMERIC(18,2), + rechargefreezebalance NUMERIC(18,2), content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -331,6 +346,12 @@ CREATE TABLE IF NOT EXISTS billiards_ods.recharge_settlements ( isfirst INT, rechargecardamount NUMERIC(18,2), giftcardamount NUMERIC(18,2), + electricityadjustmoney NUMERIC(18,2), + electricitymoney NUMERIC(18,2), + mervousalesamount NUMERIC(18,2), + plcouponsaleamount NUMERIC(18,2), + realelectricitymoney NUMERIC(18,2), + settlelist JSONB, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -469,6 +490,12 @@ CREATE TABLE IF NOT EXISTS billiards_ods.settlement_records ( isfirst INT, rechargecardamount NUMERIC(18,2), giftcardamount NUMERIC(18,2), + electricityadjustmoney NUMERIC(18,2), + electricitymoney NUMERIC(18,2), + mervousalesamount NUMERIC(18,2), + plcouponsaleamount NUMERIC(18,2), + realelectricitymoney NUMERIC(18,2), + settlelist JSONB, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -559,6 +586,7 @@ CREATE TABLE IF NOT EXISTS billiards_ods.assistant_cancellation_records ( tableName TEXT, trashReason TEXT, createTime TIMESTAMP, + tenant_id BIGINT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -792,6 +820,8 @@ CREATE TABLE IF NOT EXISTS billiards_ods.assistant_service_records ( get_grade_times INT, is_not_responding INT, is_confirm INT, + assistantteamname TEXT, + real_service_money NUMERIC(18,2), payload JSONB NOT NULL, content_hash TEXT NOT NULL, source_file TEXT, @@ -897,6 +927,7 @@ CREATE TABLE IF NOT EXISTS billiards_ods.site_tables_master ( table_status INT, temporary_light_second INT, virtual_table INT, + order_id BIGINT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -957,6 +988,14 @@ CREATE TABLE IF NOT EXISTS billiards_ods.table_fee_discount_records ( order_trade_no TEXT, is_delete INT, create_time TIMESTAMP, + area_type_id BIGINT, + charge_free BOOLEAN, + site_table_area_id BIGINT, + site_table_area_name TEXT, + sitename TEXT, + table_name TEXT, + table_price NUMERIC(18,2), + tenant_name TEXT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -1032,6 +1071,9 @@ CREATE TABLE IF NOT EXISTS billiards_ods.table_fee_transactions ( salesman_org_id BIGINT, salesman_user_id BIGINT, create_time TIMESTAMP, + activity_discount_amount NUMERIC(18,2), + order_consumption_type INT, + real_service_money NUMERIC(18,2), payload JSONB NOT NULL, content_hash TEXT NOT NULL, source_file TEXT, @@ -1234,6 +1276,7 @@ CREATE TABLE IF NOT EXISTS billiards_ods.payment_transactions ( create_time TIMESTAMP, payment_method INT, online_pay_channel INT, + tenant_id BIGINT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -1440,6 +1483,7 @@ CREATE TABLE IF NOT EXISTS billiards_ods.tenant_goods_master ( remark_name TEXT, create_time TIMESTAMP, update_time TIMESTAMP, + not_sale BOOLEAN, payload JSONB NOT NULL, content_hash TEXT NOT NULL, source_file TEXT, @@ -1522,6 +1566,9 @@ CREATE TABLE IF NOT EXISTS billiards_ods.group_buy_packages ( area_tag_type INT, creator_name TEXT, create_time TIMESTAMP, + is_first_limit BOOLEAN, + sort INT, + tenantcouponsaleorderitemid BIGINT, content_hash TEXT NOT NULL, source_file TEXT, source_endpoint TEXT, @@ -1616,6 +1663,15 @@ CREATE TABLE IF NOT EXISTS billiards_ods.group_buy_redemption_records ( is_single_order INT, is_delete INT, create_time TIMESTAMP, + assistant_service_share_money NUMERIC(18,2), + assistant_share_money NUMERIC(18,2), + coupon_sale_id BIGINT, + good_service_share_money NUMERIC(18,2), + goods_share_money NUMERIC(18,2), + member_discount_money NUMERIC(18,2), + recharge_share_money NUMERIC(18,2), + table_service_share_money NUMERIC(18,2), + table_share_money NUMERIC(18,2), payload JSONB NOT NULL, content_hash TEXT NOT NULL, source_file TEXT, @@ -1812,6 +1868,8 @@ CREATE TABLE IF NOT EXISTS billiards_ods.store_goods_master ( goods_cover TEXT, create_time TIMESTAMP, update_time TIMESTAMP, + commodity_code TEXT, + not_sale INTEGER, payload JSONB NOT NULL, content_hash TEXT NOT NULL, source_file TEXT, @@ -1923,6 +1981,7 @@ CREATE TABLE IF NOT EXISTS billiards_ods.store_goods_sales_records ( tenant_goods_business_id BIGINT, tenant_goods_category_id BIGINT, create_time TIMESTAMP, + coupon_share_money NUMERIC(18,2), payload JSONB NOT NULL, content_hash TEXT NOT NULL, source_file TEXT, diff --git a/etl_billiards/database/schema_dwd_doc.sql b/etl_billiards/database/schema_dwd_doc.sql index ab99a10..cfbfd69 100644 --- a/etl_billiards/database/schema_dwd_doc.sql +++ b/etl_billiards/database/schema_dwd_doc.sql @@ -107,7 +107,7 @@ COMMENT ON COLUMN billiards_dwd.dim_site.scd2_is_current IS '【说明】SCD2 COMMENT ON COLUMN billiards_dwd.dim_site.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】table_fee_transactions - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_site_Ex ( +CREATE TABLE IF NOT EXISTS dim_site_ex ( site_id BIGINT, avatar TEXT, address TEXT, @@ -172,6 +172,7 @@ CREATE TABLE IF NOT EXISTS dim_table ( site_table_area_name TEXT, tenant_table_area_id BIGINT, table_price NUMERIC(18,2), + order_id BIGINT, SCD2_start_time TIMESTAMPTZ DEFAULT now(), SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31', SCD2_is_current INT DEFAULT 1, @@ -193,7 +194,7 @@ COMMENT ON COLUMN billiards_dwd.dim_table.scd2_is_current IS '【说明】SCD2 COMMENT ON COLUMN billiards_dwd.dim_table.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】site_tables_master - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_table_Ex ( +CREATE TABLE IF NOT EXISTS dim_table_ex ( table_id BIGINT, show_status INTEGER, is_online_reservation INTEGER, @@ -265,7 +266,7 @@ COMMENT ON COLUMN billiards_dwd.dim_assistant.scd2_is_current IS '【说明】SC COMMENT ON COLUMN billiards_dwd.dim_assistant.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】assistant_accounts_master - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_assistant_Ex ( +CREATE TABLE IF NOT EXISTS dim_assistant_ex ( assistant_id BIGINT, gender INTEGER, birth_date TIMESTAMPTZ, @@ -379,6 +380,8 @@ CREATE TABLE IF NOT EXISTS dim_member ( member_card_grade_name TEXT, create_time TIMESTAMPTZ, update_time TIMESTAMPTZ, + pay_money_sum NUMERIC(18,2), + recharge_money_sum NUMERIC(18,2), SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -403,7 +406,7 @@ COMMENT ON COLUMN billiards_dwd.dim_member.scd2_is_current IS '【说明】SCD2 COMMENT ON COLUMN billiards_dwd.dim_member.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】member_profiles - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_member_Ex ( +CREATE TABLE IF NOT EXISTS dim_member_ex ( member_id BIGINT, referrer_member_id BIGINT, point NUMERIC(18,2), @@ -411,6 +414,9 @@ CREATE TABLE IF NOT EXISTS dim_member_Ex ( growth_value NUMERIC(18,2), user_status INTEGER, status INTEGER, + person_tenant_org_id BIGINT, + person_tenant_org_name TEXT, + register_source TEXT, SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -450,6 +456,8 @@ CREATE TABLE IF NOT EXISTS dim_member_card_account ( last_consume_time TIMESTAMPTZ, status INTEGER, is_delete INTEGER, + principal_balance NUMERIC(18,2), + member_grade BIGINT, SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -481,7 +489,7 @@ COMMENT ON COLUMN billiards_dwd.dim_member_card_account.scd2_is_current IS '【 COMMENT ON COLUMN billiards_dwd.dim_member_card_account.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】member_stored_value_cards - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_member_card_account_Ex ( +CREATE TABLE IF NOT EXISTS dim_member_card_account_ex ( member_card_id BIGINT, site_name TEXT, tenant_name VARCHAR(64), @@ -534,6 +542,11 @@ CREATE TABLE IF NOT EXISTS dim_member_card_account_Ex ( goodsCategoryId TEXT, pdAssisnatLevel TEXT, cxAssisnatLevel TEXT, + able_share_member_discount BOOLEAN, + electricity_deduct_radio NUMERIC(18,4), + electricity_discount NUMERIC(18,4), + electricity_card_deduct BOOLEAN, + recharge_freeze_balance NUMERIC(18,2), SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -615,6 +628,7 @@ CREATE TABLE IF NOT EXISTS dim_tenant_goods ( create_time TIMESTAMPTZ, update_time TIMESTAMPTZ, is_delete INTEGER, + not_sale INTEGER, SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -643,7 +657,7 @@ COMMENT ON COLUMN billiards_dwd.dim_tenant_goods.scd2_is_current IS '【说明 COMMENT ON COLUMN billiards_dwd.dim_tenant_goods.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】tenant_goods_master - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_tenant_goods_Ex ( +CREATE TABLE IF NOT EXISTS dim_tenant_goods_ex ( tenant_goods_id BIGINT, remark_name VARCHAR(128), pinyin_initial VARCHAR(128), @@ -715,6 +729,8 @@ CREATE TABLE IF NOT EXISTS dim_store_goods ( enable_status INTEGER, send_state INTEGER, is_delete INTEGER, + commodity_code TEXT, + not_sale INTEGER, SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -749,7 +765,7 @@ COMMENT ON COLUMN billiards_dwd.dim_store_goods.scd2_is_current IS '【说明】 COMMENT ON COLUMN billiards_dwd.dim_store_goods.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】store_goods_master - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_store_goods_Ex ( +CREATE TABLE IF NOT EXISTS dim_store_goods_ex ( site_goods_id BIGINT, site_name TEXT, unit TEXT, @@ -872,6 +888,8 @@ CREATE TABLE IF NOT EXISTS dim_groupbuy_package ( create_time TIMESTAMPTZ, tenant_table_area_id_list VARCHAR(512), card_type_ids VARCHAR(255), + sort INTEGER, + is_first_limit BOOLEAN, SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -902,7 +920,7 @@ COMMENT ON COLUMN billiards_dwd.dim_groupbuy_package.scd2_is_current IS '【说 COMMENT ON COLUMN billiards_dwd.dim_groupbuy_package.scd2_version IS '【说明】SCD2 版本号（自增），用于与时间段一起避免版本重叠。 【示例】1（SCD2 版本号（自增），用于与时间段一起避免版本重叠）。 【ODS来源】group_buy_packages - 无（DWD慢变元数据）。 【JSON字段】无 - DWD慢变元数据 - 无。'; -CREATE TABLE IF NOT EXISTS dim_groupbuy_package_Ex ( +CREATE TABLE IF NOT EXISTS dim_groupbuy_package_ex ( groupbuy_package_id BIGINT, site_name VARCHAR(100), usable_count INTEGER, @@ -923,6 +941,7 @@ CREATE TABLE IF NOT EXISTS dim_groupbuy_package_Ex ( effective_status INTEGER, max_selectable_categories INTEGER, creator_name VARCHAR(100), + tenant_coupon_sale_order_item_id BIGINT, SCD2_start_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ, SCD2_is_current INT, @@ -990,6 +1009,11 @@ CREATE TABLE IF NOT EXISTS dwd_settlement_head ( coupon_amount NUMERIC(18,2), rounding_amount NUMERIC(18,2), point_amount NUMERIC(18,2), + electricity_money NUMERIC(18,2), + real_electricity_money NUMERIC(18,2), + electricity_adjust_money NUMERIC(18,2), + pl_coupon_sale_amount NUMERIC(18,2), + mervou_sales_amount NUMERIC(18,2), PRIMARY KEY (order_settle_id) ); @@ -1028,7 +1052,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_settlement_head.rounding_amount IS '【说 COMMENT ON COLUMN billiards_dwd.dwd_settlement_head.point_amount IS '【说明】金额字段，用于计费/结算/核算等金额计算。 【示例】NULL（金额字段，用于计费/结算/核算等金额计算）。 【ODS来源】settlement_records - pointamount。 【JSON字段】settlement_records.json - $ - pointamount。'; -CREATE TABLE IF NOT EXISTS dwd_settlement_head_Ex ( +CREATE TABLE IF NOT EXISTS dwd_settlement_head_ex ( order_settle_id BIGINT, serial_number INTEGER, settle_status INTEGER, @@ -1059,6 +1083,7 @@ CREATE TABLE IF NOT EXISTS dwd_settlement_head_Ex ( order_remark VARCHAR(255), operator_id BIGINT, salesman_user_id BIGINT, + settle_list JSONB, PRIMARY KEY (order_settle_id) ); @@ -1123,6 +1148,8 @@ CREATE TABLE IF NOT EXISTS dwd_table_fee_log ( ledger_status INTEGER, is_single_order INTEGER, is_delete INTEGER, + activity_discount_amount NUMERIC(18,2), + real_service_money NUMERIC(18,2), PRIMARY KEY (table_fee_log_id) ); @@ -1156,7 +1183,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_table_fee_log.is_single_order IS '【说明 COMMENT ON COLUMN billiards_dwd.dwd_table_fee_log.is_delete IS '【说明】布尔/开关字段，用于表示是否/可用性等业务开关。 【示例】0（布尔/开关字段，用于表示是否/可用性等业务开关）。 【ODS来源】table_fee_transactions - is_delete。 【JSON字段】table_fee_transactions.json - data.siteTableUseDetailsList - is_delete。'; -CREATE TABLE IF NOT EXISTS dwd_table_fee_log_Ex ( +CREATE TABLE IF NOT EXISTS dwd_table_fee_log_ex ( table_fee_log_id BIGINT, operator_name VARCHAR(64), salesman_name VARCHAR(64), @@ -1169,6 +1196,7 @@ CREATE TABLE IF NOT EXISTS dwd_table_fee_log_Ex ( operator_id BIGINT, salesman_user_id BIGINT, salesman_org_id BIGINT, + order_consumption_type INTEGER, PRIMARY KEY (table_fee_log_id) ); @@ -1201,6 +1229,9 @@ CREATE TABLE IF NOT EXISTS dwd_table_fee_adjust ( ledger_status INTEGER, is_delete INTEGER, adjust_time TIMESTAMPTZ, + table_name TEXT, + table_price NUMERIC(18,2), + charge_free BOOLEAN, PRIMARY KEY (table_fee_adjust_id) ); @@ -1220,7 +1251,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_table_fee_adjust.is_delete IS '【说明】 COMMENT ON COLUMN billiards_dwd.dwd_table_fee_adjust.adjust_time IS '【说明】时间/日期字段，用于记录业务时间与统计口径对齐。 【示例】2025-11-09 23:25:11（时间/日期字段，用于记录业务时间与统计口径对齐）。 【ODS来源】table_fee_discount_records - create_time。 【JSON字段】table_fee_discount_records.json - data.taiFeeAdjustInfos - create_time。'; -CREATE TABLE IF NOT EXISTS dwd_table_fee_adjust_Ex ( +CREATE TABLE IF NOT EXISTS dwd_table_fee_adjust_ex ( table_fee_adjust_id BIGINT, adjust_type INTEGER, ledger_count INTEGER, @@ -1229,6 +1260,11 @@ CREATE TABLE IF NOT EXISTS dwd_table_fee_adjust_Ex ( operator_name VARCHAR(64), applicant_id BIGINT, operator_id BIGINT, + area_type_id BIGINT, + site_table_area_id BIGINT, + site_table_area_name TEXT, + site_name TEXT, + tenant_name TEXT, PRIMARY KEY (table_fee_adjust_id) ); @@ -1267,6 +1303,7 @@ CREATE TABLE IF NOT EXISTS dwd_store_goods_sale ( ledger_status INTEGER, is_delete INTEGER, create_time TIMESTAMPTZ, + coupon_share_money NUMERIC(18,2), PRIMARY KEY (store_goods_sale_id) ); @@ -1296,7 +1333,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_store_goods_sale.is_delete IS '【说明】 COMMENT ON COLUMN billiards_dwd.dwd_store_goods_sale.create_time IS '【说明】时间/日期字段，用于记录业务时间与统计口径对齐。 【示例】2025-11-09 23:35:57（时间/日期字段，用于记录业务时间与统计口径对齐）。 【ODS来源】store_goods_sales_records - create_time。 【JSON字段】store_goods_sales_records.json - data.orderGoodsLedgers - create_time。'; -CREATE TABLE IF NOT EXISTS dwd_store_goods_sale_Ex ( +CREATE TABLE IF NOT EXISTS dwd_store_goods_sale_ex ( store_goods_sale_id BIGINT, legacy_order_goods_id BIGINT, site_name TEXT, @@ -1392,6 +1429,7 @@ CREATE TABLE IF NOT EXISTS dwd_assistant_service_log ( start_use_time TIMESTAMPTZ, last_use_time TIMESTAMPTZ, is_delete INTEGER, + real_service_money NUMERIC(18,2), PRIMARY KEY (assistant_service_id) ); @@ -1430,7 +1468,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_assistant_service_log.last_use_time IS '【 COMMENT ON COLUMN billiards_dwd.dwd_assistant_service_log.is_delete IS '【说明】布尔/开关字段，用于表示是否/可用性等业务开关。 【示例】0（布尔/开关字段，用于表示是否/可用性等业务开关）。 【ODS来源】assistant_service_records - is_delete。 【JSON字段】assistant_service_records.json - data.orderAssistantDetails - is_delete。'; -CREATE TABLE IF NOT EXISTS dwd_assistant_service_log_Ex ( +CREATE TABLE IF NOT EXISTS dwd_assistant_service_log_ex ( assistant_service_id BIGINT, table_name VARCHAR(64), assistant_name VARCHAR(64), @@ -1461,6 +1499,7 @@ CREATE TABLE IF NOT EXISTS dwd_assistant_service_log_Ex ( get_grade_times INTEGER, grade_status INTEGER, composite_grade_time TIMESTAMPTZ, + assistant_team_name TEXT, PRIMARY KEY (assistant_service_id) ); @@ -1508,6 +1547,7 @@ CREATE TABLE IF NOT EXISTS dwd_assistant_trash_event ( abolish_amount NUMERIC(18,2), trash_reason VARCHAR(255), create_time TIMESTAMPTZ, + tenant_id BIGINT, PRIMARY KEY (assistant_trash_event_id) ); @@ -1524,7 +1564,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_assistant_trash_event.trash_reason IS '【 COMMENT ON COLUMN billiards_dwd.dwd_assistant_trash_event.create_time IS '【说明】时间/日期字段，用于记录业务时间与统计口径对齐。 【示例】2025-11-09 19:23:29（时间/日期字段，用于记录业务时间与统计口径对齐）。 【ODS来源】assistant_cancellation_records - createTime。 【JSON字段】assistant_cancellation_records.json - data.abolitionAssistants - createTime。'; -CREATE TABLE IF NOT EXISTS dwd_assistant_trash_event_Ex ( +CREATE TABLE IF NOT EXISTS dwd_assistant_trash_event_ex ( assistant_trash_event_id BIGINT, table_name VARCHAR(64), table_area_name VARCHAR(64), @@ -1557,6 +1597,8 @@ CREATE TABLE IF NOT EXISTS dwd_member_balance_change ( change_time TIMESTAMPTZ, is_delete INTEGER, remark VARCHAR(255), + principal_before NUMERIC(18,2), + principal_after NUMERIC(18,2), PRIMARY KEY (balance_change_id) ); @@ -1582,13 +1624,14 @@ COMMENT ON COLUMN billiards_dwd.dwd_member_balance_change.is_delete IS '【说 COMMENT ON COLUMN billiards_dwd.dwd_member_balance_change.remark IS '【说明】明细字段，用于记录事实取值。 【示例】充值退款（明细字段，用于记录事实取值）。 【ODS来源】member_balance_changes - remark。 【JSON字段】member_balance_changes.json - data.tenantMemberCardLogs - remark。'; -CREATE TABLE IF NOT EXISTS dwd_member_balance_change_EX ( +CREATE TABLE IF NOT EXISTS dwd_member_balance_change_ex ( balance_change_id BIGINT, pay_site_name VARCHAR(64), register_site_name VARCHAR(64), refund_amount NUMERIC(18,2), operator_id BIGINT, operator_name VARCHAR(64), + principal_data TEXT, PRIMARY KEY (balance_change_id) ); @@ -1625,6 +1668,8 @@ CREATE TABLE IF NOT EXISTS dwd_groupbuy_redemption ( is_delete INTEGER, ledger_name VARCHAR(128), create_time TIMESTAMPTZ, + member_discount_money NUMERIC(18,2), + coupon_sale_id BIGINT, PRIMARY KEY (redemption_id) ); @@ -1654,7 +1699,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_groupbuy_redemption.ledger_name IS '【说 COMMENT ON COLUMN billiards_dwd.dwd_groupbuy_redemption.create_time IS '【说明】时间/日期字段，用于记录业务时间与统计口径对齐。 【示例】2025-11-09 23:35:57（时间/日期字段，用于记录业务时间与统计口径对齐）。 【ODS来源】group_buy_redemption_records - create_time。 【JSON字段】group_buy_redemption_records.json - data.siteTableUseDetailsList - create_time。'; -CREATE TABLE IF NOT EXISTS dwd_groupbuy_redemption_Ex ( +CREATE TABLE IF NOT EXISTS dwd_groupbuy_redemption_ex ( redemption_id BIGINT, site_name VARCHAR(64), table_name VARCHAR(64), @@ -1676,6 +1721,13 @@ CREATE TABLE IF NOT EXISTS dwd_groupbuy_redemption_Ex ( salesman_role_id BIGINT, salesman_org_id BIGINT, ledger_group_name VARCHAR(128), + table_share_money NUMERIC(18,2), + table_service_share_money NUMERIC(18,2), + goods_share_money NUMERIC(18,2), + good_service_share_money NUMERIC(18,2), + assistant_share_money NUMERIC(18,2), + assistant_service_share_money NUMERIC(18,2), + recharge_share_money NUMERIC(18,2), PRIMARY KEY (redemption_id) ); @@ -1750,7 +1802,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_platform_coupon_redemption.create_time IS ' COMMENT ON COLUMN billiards_dwd.dwd_platform_coupon_redemption.consume_time IS '【说明】时间/日期字段，用于记录业务时间与统计口径对齐。 【示例】2025-11-09 23:41:04（时间/日期字段，用于记录业务时间与统计口径对齐）。 【ODS来源】platform_coupon_redemption_records - consume_time。 【JSON字段】platform_coupon_redemption_records.json - $ - consume_time。'; -CREATE TABLE IF NOT EXISTS dwd_platform_coupon_redemption_Ex ( +CREATE TABLE IF NOT EXISTS dwd_platform_coupon_redemption_ex ( platform_coupon_redemption_id BIGINT, coupon_cover VARCHAR(255), coupon_remark VARCHAR(255), @@ -1814,7 +1866,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_recharge_order.create_time IS '【说明】 COMMENT ON COLUMN billiards_dwd.dwd_recharge_order.pay_time IS '【说明】时间/日期字段，用于记录业务时间与统计口径对齐。 【示例】NULL（时间/日期字段，用于记录业务时间与统计口径对齐）。 【ODS来源】recharge_settlements - paytime。 【JSON字段】recharge_settlements.json - $ - paytime。'; -CREATE TABLE IF NOT EXISTS dwd_recharge_order_Ex ( +CREATE TABLE IF NOT EXISTS dwd_recharge_order_ex ( recharge_order_id BIGINT, site_name_snapshot TEXT, settle_status INTEGER, @@ -1919,6 +1971,7 @@ CREATE TABLE IF NOT EXISTS dwd_payment ( create_time TIMESTAMPTZ, pay_time TIMESTAMPTZ, pay_date DATE, + tenant_id BIGINT, PRIMARY KEY (payment_id) ); @@ -1967,7 +2020,7 @@ COMMENT ON COLUMN billiards_dwd.dwd_refund.member_id IS '【说明】标识类 I COMMENT ON COLUMN billiards_dwd.dwd_refund.member_card_id IS '【说明】标识类 ID 字段，用于关联/定位相关实体。 【示例】0（标识类 ID 字段，用于关联/定位相关实体）。 【ODS来源】refund_transactions - member_card_id。 【JSON字段】refund_transactions.json - $ - member_card_id。'; - CREATE TABLE IF NOT EXISTS dwd_refund_Ex ( + CREATE TABLE IF NOT EXISTS dwd_refund_ex ( refund_id BIGINT, tenant_name VARCHAR(64), pay_sn BIGINT, diff --git a/etl_billiards/database/schema_dws.sql b/etl_billiards/database/schema_dws.sql index 03e1a86..524524b 100644 --- a/etl_billiards/database/schema_dws.sql +++ b/etl_billiards/database/schema_dws.sql @@ -1,50 +1,1298 @@ --- DWS schema for aggregated / serving tables. +-- ============================================================================= +-- DWS 数据层完整 DDL +-- 版本: v3.0 +-- 创建日期: 2026-02-01 +-- 描述: 包含配置表(5张)、助教维度(5张)、客户维度(2张)、财务维度(7张)、订单汇总(1张) +-- ============================================================================= + +-- 创建 DWS Schema CREATE SCHEMA IF NOT EXISTS billiards_dws; -CREATE TABLE IF NOT EXISTS billiards_dws.dws_order_summary ( - site_id BIGINT NOT NULL, - order_settle_id BIGINT NOT NULL, - order_trade_no TEXT, - order_date DATE, - tenant_id BIGINT, - member_id BIGINT, - member_flag BOOLEAN, - recharge_order_flag BOOLEAN, - item_count INT, - total_item_quantity NUMERIC, - table_fee_amount NUMERIC, - assistant_service_amount NUMERIC, - goods_amount NUMERIC, - group_amount NUMERIC, - total_coupon_deduction NUMERIC, - member_discount_amount NUMERIC, - manual_discount_amount NUMERIC, - order_original_amount NUMERIC, - order_final_amount NUMERIC, - stored_card_deduct NUMERIC, - external_paid_amount NUMERIC, - total_paid_amount NUMERIC, - book_table_flow NUMERIC, - book_assistant_flow NUMERIC, - book_goods_flow NUMERIC, - book_group_flow NUMERIC, - book_order_flow NUMERIC, - order_effective_consume_cash NUMERIC, - order_effective_recharge_cash NUMERIC, - order_effective_flow NUMERIC, - refund_amount NUMERIC, - net_income NUMERIC, - created_at TIMESTAMPTZ DEFAULT now(), - updated_at TIMESTAMPTZ DEFAULT now(), - PRIMARY KEY (site_id, order_settle_id) +-- ============================================================================= +-- 第一部分：配置表（5张） +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 1. cfg_performance_tier - 绩效档位配置表 +-- 说明: +-- - 助教绩效档位(共6档)配置，包含阈值、抽成比例、假期天数 +-- - 数据来源：DWS 数据库处理需求.md 第35-41行 +-- - 基础课收入 = 基础课小时数 × (客户支付价格 - 专业课抽成) +-- - 附加课收入 = 附加课小时数 × 190 × (1 - 打赏课抽成比例) +-- - 支持按时间生效，通过 effective_from/effective_to 控制历史口径 +-- - 新入职定档规则: 月1日0点之后入职的，计算为新入职 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.cfg_performance_tier CASCADE; +CREATE TABLE billiards_dws.cfg_performance_tier ( + tier_id SERIAL PRIMARY KEY, -- 档位ID（自增） + tier_code VARCHAR(20) NOT NULL, -- 档位代码（T0-T5, NEW） + tier_name VARCHAR(50) NOT NULL, -- 档位名称 + tier_level INTEGER NOT NULL, -- 档位等级（数字越大档位越高） + min_hours NUMERIC(10,2) NOT NULL, -- 最低业绩小时数阈值（>=） + max_hours NUMERIC(10,2), -- 最高业绩小时数阈值（<，NULL表示无上限） + base_deduction NUMERIC(10,2) NOT NULL DEFAULT 0, -- 专业课抽成（元/小时），球房从基础课扣除 + bonus_deduction_ratio NUMERIC(5,4) NOT NULL DEFAULT 0, -- 打赏课抽成比例（0-1），球房从附加课扣除 + vacation_days INTEGER NOT NULL DEFAULT 0, -- 次月可休假天数 + vacation_unlimited BOOLEAN NOT NULL DEFAULT FALSE, -- 是否休假自由（5档特殊） + is_new_hire_tier BOOLEAN NOT NULL DEFAULT FALSE, -- 是否为新入职专用档位 + effective_from DATE NOT NULL DEFAULT '2000-01-01', -- 生效起始日期（含） + effective_to DATE NOT NULL DEFAULT '9999-12-31', -- 生效截止日期（含） + description TEXT, -- 档位说明 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_performance_tier UNIQUE (tier_code, effective_from) ); -CREATE INDEX IF NOT EXISTS idx_dws_order_summary_order_date - ON billiards_dws.dws_order_summary (order_date); +COMMENT ON TABLE billiards_dws.cfg_performance_tier IS '绩效档位配置表：定义6档绩效阈值、抽成比例、假期，数据来源DWS数据库处理需求.md'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.tier_code IS '档位代码：T0-T5普通档位，NEW为新入职专用'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.min_hours IS '业绩小时数下限（含），基础课+附加课总和'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.max_hours IS '业绩小时数上限（不含），NULL表示无上限'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.base_deduction IS '专业课抽成（元/小时）：球房从基础课每小时扣除的金额'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.bonus_deduction_ratio IS '打赏课抽成比例：球房从附加课收入中扣除的比例，如0.35表示35%'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.vacation_days IS '次月可休假天数'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.vacation_unlimited IS '休假自由标记：5档为TRUE'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.is_new_hire_tier IS '新入职档位标记，月1日0点后入职者首月使用'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.effective_from IS '规则生效起始日期，用于历史月份正确取档'; -CREATE INDEX IF NOT EXISTS idx_dws_order_summary_tenant_date - ON billiards_dws.dws_order_summary (tenant_id, order_date); +-- 创建查询索引 +CREATE INDEX idx_cfg_performance_tier_effective + ON billiards_dws.cfg_performance_tier (effective_from, effective_to); -CREATE INDEX IF NOT EXISTS idx_dws_order_summary_member_date - ON billiards_dws.dws_order_summary (member_id, order_date); +-- ----------------------------------------------------------------------------- +-- 2. cfg_assistant_level_price - 助教等级定价表 +-- 说明: +-- - 助教等级（初级/中级/高级/星级）对应的基础课和附加课单价 +-- - 支持按时间生效，便于历史月份薪资计算使用历史单价 +-- - SCD2口径: 助教等级来自dim_assistant，取数时需按有效期as-of join +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.cfg_assistant_level_price CASCADE; +CREATE TABLE billiards_dws.cfg_assistant_level_price ( + price_id SERIAL PRIMARY KEY, -- 定价ID（自增） + level_code INTEGER NOT NULL, -- 等级代码（来自dim_assistant.assistant_level） + level_name VARCHAR(20) NOT NULL, -- 等级名称（初级/中级/高级/星级） + base_course_price NUMERIC(10,2) NOT NULL, -- 基础课单价（元/小时） + bonus_course_price NUMERIC(10,2) NOT NULL, -- 附加课单价（元/小时），固定50元 + effective_from DATE NOT NULL DEFAULT '2000-01-01', -- 生效起始日期（含） + effective_to DATE NOT NULL DEFAULT '9999-12-31', -- 生效截止日期（含） + description TEXT, -- 说明 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_assistant_level_price UNIQUE (level_code, effective_from) +); + +COMMENT ON TABLE billiards_dws.cfg_assistant_level_price IS '助教等级定价表：初级/中级/高级/星级的基础课和附加课单价，支持按时间生效'; +COMMENT ON COLUMN billiards_dws.cfg_assistant_level_price.level_code IS '等级代码：8=初级, 10=中级, 20=高级, 30=星级, 40=金牌'; +COMMENT ON COLUMN billiards_dws.cfg_assistant_level_price.base_course_price IS '基础课（陪打/PD）单价，按等级不同'; +COMMENT ON COLUMN billiards_dws.cfg_assistant_level_price.bonus_course_price IS '附加课（超休/CX）单价，固定50元/小时'; + +CREATE INDEX idx_cfg_assistant_level_price_effective + ON billiards_dws.cfg_assistant_level_price (effective_from, effective_to); + + +-- ----------------------------------------------------------------------------- +-- 3. cfg_bonus_rules - 奖金规则配置表 +-- 说明: +-- - 包含冲刺奖金（按小时阈值）和Top3奖金（按排名） +-- - Top3排名口径: 按绩效总小时数，如遇并列则都算（如2个第一，则记为2个第一，一个第三） +-- - 冲刺奖金: H>=190得300元, H>=220得800元（不累计） +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.cfg_bonus_rules CASCADE; +CREATE TABLE billiards_dws.cfg_bonus_rules ( + rule_id SERIAL PRIMARY KEY, -- 规则ID（自增） + rule_type VARCHAR(20) NOT NULL, -- 规则类型: SPRINT（冲刺奖金）, TOP_RANK（Top排名奖金） + rule_code VARCHAR(30) NOT NULL, -- 规则代码: SPRINT_190, SPRINT_220, TOP_1, TOP_2, TOP_3 + rule_name VARCHAR(50) NOT NULL, -- 规则名称 + threshold_hours NUMERIC(10,2), -- 小时数阈值（冲刺奖金用） + rank_position INTEGER, -- 排名位置（Top奖金用） + bonus_amount NUMERIC(12,2) NOT NULL, -- 奖金金额（元） + is_cumulative BOOLEAN NOT NULL DEFAULT FALSE, -- 是否可累计（冲刺奖金为FALSE，取最高档） + priority INTEGER NOT NULL DEFAULT 0, -- 优先级（数字越大优先级越高，用于非累计时取最高） + effective_from DATE NOT NULL DEFAULT '2000-01-01', -- 生效起始日期（含） + effective_to DATE NOT NULL DEFAULT '9999-12-31', -- 生效截止日期（含） + description TEXT, -- 说明 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_bonus_rules UNIQUE (rule_type, rule_code, effective_from) +); + +COMMENT ON TABLE billiards_dws.cfg_bonus_rules IS '奖金规则配置表：冲刺奖金(按小时阈值)和Top3奖金(按排名)，支持按时间生效'; +COMMENT ON COLUMN billiards_dws.cfg_bonus_rules.rule_type IS '规则类型：SPRINT=冲刺奖金, TOP_RANK=Top排名奖金'; +COMMENT ON COLUMN billiards_dws.cfg_bonus_rules.is_cumulative IS '是否累计：冲刺奖金不累计取最高档，Top奖金独立发放'; +COMMENT ON COLUMN billiards_dws.cfg_bonus_rules.priority IS '优先级：非累计时用于取最高档奖金'; + +CREATE INDEX idx_cfg_bonus_rules_effective + ON billiards_dws.cfg_bonus_rules (effective_from, effective_to); +CREATE INDEX idx_cfg_bonus_rules_type + ON billiards_dws.cfg_bonus_rules (rule_type); + + +-- ----------------------------------------------------------------------------- +-- 4. cfg_area_category - 台区分类映射表 +-- 说明: +-- - 将 dim_table.site_table_area_name 映射到财务报表区域分类 +-- - 数据来源: BD_manual_dim_table.md 中的实际台区分布 +-- - 分类设计: +-- * BILLIARD: 台球散台（A区/B区/C区/TV台） +-- * BILLIARD_VIP: 台球VIP包厢 +-- * SNOOKER: 斯诺克区 +-- * MAHJONG: 麻将棋牌（麻将房/M7/M8/666/发财） +-- * KTV: K歌娱乐（K包/k包活动区/幸会158） +-- * SPECIAL: 特殊（补时长） +-- * OTHER: 其他 +-- - 映射规则: 精确匹配 > 模糊匹配 > 默认兜底 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.cfg_area_category CASCADE; +CREATE TABLE billiards_dws.cfg_area_category ( + category_id SERIAL PRIMARY KEY, -- 分类ID（自增） + source_area_name VARCHAR(100) NOT NULL, -- 源区域名称（来自dim_table.site_table_area_name） + category_code VARCHAR(20) NOT NULL, -- 分类代码: BILLIARD, BILLIARD_VIP, SNOOKER, MAHJONG, KTV, SPECIAL, OTHER + category_name VARCHAR(50) NOT NULL, -- 分类名称: 台球散台、台球VIP、斯诺克、麻将棋牌、K歌娱乐、补时长、其他 + match_type VARCHAR(10) NOT NULL DEFAULT 'EXACT', -- 匹配类型: EXACT(精确), LIKE(模糊), DEFAULT(兜底) + match_priority INTEGER NOT NULL DEFAULT 100, -- 匹配优先级（数字越小优先级越高） + is_active BOOLEAN NOT NULL DEFAULT TRUE, -- 是否启用 + description TEXT, -- 说明 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_area_category UNIQUE (source_area_name) +); + +COMMENT ON TABLE billiards_dws.cfg_area_category IS '台区分类映射表：将dim_table区域名称映射到财务报表分类，基于BD_manual_dim_table.md实际数据'; +COMMENT ON COLUMN billiards_dws.cfg_area_category.category_code IS '分类代码：BILLIARD台球散台, BILLIARD_VIP台球VIP, SNOOKER斯诺克, MAHJONG麻将, KTV K歌, SPECIAL特殊, OTHER其他'; +COMMENT ON COLUMN billiards_dws.cfg_area_category.match_type IS '匹配类型：EXACT精确匹配, LIKE模糊匹配(用于包含关系), DEFAULT兜底'; +COMMENT ON COLUMN billiards_dws.cfg_area_category.match_priority IS '匹配优先级：多条匹配时取优先级最高的'; + +CREATE INDEX idx_cfg_area_category_code ON billiards_dws.cfg_area_category (category_code); + + +-- ----------------------------------------------------------------------------- +-- 5. cfg_skill_type - 技能→课程类型映射表 +-- 说明: +-- - 将 skill_id 映射到课程类型（基础课/附加课） +-- - 基础课（陪打/PD）: skill_id = 2791903611396869 +-- - 附加课（超休/CX）: skill_id = 2807440316432197 +-- - 避免依赖 skill_name 文本匹配，使用配置表管理 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.cfg_skill_type CASCADE; +CREATE TABLE billiards_dws.cfg_skill_type ( + skill_type_id SERIAL PRIMARY KEY, -- 映射ID（自增） + skill_id BIGINT NOT NULL, -- 技能ID（来自dwd_assistant_service_log.skill_id） + skill_name VARCHAR(50), -- 技能名称（仅用于展示和校验） + course_type_code VARCHAR(10) NOT NULL, -- 课程类型代码: BASE（基础课）, BONUS（附加课） + course_type_name VARCHAR(20) NOT NULL, -- 课程类型名称: 基础课/陪打, 附加课/超休 + is_active BOOLEAN NOT NULL DEFAULT TRUE, -- 是否启用 + description TEXT, -- 说明 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_skill_type UNIQUE (skill_id) +); + +COMMENT ON TABLE billiards_dws.cfg_skill_type IS '技能→课程类型映射表：将skill_id映射到基础课/附加课，避免依赖skill_name文本'; +COMMENT ON COLUMN billiards_dws.cfg_skill_type.course_type_code IS '课程类型：BASE=基础课(陪打), BONUS=附加课(超休)'; + +CREATE INDEX idx_cfg_skill_type_course ON billiards_dws.cfg_skill_type (course_type_code); + + +-- ============================================================================= +-- 第二部分：助教维度（5张） +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 6. dws_assistant_daily_detail - 助教日度业绩明细表 +-- 说明: +-- - 以"助教+日期"为粒度，汇总每日业绩明细 +-- - 数据来源: dwd_assistant_service_log + dwd_assistant_trash_event（排除废除记录） +-- - 更新频率: 每小时增量更新 +-- - 时间分层: 通过 stat_date 筛选实现近2天/近1月/近3月/全量 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_assistant_daily_detail CASCADE; +CREATE TABLE billiards_dws.dws_assistant_daily_detail ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + assistant_id BIGINT NOT NULL, -- 助教ID（dim_assistant.site_assistant_id） + assistant_nickname VARCHAR(50), -- 助教花名（冗余，便于查询展示） + stat_date DATE NOT NULL, -- 统计日期 + -- 等级信息（as-of取值，使用统计日期时点的等级） + assistant_level_code INTEGER, -- 助教等级代码（统计日当日生效的等级） + assistant_level_name VARCHAR(20), -- 助教等级名称 + -- 业绩统计 + total_service_count INTEGER NOT NULL DEFAULT 0, -- 总服务次数 + base_service_count INTEGER NOT NULL DEFAULT 0, -- 基础课服务次数 + bonus_service_count INTEGER NOT NULL DEFAULT 0, -- 附加课服务次数 + total_seconds INTEGER NOT NULL DEFAULT 0, -- 总计费时长（秒） + base_seconds INTEGER NOT NULL DEFAULT 0, -- 基础课计费时长（秒） + bonus_seconds INTEGER NOT NULL DEFAULT 0, -- 附加课计费时长（秒） + total_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 总计费小时数（total_seconds/3600） + base_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 基础课小时数 + bonus_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 附加课小时数 + -- 金额统计 + total_ledger_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 总计费金额（元） + base_ledger_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 基础课计费金额 + bonus_ledger_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 附加课计费金额 + -- 客户与台桌统计 + unique_customers INTEGER NOT NULL DEFAULT 0, -- 服务客户数（去重） + unique_tables INTEGER NOT NULL DEFAULT 0, -- 服务台桌数（去重） + -- 废除记录统计 + trashed_seconds INTEGER NOT NULL DEFAULT 0, -- 被废除的服务时长（秒） + trashed_count INTEGER NOT NULL DEFAULT 0, -- 被废除的服务次数 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_assistant_daily UNIQUE (site_id, assistant_id, stat_date) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_daily_detail IS '助教日度业绩明细：按助教+日期汇总服务次数、时长、金额，支持时间分层查询'; +COMMENT ON COLUMN billiards_dws.dws_assistant_daily_detail.assistant_level_code IS 'SCD2口径：取stat_date当日生效的助教等级'; +COMMENT ON COLUMN billiards_dws.dws_assistant_daily_detail.trashed_seconds IS '被废除时长：来自dwd_assistant_trash_event，影响有效业绩'; + +-- 时间分层查询索引（核心） +CREATE INDEX idx_dws_assistant_daily_date ON billiards_dws.dws_assistant_daily_detail (stat_date); +CREATE INDEX idx_dws_assistant_daily_asst_date ON billiards_dws.dws_assistant_daily_detail (assistant_id, stat_date); +CREATE INDEX idx_dws_assistant_daily_site_date ON billiards_dws.dws_assistant_daily_detail (site_id, stat_date); + + +-- ----------------------------------------------------------------------------- +-- 7. dws_assistant_monthly_summary - 助教月度业绩汇总表 +-- 说明: +-- - 以"助教+月份"为粒度，汇总月度业绩及档位计算 +-- - 数据来源: dws_assistant_daily_detail 聚合 + cfg_performance_tier 档位匹配 +-- - 更新频率: 每日更新当月数据 +-- - 新入职判断: 入职日期在月1日0点之后则为新入职 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_assistant_monthly_summary CASCADE; +CREATE TABLE billiards_dws.dws_assistant_monthly_summary ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + assistant_id BIGINT NOT NULL, -- 助教ID + assistant_nickname VARCHAR(50), -- 助教花名 + stat_month DATE NOT NULL, -- 统计月份（月第一天，如2026-01-01） + -- 等级信息（as-of取值，使用月末时点的等级） + assistant_level_code INTEGER, -- 助教等级代码 + assistant_level_name VARCHAR(20), -- 助教等级名称 + -- 入职信息 + hire_date DATE, -- 入职日期（来自dim_assistant） + is_new_hire BOOLEAN NOT NULL DEFAULT FALSE, -- 是否新入职（入职日期 >= 统计月1日0点） + -- 月度业绩汇总 + work_days INTEGER NOT NULL DEFAULT 0, -- 有服务天数 + total_service_count INTEGER NOT NULL DEFAULT 0, -- 总服务次数 + base_service_count INTEGER NOT NULL DEFAULT 0, -- 基础课服务次数 + bonus_service_count INTEGER NOT NULL DEFAULT 0, -- 附加课服务次数 + total_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 总计费小时数 + base_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 基础课小时数 + bonus_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 附加课小时数 + -- 有效业绩（扣除废除记录后） + effective_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 有效业绩小时数（影响档位） + trashed_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 被废除小时数 + -- 金额统计 + total_ledger_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 总计费金额 + base_ledger_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 基础课计费金额 + bonus_ledger_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 附加课计费金额 + -- 客户统计 + unique_customers INTEGER NOT NULL DEFAULT 0, -- 月度服务客户数（去重） + unique_tables INTEGER NOT NULL DEFAULT 0, -- 月度服务台桌数（去重） + avg_service_seconds NUMERIC(10,2) NOT NULL DEFAULT 0, -- 平均单次服务时长（秒） + -- 档位信息（根据有效业绩匹配） + tier_id INTEGER, -- 匹配的档位ID + tier_code VARCHAR(20), -- 档位代码 + tier_name VARCHAR(50), -- 档位名称 + -- 排名信息（用于Top3奖金，按有效业绩小时数排名） + rank_by_hours INTEGER, -- 月度排名（按effective_hours降序） + rank_with_ties INTEGER, -- 考虑并列的排名（如2个第一则都是1） + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_assistant_monthly UNIQUE (site_id, assistant_id, stat_month) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_monthly_summary IS '助教月度业绩汇总：按助教+月份汇总业绩、档位匹配、排名计算'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.stat_month IS '统计月份：存储月第一天日期，如2026-01-01'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.is_new_hire IS '新入职标记：入职日期>=月1日0点则为TRUE'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.effective_hours IS '有效业绩：total_hours - trashed_hours，用于档位匹配'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.rank_with_ties IS 'Top3排名口径：如遇并列都算，如2个第一则都是1，下一个是3'; + +CREATE INDEX idx_dws_assistant_monthly_month ON billiards_dws.dws_assistant_monthly_summary (stat_month); +CREATE INDEX idx_dws_assistant_monthly_asst ON billiards_dws.dws_assistant_monthly_summary (assistant_id, stat_month); +CREATE INDEX idx_dws_assistant_monthly_tier ON billiards_dws.dws_assistant_monthly_summary (tier_code); + + +-- ----------------------------------------------------------------------------- +-- 8. dws_assistant_customer_stats - 助教服务客户统计表 +-- 说明: +-- - 以"助教+客户"为粒度，统计服务关系和滚动窗口指标 +-- - 滚动窗口: 7/10/15/30/60/90天，从统计日期往前计算 +-- - 更新频率: 每日更新 +-- - 散客处理: member_id=0 不进入此表统计 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_assistant_customer_stats CASCADE; +CREATE TABLE billiards_dws.dws_assistant_customer_stats ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + assistant_id BIGINT NOT NULL, -- 助教ID + assistant_nickname VARCHAR(50), -- 助教花名 + member_id BIGINT NOT NULL, -- 客户ID（member_id=0散客不入此表） + member_nickname VARCHAR(100), -- 客户昵称 + member_mobile VARCHAR(20), -- 客户手机号（脱敏） + stat_date DATE NOT NULL, -- 统计基准日期 + -- 全量累计统计 + first_service_date DATE, -- 首次服务日期 + last_service_date DATE, -- 最近服务日期 + total_service_count INTEGER NOT NULL DEFAULT 0, -- 累计服务次数 + total_service_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 累计服务小时数 + total_service_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 累计服务金额 + -- 滚动窗口统计（近N天） + service_count_7d INTEGER NOT NULL DEFAULT 0, -- 近7天服务次数 + service_count_10d INTEGER NOT NULL DEFAULT 0, -- 近10天服务次数 + service_count_15d INTEGER NOT NULL DEFAULT 0, -- 近15天服务次数 + service_count_30d INTEGER NOT NULL DEFAULT 0, -- 近30天服务次数 + service_count_60d INTEGER NOT NULL DEFAULT 0, -- 近60天服务次数 + service_count_90d INTEGER NOT NULL DEFAULT 0, -- 近90天服务次数 + service_hours_7d NUMERIC(10,2) NOT NULL DEFAULT 0, -- 近7天服务小时数 + service_hours_10d NUMERIC(10,2) NOT NULL DEFAULT 0, -- 近10天服务小时数 + service_hours_15d NUMERIC(10,2) NOT NULL DEFAULT 0, -- 近15天服务小时数 + service_hours_30d NUMERIC(10,2) NOT NULL DEFAULT 0, -- 近30天服务小时数 + service_hours_60d NUMERIC(10,2) NOT NULL DEFAULT 0, -- 近60天服务小时数 + service_hours_90d NUMERIC(10,2) NOT NULL DEFAULT 0, -- 近90天服务小时数 + service_amount_7d NUMERIC(12,2) NOT NULL DEFAULT 0, -- 近7天服务金额 + service_amount_10d NUMERIC(12,2) NOT NULL DEFAULT 0, -- 近10天服务金额 + service_amount_15d NUMERIC(12,2) NOT NULL DEFAULT 0, -- 近15天服务金额 + service_amount_30d NUMERIC(12,2) NOT NULL DEFAULT 0, -- 近30天服务金额 + service_amount_60d NUMERIC(12,2) NOT NULL DEFAULT 0, -- 近60天服务金额 + service_amount_90d NUMERIC(12,2) NOT NULL DEFAULT 0, -- 近90天服务金额 + -- 活跃度指标 + days_since_last INTEGER, -- 距离最近服务的天数 + is_active_7d BOOLEAN NOT NULL DEFAULT FALSE, -- 近7天是否活跃 + is_active_30d BOOLEAN NOT NULL DEFAULT FALSE, -- 近30天是否活跃 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_assistant_customer UNIQUE (site_id, assistant_id, member_id, stat_date) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_customer_stats IS '助教服务客户统计：按助教+客户统计服务关系和滚动窗口指标'; +COMMENT ON COLUMN billiards_dws.dws_assistant_customer_stats.member_id IS '客户ID：member_id=0散客不入此表'; +COMMENT ON COLUMN billiards_dws.dws_assistant_customer_stats.service_count_7d IS '滚动窗口：从stat_date往前7天的服务次数'; + +CREATE INDEX idx_dws_assistant_customer_date ON billiards_dws.dws_assistant_customer_stats (stat_date); +CREATE INDEX idx_dws_assistant_customer_asst ON billiards_dws.dws_assistant_customer_stats (assistant_id, stat_date); +CREATE INDEX idx_dws_assistant_customer_member ON billiards_dws.dws_assistant_customer_stats (member_id, stat_date); + + +-- ----------------------------------------------------------------------------- +-- 9. dws_assistant_salary_calc - 助教工资计算详情表 +-- 说明: +-- - 以"助教+月份"为粒度，计算月度工资明细 +-- - 数据来源: dws_assistant_monthly_summary + cfg_* 配置表 +-- - 计算公式（来自DWS数据库处理需求.md）: +-- * 基础课收入 = 基础课小时数 × (客户支付价格 - 专业课抽成) +-- * 附加课收入 = 附加课小时数 × 190 × (1 - 打赏课抽成比例) +-- * 应发工资 = 课时收入 + 奖金 +-- - 更新频率: 月初计算上月工资 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_assistant_salary_calc CASCADE; +CREATE TABLE billiards_dws.dws_assistant_salary_calc ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + assistant_id BIGINT NOT NULL, -- 助教ID + assistant_nickname VARCHAR(50), -- 助教花名 + salary_month DATE NOT NULL, -- 工资月份（月第一天） + -- 助教信息快照 + assistant_level_code INTEGER, -- 助教等级代码（8/10/20/30/40） + assistant_level_name VARCHAR(20), -- 助教等级名称（初级/中级/高级/星级） + hire_date DATE, -- 入职日期 + is_new_hire BOOLEAN NOT NULL DEFAULT FALSE, -- 是否新入职 + -- 业绩数据（来自monthly_summary） + effective_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 有效业绩小时数（基础课+附加课-废除） + base_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 基础课/专业课小时数 + bonus_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 附加课/打赏课小时数 + -- 档位信息（来自cfg_performance_tier） + tier_id INTEGER, -- 档位ID + tier_code VARCHAR(20), -- 档位代码（T0-T5/NEW） + tier_name VARCHAR(50), -- 档位名称 + -- 排名信息 + rank_with_ties INTEGER, -- 月度排名（考虑并列，用于Top3奖金） + -- 定价信息（SCD2口径，取salary_month对应的值） + base_course_price NUMERIC(10,2) NOT NULL DEFAULT 0, -- 基础课客户支付价格（98/108/118/138） + bonus_course_price NUMERIC(10,2) NOT NULL DEFAULT 0, -- 附加课客户支付价格（固定190） + base_deduction NUMERIC(10,2) NOT NULL DEFAULT 0, -- 专业课抽成（元/小时） + bonus_deduction_ratio NUMERIC(5,4) NOT NULL DEFAULT 0, -- 打赏课抽成比例（0-1） + -- 工资计算明细 + base_income NUMERIC(12,2) NOT NULL DEFAULT 0, -- 基础课收入 = base_hours × (base_course_price - base_deduction) + bonus_income NUMERIC(12,2) NOT NULL DEFAULT 0, -- 附加课收入 = bonus_hours × 190 × (1 - bonus_deduction_ratio) + total_course_income NUMERIC(12,2) NOT NULL DEFAULT 0, -- 课时收入合计 + -- 奖金 + sprint_bonus NUMERIC(12,2) NOT NULL DEFAULT 0, -- 冲刺奖金（H>=190:300, H>=220:800，不累计取最高） + top_rank_bonus NUMERIC(12,2) NOT NULL DEFAULT 0, -- Top3排名奖金（1st:1000, 2nd:600, 3rd:400） + recharge_commission NUMERIC(12,2) NOT NULL DEFAULT 0, -- 充值提成（来自dws_assistant_recharge_commission） + other_bonus NUMERIC(12,2) NOT NULL DEFAULT 0, -- 其他奖金（手动调整） + total_bonus NUMERIC(12,2) NOT NULL DEFAULT 0, -- 奖金合计 + -- 工资汇总 + gross_salary NUMERIC(12,2) NOT NULL DEFAULT 0, -- 应发工资 = total_course_income + total_bonus + -- 假期信息 + vacation_days INTEGER NOT NULL DEFAULT 0, -- 次月可休假天数 + vacation_unlimited BOOLEAN NOT NULL DEFAULT FALSE, -- 休假自由标记（5档为TRUE） + -- 备注 + calc_notes TEXT, -- 计算备注（异常说明等） + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_assistant_salary UNIQUE (site_id, assistant_id, salary_month) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_salary_calc IS '助教工资计算详情：按DWS数据库处理需求.md公式计算，包含课时收入和各类奖金'; +COMMENT ON COLUMN billiards_dws.dws_assistant_salary_calc.base_deduction IS '专业课抽成（元/小时）：档位决定，球房从基础课每小时扣除'; +COMMENT ON COLUMN billiards_dws.dws_assistant_salary_calc.bonus_deduction_ratio IS '打赏课抽成比例：档位决定，球房从附加课收入扣除的比例'; +COMMENT ON COLUMN billiards_dws.dws_assistant_salary_calc.base_income IS '基础课收入 = 小时数 × (客户价格 - 专业课抽成)，如170×(108-13)=16150'; +COMMENT ON COLUMN billiards_dws.dws_assistant_salary_calc.bonus_income IS '附加课收入 = 小时数 × 190 × (1 - 抽成比例)，如15×190×0.65=1852.5'; +COMMENT ON COLUMN billiards_dws.dws_assistant_salary_calc.sprint_bonus IS '冲刺奖金：H>=190得300, H>=220得800，不累计取最高档'; +COMMENT ON COLUMN billiards_dws.dws_assistant_salary_calc.top_rank_bonus IS 'Top3奖金：按effective_hours排名，并列都算（如2个第1则无第2）'; + +CREATE INDEX idx_dws_assistant_salary_month ON billiards_dws.dws_assistant_salary_calc (salary_month); +CREATE INDEX idx_dws_assistant_salary_asst ON billiards_dws.dws_assistant_salary_calc (assistant_id, salary_month); + + +-- ----------------------------------------------------------------------------- +-- 10. dws_assistant_recharge_commission - 助教充值提成表 +-- 说明: +-- - 以"助教+月份+充值订单"为粒度，记录充值提成 +-- - 数据来源: Excel手动导入 +-- - 导入字段: 月份、充值订单金额、助教获得的提成金额 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_assistant_recharge_commission CASCADE; +CREATE TABLE billiards_dws.dws_assistant_recharge_commission ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + assistant_id BIGINT NOT NULL, -- 助教ID + assistant_nickname VARCHAR(50), -- 助教花名 + commission_month DATE NOT NULL, -- 提成月份（月第一天） + -- 充值订单关联 + recharge_order_id BIGINT, -- 充值订单ID（可选，关联dwd_recharge_order） + recharge_order_no VARCHAR(50), -- 充值订单号 + -- 提成信息 + recharge_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 充值订单金额 + commission_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 提成金额 + commission_ratio NUMERIC(5,4), -- 提成比例（可选，反算或导入） + -- 导入信息 + import_batch_no VARCHAR(50), -- 导入批次号 + import_file_name VARCHAR(200), -- 导入文件名 + import_time TIMESTAMPTZ, -- 导入时间 + import_user VARCHAR(50), -- 导入操作人 + -- 备注 + remark TEXT, -- 备注 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE billiards_dws.dws_assistant_recharge_commission IS '助教充值提成：Excel导入，记录月份、充值金额、提成金额'; +COMMENT ON COLUMN billiards_dws.dws_assistant_recharge_commission.commission_month IS '提成月份：导入表格中明确的月份'; +COMMENT ON COLUMN billiards_dws.dws_assistant_recharge_commission.import_batch_no IS '导入批次号：用于追溯和去重'; + +CREATE INDEX idx_dws_assistant_commission_month ON billiards_dws.dws_assistant_recharge_commission (commission_month); +CREATE INDEX idx_dws_assistant_commission_asst ON billiards_dws.dws_assistant_recharge_commission (assistant_id, commission_month); +CREATE INDEX idx_dws_assistant_commission_batch ON billiards_dws.dws_assistant_recharge_commission (import_batch_no); + + +-- ============================================================================= +-- 第三部分：客户维度（2张） +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 11. dws_member_consumption_summary - 会员消费汇总表 +-- 说明: +-- - 以"会员"为粒度，统计消费行为和滚动窗口指标 +-- - 散客处理: member_id=0 不进入此表 +-- - 滚动窗口: 7/10/15/30/60/90天 +-- - 更新频率: 每日更新 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_member_consumption_summary CASCADE; +CREATE TABLE billiards_dws.dws_member_consumption_summary ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + member_id BIGINT NOT NULL, -- 会员ID（member_id=0散客不入此表） + stat_date DATE NOT NULL, -- 统计基准日期 + -- 会员基本信息快照 + member_nickname VARCHAR(100), -- 会员昵称 + member_mobile VARCHAR(20), -- 手机号（脱敏） + card_grade_name VARCHAR(50), -- 卡等级名称 + register_date DATE, -- 注册日期 + -- 全量累计统计 + first_consume_date DATE, -- 首次消费日期 + last_consume_date DATE, -- 最近消费日期 + total_visit_count INTEGER NOT NULL DEFAULT 0, -- 累计到店次数 + total_consume_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 累计消费金额 + total_recharge_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 累计充值金额 + total_table_fee NUMERIC(14,2) NOT NULL DEFAULT 0, -- 累计台费 + total_goods_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 累计商品消费 + total_assistant_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 累计助教服务消费 + -- 滚动窗口统计（近N天） + visit_count_7d INTEGER NOT NULL DEFAULT 0, -- 近7天到店次数 + visit_count_10d INTEGER NOT NULL DEFAULT 0, -- 近10天到店次数 + visit_count_15d INTEGER NOT NULL DEFAULT 0, -- 近15天到店次数 + visit_count_30d INTEGER NOT NULL DEFAULT 0, -- 近30天到店次数 + visit_count_60d INTEGER NOT NULL DEFAULT 0, -- 近60天到店次数 + visit_count_90d INTEGER NOT NULL DEFAULT 0, -- 近90天到店次数 + consume_amount_7d NUMERIC(14,2) NOT NULL DEFAULT 0, -- 近7天消费金额 + consume_amount_10d NUMERIC(14,2) NOT NULL DEFAULT 0, -- 近10天消费金额 + consume_amount_15d NUMERIC(14,2) NOT NULL DEFAULT 0, -- 近15天消费金额 + consume_amount_30d NUMERIC(14,2) NOT NULL DEFAULT 0, -- 近30天消费金额 + consume_amount_60d NUMERIC(14,2) NOT NULL DEFAULT 0, -- 近60天消费金额 + consume_amount_90d NUMERIC(14,2) NOT NULL DEFAULT 0, -- 近90天消费金额 + -- 会员卡余额快照 + cash_card_balance NUMERIC(14,2) NOT NULL DEFAULT 0, -- 储值卡余额（现金卡） + gift_card_balance NUMERIC(14,2) NOT NULL DEFAULT 0, -- 赠送卡余额（台费卡+酒水卡+活动券） + total_card_balance NUMERIC(14,2) NOT NULL DEFAULT 0, -- 总卡余额 + -- 活跃度指标 + days_since_last INTEGER, -- 距离最近消费的天数 + is_active_7d BOOLEAN NOT NULL DEFAULT FALSE, -- 近7天是否活跃 + is_active_30d BOOLEAN NOT NULL DEFAULT FALSE, -- 近30天是否活跃 + is_active_90d BOOLEAN NOT NULL DEFAULT FALSE, -- 近90天是否活跃 + -- 客户分层标签 + customer_tier VARCHAR(20), -- 客户分层（高价值/中等/低活跃/流失） + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_member_consumption UNIQUE (site_id, member_id, stat_date) +); + +COMMENT ON TABLE billiards_dws.dws_member_consumption_summary IS '会员消费汇总：按会员统计消费行为和滚动窗口指标，散客不入此表'; +COMMENT ON COLUMN billiards_dws.dws_member_consumption_summary.member_id IS '会员ID：member_id=0散客不统计'; +COMMENT ON COLUMN billiards_dws.dws_member_consumption_summary.cash_card_balance IS '储值卡余额：card_type_id=2793249295533893'; +COMMENT ON COLUMN billiards_dws.dws_member_consumption_summary.gift_card_balance IS '赠送卡余额：台费卡+酒水卡+活动抵用券'; + +CREATE INDEX idx_dws_member_consumption_date ON billiards_dws.dws_member_consumption_summary (stat_date); +CREATE INDEX idx_dws_member_consumption_member ON billiards_dws.dws_member_consumption_summary (member_id, stat_date); +CREATE INDEX idx_dws_member_consumption_tier ON billiards_dws.dws_member_consumption_summary (customer_tier); + + +-- ----------------------------------------------------------------------------- +-- 12. dws_member_visit_detail - 会员来店明细表 +-- 说明: +-- - 以"会员+订单"为粒度，记录每次来店消费明细 +-- - 散客处理: member_id=0 不进入此表 +-- - 数据来源: dwd_settlement_head + 关联明细表 +-- - 更新频率: 每日增量更新 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_member_visit_detail CASCADE; +CREATE TABLE billiards_dws.dws_member_visit_detail ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + member_id BIGINT NOT NULL, -- 会员ID（散客不入此表） + order_settle_id BIGINT NOT NULL, -- 结账单ID + visit_date DATE NOT NULL, -- 来店日期 + visit_time TIMESTAMPTZ, -- 来店时间 + -- 会员信息快照 + member_nickname VARCHAR(100), -- 会员昵称 + member_mobile VARCHAR(20), -- 手机号 + member_birthday DATE, -- 会员生日（关联dim_member） + -- 台桌信息 + table_id BIGINT, -- 台桌ID + table_name VARCHAR(50), -- 台桌名称 + area_name VARCHAR(50), -- 区域名称（原始） + area_category VARCHAR(20), -- 区域分类（散台区/包厢区/VIP区） + -- 消费金额明细 + table_fee NUMERIC(12,2) NOT NULL DEFAULT 0, -- 台费 + goods_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 商品金额 + assistant_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 助教服务金额 + total_consume NUMERIC(12,2) NOT NULL DEFAULT 0, -- 消费总额（正价） + total_discount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 优惠总额 + actual_pay NUMERIC(12,2) NOT NULL DEFAULT 0, -- 实付金额 + -- 支付方式明细 + cash_pay NUMERIC(12,2) NOT NULL DEFAULT 0, -- 现金/刷卡支付 + cash_card_pay NUMERIC(12,2) NOT NULL DEFAULT 0, -- 储值卡支付 + gift_card_pay NUMERIC(12,2) NOT NULL DEFAULT 0, -- 赠送卡支付 + groupbuy_pay NUMERIC(12,2) NOT NULL DEFAULT 0, -- 团购券支付 + -- 时长信息 + table_duration_min INTEGER NOT NULL DEFAULT 0, -- 台桌使用时长（分钟） + assistant_duration_min INTEGER NOT NULL DEFAULT 0, -- 助教服务时长（分钟） + -- 助教服务明细（JSON格式，便于存储多个助教） + assistant_services JSONB, -- 助教服务列表 [{assistant_id, nickname, duration_min, amount}] + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_member_visit UNIQUE (site_id, member_id, order_settle_id) +); + +COMMENT ON TABLE billiards_dws.dws_member_visit_detail IS '会员来店明细：按会员+订单记录每次来店消费，包含台桌、助教、支付明细'; +COMMENT ON COLUMN billiards_dws.dws_member_visit_detail.member_id IS '会员ID：member_id=0散客不入此表'; +COMMENT ON COLUMN billiards_dws.dws_member_visit_detail.area_category IS '区域分类：来自cfg_area_category映射'; +COMMENT ON COLUMN billiards_dws.dws_member_visit_detail.assistant_services IS 'JSON格式：[{assistant_id, nickname, duration_min, amount}]'; + +CREATE INDEX idx_dws_member_visit_date ON billiards_dws.dws_member_visit_detail (visit_date); +CREATE INDEX idx_dws_member_visit_member ON billiards_dws.dws_member_visit_detail (member_id, visit_date); +CREATE INDEX idx_dws_member_visit_order ON billiards_dws.dws_member_visit_detail (order_settle_id); + + +-- ============================================================================= +-- 第四部分：财务维度（7张） +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 13. dws_finance_daily_summary - 财务日度汇总表 +-- 说明: +-- - 以"日期"为粒度，汇总当日财务数据 +-- - 时间分层: 通过 stat_date 筛选实现近2天/近1月/近3月/全量 +-- - 时间口径: 本周起始为周一，本月/季度起始为第一天0点 +-- - 更新频率: 每小时更新当日数据 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_finance_daily_summary CASCADE; +CREATE TABLE billiards_dws.dws_finance_daily_summary ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + stat_date DATE NOT NULL, -- 统计日期 + -- 发生额（正价） + gross_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 发生额合计 + table_fee_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 台费正价 + goods_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 商品正价 + assistant_pd_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 助教基础课正价（陪打） + assistant_cx_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 助教激励课正价（超休） + -- 优惠拆分 + discount_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 优惠合计 + discount_groupbuy NUMERIC(14,2) NOT NULL DEFAULT 0, -- 团购优惠 = coupon_amount - 团购支付金额 + discount_vip NUMERIC(14,2) NOT NULL DEFAULT 0, -- 会员折扣（member_discount_amount） + discount_gift_card NUMERIC(14,2) NOT NULL DEFAULT 0, -- 赠送卡抵扣 + discount_manual NUMERIC(14,2) NOT NULL DEFAULT 0, -- 手动调整（adjust_amount） + discount_rounding NUMERIC(14,2) NOT NULL DEFAULT 0, -- 抹零（rounding_amount） + discount_other NUMERIC(14,2) NOT NULL DEFAULT 0, -- 其他优惠 + -- 确认收入 + confirmed_income NUMERIC(14,2) NOT NULL DEFAULT 0, -- 确认收入 = 发生额 - 优惠 + -- 现金流入 + cash_inflow_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 现金流入合计 + cash_pay_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 收银实付（pay_amount） + groupbuy_pay_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 团购支付金额 + platform_settlement_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 平台回款金额（导入） + platform_fee_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 平台佣金+服务费（导入） + recharge_cash_inflow NUMERIC(14,2) NOT NULL DEFAULT 0, -- 充值现金流入（不含赠送） + -- 储值卡消费（非现金流入） + card_consume_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 卡消费合计 + cash_card_consume NUMERIC(14,2) NOT NULL DEFAULT 0, -- 储值卡消费 + gift_card_consume NUMERIC(14,2) NOT NULL DEFAULT 0, -- 赠送卡消费 + -- 现金流出 + cash_outflow_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 现金流出合计（支出汇总） + -- 现金余额变动 + cash_balance_change NUMERIC(14,2) NOT NULL DEFAULT 0, -- 现金余额变动 = 流入 - 流出 + -- 充值统计 + recharge_count INTEGER NOT NULL DEFAULT 0, -- 充值笔数 + recharge_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 充值总额（含赠送） + recharge_cash NUMERIC(14,2) NOT NULL DEFAULT 0, -- 充值现金部分 + recharge_gift NUMERIC(14,2) NOT NULL DEFAULT 0, -- 充值赠送部分 + first_recharge_count INTEGER NOT NULL DEFAULT 0, -- 首充笔数 + first_recharge_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 首充金额 + renewal_count INTEGER NOT NULL DEFAULT 0, -- 续充笔数 + renewal_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 续充金额 + -- 订单统计 + order_count INTEGER NOT NULL DEFAULT 0, -- 结账单数 + member_order_count INTEGER NOT NULL DEFAULT 0, -- 会员订单数 + guest_order_count INTEGER NOT NULL DEFAULT 0, -- 散客订单数（member_id=0） + avg_order_amount NUMERIC(12,2) NOT NULL DEFAULT 0, -- 平均客单价 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_finance_daily UNIQUE (site_id, stat_date) +); + +COMMENT ON TABLE billiards_dws.dws_finance_daily_summary IS '财务日度汇总：按日期汇总发生额、优惠、收入、现金流、充值等财务指标'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.gross_amount IS '发生额：table_charge_money + goods_money + assistant_pd_money + assistant_cx_money'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.discount_groupbuy IS '团购优惠：coupon_amount - 团购支付金额（pl_coupon_sale_amount或groupbuy_redemption.ledger_unit_price）'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.platform_settlement_amount IS '平台回款金额：来自dws_platform_settlement.settlement_amount'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.platform_fee_amount IS '平台费用：commission_amount + service_fee'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.first_recharge_count IS '首充：dwd_recharge_order.is_first=1'; + +CREATE INDEX idx_dws_finance_daily_date ON billiards_dws.dws_finance_daily_summary (stat_date); +CREATE INDEX idx_dws_finance_daily_site ON billiards_dws.dws_finance_daily_summary (site_id, stat_date); + + +-- ----------------------------------------------------------------------------- +-- 14. dws_finance_income_structure - 收入结构分析表 +-- 说明: +-- - 以"日期+区域/类型"为粒度，分析收入结构 +-- - 区域分类: 使用cfg_area_category映射 +-- - 更新频率: 每日更新 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_finance_income_structure CASCADE; +CREATE TABLE billiards_dws.dws_finance_income_structure ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + stat_date DATE NOT NULL, -- 统计日期 + -- 分类维度 + structure_type VARCHAR(20) NOT NULL, -- 结构类型: AREA（区域）, INCOME_TYPE（收入类型） + category_code VARCHAR(30) NOT NULL, -- 分类代码 + category_name VARCHAR(50) NOT NULL, -- 分类名称 + -- 收入金额 + income_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 收入金额 + income_ratio NUMERIC(5,4) NOT NULL DEFAULT 0, -- 收入占比 + -- 订单统计 + order_count INTEGER NOT NULL DEFAULT 0, -- 订单数 + -- 时长统计（仅台费/助教相关） + duration_minutes INTEGER NOT NULL DEFAULT 0, -- 时长（分钟） + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_finance_income_structure UNIQUE (site_id, stat_date, structure_type, category_code) +); + +COMMENT ON TABLE billiards_dws.dws_finance_income_structure IS '收入结构分析：按区域/收入类型分析收入构成'; +COMMENT ON COLUMN billiards_dws.dws_finance_income_structure.structure_type IS '结构类型：AREA=按区域, INCOME_TYPE=按收入类型（台费/商品/助教）'; +COMMENT ON COLUMN billiards_dws.dws_finance_income_structure.category_code IS '分类代码：区域用SCATTER/ROOM/VIP, 收入类型用TABLE_FEE/GOODS/ASSISTANT'; + +CREATE INDEX idx_dws_finance_income_date ON billiards_dws.dws_finance_income_structure (stat_date); +CREATE INDEX idx_dws_finance_income_type ON billiards_dws.dws_finance_income_structure (structure_type, category_code); + + +-- ----------------------------------------------------------------------------- +-- 15. dws_finance_discount_detail - 优惠明细表 +-- 说明: +-- - 以"日期+优惠类型"为粒度，分析优惠构成 +-- - 优惠类型: 团购优惠、会员折扣、赠送卡、手动调整、抹零、大客户优惠、其他 +-- - 更新频率: 每日更新 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_finance_discount_detail CASCADE; +CREATE TABLE billiards_dws.dws_finance_discount_detail ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + stat_date DATE NOT NULL, -- 统计日期 + -- 优惠类型 + discount_type_code VARCHAR(30) NOT NULL, -- 优惠类型代码 + discount_type_name VARCHAR(50) NOT NULL, -- 优惠类型名称 + -- 优惠金额 + discount_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 优惠金额 + discount_ratio NUMERIC(5,4) NOT NULL DEFAULT 0, -- 优惠占比（占总优惠） + -- 使用统计 + usage_count INTEGER NOT NULL DEFAULT 0, -- 使用次数 + affected_orders INTEGER NOT NULL DEFAULT 0, -- 影响订单数 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_finance_discount_detail UNIQUE (site_id, stat_date, discount_type_code) +); + +COMMENT ON TABLE billiards_dws.dws_finance_discount_detail IS '优惠明细：按优惠类型分析优惠构成'; +COMMENT ON COLUMN billiards_dws.dws_finance_discount_detail.discount_type_code IS '优惠类型：GROUPBUY/VIP/GIFT_CARD/MANUAL/ROUNDING/BIG_CUSTOMER/OTHER'; + +CREATE INDEX idx_dws_finance_discount_date ON billiards_dws.dws_finance_discount_detail (stat_date); +CREATE INDEX idx_dws_finance_discount_type ON billiards_dws.dws_finance_discount_detail (discount_type_code); + + +-- ----------------------------------------------------------------------------- +-- 16. dws_finance_recharge_summary - 充值统计表 +-- 说明: +-- - 以"日期"为粒度，统计充值数据 +-- - 区分首充/续充（is_first字段） +-- - 区分现金充值/赠送金额 +-- - 更新频率: 每日更新 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_finance_recharge_summary CASCADE; +CREATE TABLE billiards_dws.dws_finance_recharge_summary ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + stat_date DATE NOT NULL, -- 统计日期 + -- 充值汇总 + recharge_count INTEGER NOT NULL DEFAULT 0, -- 充值笔数 + recharge_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 充值总额（含赠送） + recharge_cash NUMERIC(14,2) NOT NULL DEFAULT 0, -- 现金充值金额 + recharge_gift NUMERIC(14,2) NOT NULL DEFAULT 0, -- 赠送金额 + -- 首充统计 + first_recharge_count INTEGER NOT NULL DEFAULT 0, -- 首充笔数 + first_recharge_cash NUMERIC(14,2) NOT NULL DEFAULT 0, -- 首充现金 + first_recharge_gift NUMERIC(14,2) NOT NULL DEFAULT 0, -- 首充赠送 + first_recharge_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 首充总额 + -- 续充统计 + renewal_count INTEGER NOT NULL DEFAULT 0, -- 续充笔数 + renewal_cash NUMERIC(14,2) NOT NULL DEFAULT 0, -- 续充现金 + renewal_gift NUMERIC(14,2) NOT NULL DEFAULT 0, -- 续充赠送 + renewal_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 续充总额 + -- 充值会员统计 + recharge_member_count INTEGER NOT NULL DEFAULT 0, -- 充值会员数（去重） + new_member_count INTEGER NOT NULL DEFAULT 0, -- 新增会员数 + -- 卡余额快照（当日末） + total_card_balance NUMERIC(14,2) NOT NULL DEFAULT 0, -- 全部会员卡余额 + cash_card_balance NUMERIC(14,2) NOT NULL DEFAULT 0, -- 储值卡余额 + gift_card_balance NUMERIC(14,2) NOT NULL DEFAULT 0, -- 赠送卡余额 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_finance_recharge UNIQUE (site_id, stat_date) +); + +COMMENT ON TABLE billiards_dws.dws_finance_recharge_summary IS '充值统计：按日期统计充值数据，区分首充/续充、现金/赠送'; +COMMENT ON COLUMN billiards_dws.dws_finance_recharge_summary.first_recharge_count IS '首充：dwd_recharge_order.is_first=1'; +COMMENT ON COLUMN billiards_dws.dws_finance_recharge_summary.cash_card_balance IS '储值卡余额：card_type_id=2793249295533893'; + +CREATE INDEX idx_dws_finance_recharge_date ON billiards_dws.dws_finance_recharge_summary (stat_date); + + +-- ----------------------------------------------------------------------------- +-- 17. dws_finance_expense_summary - 支出结构表 +-- 说明: +-- - 以"月份+支出类型"为粒度，记录支出数据 +-- - 数据来源: Excel手动导入 +-- - 支出类型: 房租、水电、物业、工资、报销、平台费等 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_finance_expense_summary CASCADE; +CREATE TABLE billiards_dws.dws_finance_expense_summary ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + expense_month DATE NOT NULL, -- 支出月份（月第一天） + -- 支出分类 + expense_type_code VARCHAR(30) NOT NULL, -- 支出类型代码 + expense_type_name VARCHAR(50) NOT NULL, -- 支出类型名称 + expense_category VARCHAR(20), -- 支出大类（固定成本/变动成本/其他） + -- 支出金额 + expense_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 支出金额 + -- 明细（可选） + expense_detail TEXT, -- 支出明细说明 + -- 导入信息 + import_batch_no VARCHAR(50), -- 导入批次号 + import_file_name VARCHAR(200), -- 导入文件名 + import_time TIMESTAMPTZ, -- 导入时间 + import_user VARCHAR(50), -- 导入操作人 + -- 备注 + remark TEXT, -- 备注 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_finance_expense UNIQUE (site_id, expense_month, expense_type_code, import_batch_no) +); + +COMMENT ON TABLE billiards_dws.dws_finance_expense_summary IS '支出结构：Excel导入，按月份+类型记录支出数据'; +COMMENT ON COLUMN billiards_dws.dws_finance_expense_summary.expense_type_code IS '支出类型：RENT/UTILITY/PROPERTY/SALARY/REIMBURSE/PLATFORM_FEE/OTHER'; +COMMENT ON COLUMN billiards_dws.dws_finance_expense_summary.expense_category IS '支出大类：FIXED_COST/VARIABLE_COST/OTHER'; + +CREATE INDEX idx_dws_finance_expense_month ON billiards_dws.dws_finance_expense_summary (expense_month); +CREATE INDEX idx_dws_finance_expense_type ON billiards_dws.dws_finance_expense_summary (expense_type_code); +CREATE INDEX idx_dws_finance_expense_batch ON billiards_dws.dws_finance_expense_summary (import_batch_no); + + +-- ----------------------------------------------------------------------------- +-- 18. dws_assistant_finance_analysis - 助教收支分析表 +-- 说明: +-- - 以"日期+助教"为粒度，分析助教产出的收入和成本 +-- - 数据来源: dwd_assistant_service_log + dws_assistant_salary_calc +-- - 更新频率: 每日更新 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_assistant_finance_analysis CASCADE; +CREATE TABLE billiards_dws.dws_assistant_finance_analysis ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + stat_date DATE NOT NULL, -- 统计日期 + assistant_id BIGINT NOT NULL, -- 助教ID + assistant_nickname VARCHAR(50), -- 助教花名 + -- 收入（助教产出） + revenue_total NUMERIC(14,2) NOT NULL DEFAULT 0, -- 助教产出收入（ledger_amount汇总） + revenue_base NUMERIC(14,2) NOT NULL DEFAULT 0, -- 基础课收入 + revenue_bonus NUMERIC(14,2) NOT NULL DEFAULT 0, -- 附加课收入 + -- 成本（助教工资分摊） + cost_daily NUMERIC(14,2) NOT NULL DEFAULT 0, -- 日均工资成本（月工资/工作天数） + -- 毛利 + gross_profit NUMERIC(14,2) NOT NULL DEFAULT 0, -- 毛利 = 收入 - 成本 + gross_margin NUMERIC(5,4) NOT NULL DEFAULT 0, -- 毛利率 + -- 服务统计 + service_count INTEGER NOT NULL DEFAULT 0, -- 服务次数 + service_hours NUMERIC(10,2) NOT NULL DEFAULT 0, -- 服务小时数 + unique_customers INTEGER NOT NULL DEFAULT 0, -- 服务客户数 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_assistant_finance UNIQUE (site_id, stat_date, assistant_id) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_finance_analysis IS '助教收支分析：按日期+助教分析产出收入和工资成本'; +COMMENT ON COLUMN billiards_dws.dws_assistant_finance_analysis.revenue_total IS '助教产出收入：dwd_assistant_service_log.ledger_amount汇总'; +COMMENT ON COLUMN billiards_dws.dws_assistant_finance_analysis.cost_daily IS '日均工资成本：月工资/当月工作天数'; + +CREATE INDEX idx_dws_assistant_finance_date ON billiards_dws.dws_assistant_finance_analysis (stat_date); +CREATE INDEX idx_dws_assistant_finance_asst ON billiards_dws.dws_assistant_finance_analysis (assistant_id, stat_date); + + +-- ----------------------------------------------------------------------------- +-- 19. dws_platform_settlement - 平台回款/服务费表 +-- 说明: +-- - 以"回款日期+平台+订单"为粒度，记录平台结算数据 +-- - 数据来源: Excel手动导入 +-- - 字段: 回款金额、佣金、服务费、回款日期、平台类型、订单关联键 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_platform_settlement CASCADE; +CREATE TABLE billiards_dws.dws_platform_settlement ( + id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + settlement_date DATE NOT NULL, -- 回款日期 + -- 平台信息 + platform_type VARCHAR(30) NOT NULL, -- 平台类型（美团/抖音/大众点评/其他） + platform_name VARCHAR(50), -- 平台名称 + -- 订单关联 + platform_order_no VARCHAR(100), -- 平台订单号 + order_settle_id BIGINT, -- 关联的结账单ID（可选） + -- 金额明细 + settlement_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 回款金额（实际入账） + commission_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 佣金（平台抽成） + service_fee NUMERIC(14,2) NOT NULL DEFAULT 0, -- 服务费 + gross_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 订单原始金额 + -- 导入信息 + import_batch_no VARCHAR(50), -- 导入批次号 + import_file_name VARCHAR(200), -- 导入文件名 + import_time TIMESTAMPTZ, -- 导入时间 + import_user VARCHAR(50), -- 导入操作人 + -- 备注 + remark TEXT, -- 备注 + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +COMMENT ON TABLE billiards_dws.dws_platform_settlement IS '平台回款/服务费：Excel导入，记录各平台结算明细'; +COMMENT ON COLUMN billiards_dws.dws_platform_settlement.platform_type IS '平台类型：MEITUAN/DOUYIN/DIANPING/OTHER'; +COMMENT ON COLUMN billiards_dws.dws_platform_settlement.settlement_amount IS '回款金额：实际入账金额 = gross_amount - commission_amount - service_fee'; + +CREATE INDEX idx_dws_platform_settlement_date ON billiards_dws.dws_platform_settlement (settlement_date); +CREATE INDEX idx_dws_platform_settlement_platform ON billiards_dws.dws_platform_settlement (platform_type); +CREATE INDEX idx_dws_platform_settlement_order ON billiards_dws.dws_platform_settlement (order_settle_id); +CREATE INDEX idx_dws_platform_settlement_batch ON billiards_dws.dws_platform_settlement (import_batch_no); + + +-- ============================================================================= +-- 第五部分：订单汇总（保留原有表，增强字段） +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 20. dws_order_summary - 订单汇总表（已存在，仅添加注释） +-- 说明: +-- - 以"订单"为粒度，汇总订单级别的数据 +-- - 保持原有结构，作为订单级别的聚合层 +-- ----------------------------------------------------------------------------- +-- 原表已存在，添加注释 +COMMENT ON TABLE billiards_dws.dws_order_summary IS '订单汇总：按订单汇总各项金额、优惠、支付信息'; + + +-- ============================================================================= +-- 第六部分：时间分层辅助视图 +-- 说明: +-- - 时间分层通过查询条件实现，不单独创建分层表 +-- - 提供常用时间窗口的参考视图 +-- - 时间口径: 周起始为周一，月/季度起始为第一天0点 +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 时间窗口计算函数 +-- ----------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION billiards_dws.get_time_window( + p_window_type VARCHAR(30), -- 窗口类型: THIS_WEEK, LAST_WEEK, THIS_MONTH, LAST_MONTH, LAST_3_MONTHS_EXCL_CURRENT, LAST_3_MONTHS_INCL_CURRENT, THIS_QUARTER, LAST_QUARTER, LAST_6_MONTHS, LAST_2_DAYS, LAST_1_MONTH, LAST_3_MONTHS + p_base_date DATE DEFAULT CURRENT_DATE +) +RETURNS TABLE ( + window_start DATE, + window_end DATE +) AS $$ +DECLARE + v_year INTEGER; + v_month INTEGER; + v_quarter INTEGER; +BEGIN + v_year := EXTRACT(YEAR FROM p_base_date); + v_month := EXTRACT(MONTH FROM p_base_date); + v_quarter := EXTRACT(QUARTER FROM p_base_date); + + CASE p_window_type + -- 本周（周一起始） + WHEN 'THIS_WEEK' THEN + window_start := DATE_TRUNC('week', p_base_date)::DATE; + window_end := p_base_date; + -- 上周 + WHEN 'LAST_WEEK' THEN + window_start := (DATE_TRUNC('week', p_base_date) - INTERVAL '7 days')::DATE; + window_end := (DATE_TRUNC('week', p_base_date) - INTERVAL '1 day')::DATE; + -- 本月 + WHEN 'THIS_MONTH' THEN + window_start := DATE_TRUNC('month', p_base_date)::DATE; + window_end := p_base_date; + -- 上月 + WHEN 'LAST_MONTH' THEN + window_start := (DATE_TRUNC('month', p_base_date) - INTERVAL '1 month')::DATE; + window_end := (DATE_TRUNC('month', p_base_date) - INTERVAL '1 day')::DATE; + -- 前3个月（不含本月） + WHEN 'LAST_3_MONTHS_EXCL_CURRENT' THEN + window_start := (DATE_TRUNC('month', p_base_date) - INTERVAL '3 months')::DATE; + window_end := (DATE_TRUNC('month', p_base_date) - INTERVAL '1 day')::DATE; + -- 前3个月（含本月） + WHEN 'LAST_3_MONTHS_INCL_CURRENT' THEN + window_start := (DATE_TRUNC('month', p_base_date) - INTERVAL '2 months')::DATE; + window_end := p_base_date; + -- 本季度 + WHEN 'THIS_QUARTER' THEN + window_start := DATE_TRUNC('quarter', p_base_date)::DATE; + window_end := p_base_date; + -- 上季度 + WHEN 'LAST_QUARTER' THEN + window_start := (DATE_TRUNC('quarter', p_base_date) - INTERVAL '3 months')::DATE; + window_end := (DATE_TRUNC('quarter', p_base_date) - INTERVAL '1 day')::DATE; + -- 最近半年（不含本月） + WHEN 'LAST_6_MONTHS' THEN + window_start := (DATE_TRUNC('month', p_base_date) - INTERVAL '6 months')::DATE; + window_end := (DATE_TRUNC('month', p_base_date) - INTERVAL '1 day')::DATE; + -- 近2天 + WHEN 'LAST_2_DAYS' THEN + window_start := (p_base_date - INTERVAL '1 day')::DATE; + window_end := p_base_date; + -- 近1月 + WHEN 'LAST_1_MONTH' THEN + window_start := (p_base_date - INTERVAL '1 month')::DATE; + window_end := p_base_date; + -- 近3月 + WHEN 'LAST_3_MONTHS' THEN + window_start := (p_base_date - INTERVAL '3 months')::DATE; + window_end := p_base_date; + ELSE + -- 默认全量 + window_start := '2000-01-01'::DATE; + window_end := p_base_date; + END CASE; + + RETURN NEXT; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +COMMENT ON FUNCTION billiards_dws.get_time_window IS '时间窗口计算函数：根据窗口类型返回起止日期，周起始为周一'; + + +-- ----------------------------------------------------------------------------- +-- 环比计算辅助函数 +-- 说明: 环比规则为"对比上一个等长区间" +-- ----------------------------------------------------------------------------- +CREATE OR REPLACE FUNCTION billiards_dws.get_comparison_window( + p_start_date DATE, + p_end_date DATE +) +RETURNS TABLE ( + prev_start DATE, + prev_end DATE +) AS $$ +DECLARE + v_interval INTERVAL; +BEGIN + -- 计算区间长度 + v_interval := (p_end_date - p_start_date + 1) * INTERVAL '1 day'; + + -- 上一个等长区间 + prev_end := p_start_date - INTERVAL '1 day'; + prev_start := (prev_end - v_interval + INTERVAL '1 day')::DATE; + + RETURN NEXT; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +COMMENT ON FUNCTION billiards_dws.get_comparison_window IS '环比窗口计算：返回上一个等长区间的起止日期'; + + +-- ============================================================================= +-- 第六部分：指数算法（4张） +-- ============================================================================= + +-- ----------------------------------------------------------------------------- +-- 21. cfg_index_parameters - 指数算法参数配置表 +-- 说明: +-- - 存储客户召回指数和客户-助教亲密指数的算法参数 +-- - 支持按时间生效，便于参数调优和历史追溯 +-- - 参数类型: RECALL（召回指数）, INTIMACY（亲密指数） +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.cfg_index_parameters CASCADE; +CREATE TABLE billiards_dws.cfg_index_parameters ( + param_id SERIAL PRIMARY KEY, -- 参数ID（自增） + index_type VARCHAR(50) NOT NULL, -- 指数类型: RECALL/INTIMACY + param_name VARCHAR(100) NOT NULL, -- 参数名称 + param_value NUMERIC(14,6) NOT NULL, -- 参数值 + description TEXT, -- 参数说明 + effective_from DATE NOT NULL DEFAULT CURRENT_DATE, -- 生效起始日期 + effective_to DATE, -- 生效截止日期（NULL=永久有效） + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_index_parameters UNIQUE (index_type, param_name, effective_from) +); + +COMMENT ON TABLE billiards_dws.cfg_index_parameters IS '指数算法参数配置表：存储召回指数和亲密指数的算法参数'; +COMMENT ON COLUMN billiards_dws.cfg_index_parameters.index_type IS '指数类型：RECALL=客户召回指数, INTIMACY=客户-助教亲密指数'; +COMMENT ON COLUMN billiards_dws.cfg_index_parameters.param_name IS '参数名称：如lookback_days, halflife_new, weight_overdue等'; + +CREATE INDEX idx_cfg_index_params_type ON billiards_dws.cfg_index_parameters (index_type); +CREATE INDEX idx_cfg_index_params_effective ON billiards_dws.cfg_index_parameters (effective_from, effective_to); + + +-- ----------------------------------------------------------------------------- +-- 22. dws_member_recall_index - 客户召回指数表 +-- 说明: +-- - 以"会员"为粒度，计算客户召回的必要性和紧急程度 +-- - 算法基于：超期紧急性、新客户加分、刚充值加分、热度断档加分 +-- - 尊重客户个人到店周期（μ=中位数, σ=MAD） +-- - 散客(member_id=0)不参与计算 +-- - 更新频率: 每2小时 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_member_recall_index CASCADE; +CREATE TABLE billiards_dws.dws_member_recall_index ( + recall_id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + member_id BIGINT NOT NULL, -- 会员ID (散客不计算) + + -- 计算输入特征 + days_since_last_visit INTEGER, -- 距最近一次到店的天数 (t) + visit_interval_median NUMERIC(10,2), -- 到店周期中位数 (μ) + visit_interval_mad NUMERIC(10,2), -- 到店周期MAD (σ) + days_since_first_visit INTEGER, -- 距首访天数 + days_since_last_recharge INTEGER, -- 距最近充值天数 + visits_last_14_days INTEGER NOT NULL DEFAULT 0, -- 近14天到店次数 + visits_last_60_days INTEGER NOT NULL DEFAULT 0, -- 近60天到店次数 + + -- 分项得分 + score_overdue NUMERIC(10,4), -- 超期紧急性得分: 1-exp(-max(0,(t-μ)/σ)) + score_new_bonus NUMERIC(10,4), -- 新客户加分: decay(d_first, h_new) + score_recharge_bonus NUMERIC(10,4), -- 刚充值加分: decay(d_recharge, h_re) + score_hot_drop NUMERIC(10,4), -- 热度断档加分: max(0, ln(1+(r14/r60-1))) + + -- 最终分数 + raw_score NUMERIC(14,6), -- Raw Score (无上限) + display_score NUMERIC(4,2), -- Display Score (0-10) + + -- 元数据 + calc_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), -- 计算时间 + calc_version INTEGER NOT NULL DEFAULT 1, -- 计算版本（参数变更时递增） + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uk_dws_member_recall UNIQUE (site_id, member_id) +); + +COMMENT ON TABLE billiards_dws.dws_member_recall_index IS '客户召回指数：衡量客户召回的必要性和紧急程度，尊重个人到店周期'; +COMMENT ON COLUMN billiards_dws.dws_member_recall_index.visit_interval_median IS '到店周期中位数(μ)：近60天到店间隔的中位数'; +COMMENT ON COLUMN billiards_dws.dws_member_recall_index.visit_interval_mad IS '到店周期MAD(σ)：中位绝对偏差，下限为sigma_min=2'; +COMMENT ON COLUMN billiards_dws.dws_member_recall_index.score_overdue IS '超期紧急性：1-exp(-z), z=max(0,(t-μ)/σ)'; +COMMENT ON COLUMN billiards_dws.dws_member_recall_index.raw_score IS 'Raw Score：无上限，公式=w_over*overdue+w_new*new+w_re*recharge+w_hot*hot_drop'; +COMMENT ON COLUMN billiards_dws.dws_member_recall_index.display_score IS 'Display Score：0-10，Winsorize(P5,P95)+MinMax映射'; + +CREATE INDEX idx_dws_recall_display ON billiards_dws.dws_member_recall_index (site_id, display_score DESC); +CREATE INDEX idx_dws_recall_raw ON billiards_dws.dws_member_recall_index (site_id, raw_score DESC); +CREATE INDEX idx_dws_recall_calc_time ON billiards_dws.dws_member_recall_index (calc_time); + + +-- ----------------------------------------------------------------------------- +-- 23. dws_member_assistant_intimacy - 客户-助教亲密指数表 +-- 说明: +-- - 以"会员+助教"为粒度，衡量客户与助教的关系强度 +-- - 用于助教约课精力分配和约课成功率预估 +-- - 算法基于：频次强度、最近温度、归因充值、时长贡献、激增放大 +-- - 附加课权重=基础课的1.5倍 +-- - 会话合并：同一客人对同一助教，间隔<4小时算同次服务 +-- - 充值归因：服务结束后1小时内的充值算做该助教贡献 +-- - 散客(member_id=0)不参与计算 +-- - 更新频率: 每4小时 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_member_assistant_intimacy CASCADE; +CREATE TABLE billiards_dws.dws_member_assistant_intimacy ( + intimacy_id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + tenant_id BIGINT NOT NULL, -- 租户ID + member_id BIGINT NOT NULL, -- 会员ID + assistant_id BIGINT NOT NULL, -- 助教ID + + -- 计算输入特征 + session_count INTEGER NOT NULL DEFAULT 0, -- 会话次数（合并后） + total_duration_minutes INTEGER NOT NULL DEFAULT 0, -- 总服务时长(分钟) + basic_session_count INTEGER NOT NULL DEFAULT 0, -- 基础课次数 + incentive_session_count INTEGER NOT NULL DEFAULT 0, -- 附加课次数（激励课/超休） + days_since_last_session INTEGER, -- 距最近一次服务的天数 + attributed_recharge_count INTEGER NOT NULL DEFAULT 0, -- 归因充值次数 + attributed_recharge_amount NUMERIC(14,2) NOT NULL DEFAULT 0, -- 归因充值金额 + + -- 分项得分 + score_frequency NUMERIC(10,4), -- 频次强度 F: Σ(τ_i × decay(d_i, h_sess)) + score_recency NUMERIC(10,4), -- 最近温度 R: decay(d_last, h_last) + score_recharge NUMERIC(10,4), -- 归因充值强度 M: Σ(ln(1+amt/A0) × decay(d_r, h_pay)) + score_duration NUMERIC(10,4), -- 时长贡献 D: Σ(sqrt(dur/60) × τ × decay(d, h_sess)) + burst_multiplier NUMERIC(6,4), -- 激增放大因子: 1 + γ × burst + + -- 最终分数 + raw_score NUMERIC(14,6), -- Raw Score (无上限) + display_score NUMERIC(4,2), -- Display Score (0-10) + + -- 元数据 + calc_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), -- 计算时间 + calc_version INTEGER NOT NULL DEFAULT 1, -- 计算版本 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uk_dws_member_assistant_intimacy UNIQUE (site_id, member_id, assistant_id) +); + +COMMENT ON TABLE billiards_dws.dws_member_assistant_intimacy IS '客户-助教亲密指数：衡量客户与助教的关系强度，用于约课分配'; +COMMENT ON COLUMN billiards_dws.dws_member_assistant_intimacy.incentive_session_count IS '附加课次数：skill_id=2790683529513798（附加课/激励课），权重1.5倍'; +COMMENT ON COLUMN billiards_dws.dws_member_assistant_intimacy.attributed_recharge_count IS '归因充值：服务结束后1小时内发生的充值'; +COMMENT ON COLUMN billiards_dws.dws_member_assistant_intimacy.score_frequency IS '频次强度F：课型加权(τ)×时间衰减，τ=1.5(附加课)/1.0(基础课)'; +COMMENT ON COLUMN billiards_dws.dws_member_assistant_intimacy.burst_multiplier IS '激增放大：1+γ×max(0,ln(1+(F_short/F_long-1)))'; +COMMENT ON COLUMN billiards_dws.dws_member_assistant_intimacy.raw_score IS 'Raw Score：(w_F×F+w_R×R+w_M×M+w_D×D)×mult'; + +CREATE INDEX idx_dws_intimacy_member ON billiards_dws.dws_member_assistant_intimacy (site_id, member_id, display_score DESC); +CREATE INDEX idx_dws_intimacy_assistant ON billiards_dws.dws_member_assistant_intimacy (site_id, assistant_id, display_score DESC); +CREATE INDEX idx_dws_intimacy_calc_time ON billiards_dws.dws_member_assistant_intimacy (calc_time); + + +-- ----------------------------------------------------------------------------- +-- 24. dws_index_percentile_history - 分位点历史表 +-- 说明: +-- - 记录每轮指数计算的分位点，用于EWMA平滑 +-- - 防止分数分布轻微变化导致全员分数跳动 +-- - 更新频率高时（如每小时）建议启用EWMA平滑 +-- ----------------------------------------------------------------------------- +DROP TABLE IF EXISTS billiards_dws.dws_index_percentile_history CASCADE; +CREATE TABLE billiards_dws.dws_index_percentile_history ( + history_id BIGSERIAL PRIMARY KEY, -- 自增主键 + site_id BIGINT NOT NULL, -- 门店ID + index_type VARCHAR(50) NOT NULL, -- 指数类型: RECALL/INTIMACY + calc_time TIMESTAMPTZ NOT NULL, -- 计算时间 + + -- 原始分位点 + percentile_5 NUMERIC(14,6), -- 5分位（下锚） + percentile_95 NUMERIC(14,6), -- 95分位（上锚） + + -- EWMA平滑后的分位点 + percentile_5_smoothed NUMERIC(14,6), -- 平滑后的5分位 + percentile_95_smoothed NUMERIC(14,6), -- 平滑后的95分位 + + -- 统计信息 + record_count INTEGER, -- 记录数 + min_raw_score NUMERIC(14,6), -- 最小Raw Score + max_raw_score NUMERIC(14,6), -- 最大Raw Score + avg_raw_score NUMERIC(14,6), -- 平均Raw Score + + -- 元数据 + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + + CONSTRAINT uk_dws_index_percentile_history UNIQUE (site_id, index_type, calc_time) +); + +COMMENT ON TABLE billiards_dws.dws_index_percentile_history IS '分位点历史表：记录每轮指数计算的分位点，用于EWMA平滑防抖'; +COMMENT ON COLUMN billiards_dws.dws_index_percentile_history.percentile_5_smoothed IS 'EWMA平滑：Q_t=(1-α)×Q_{t-1}+α×Q_now，α=0.2'; + +CREATE INDEX idx_dws_percentile_history ON billiards_dws.dws_index_percentile_history (site_id, index_type, calc_time DESC); + + +-- ============================================================================= +-- 完成提示 +-- ============================================================================= +-- DDL执行完成，共创建： +-- - 6张配置表（cfg_*） +-- - 5张助教维度表（dws_assistant_*） +-- - 4张客户维度表（dws_member_*）：新增2张指数表 +-- - 7张财务维度表（dws_finance_* + dws_assistant_finance_* + dws_platform_*） +-- - 1张订单汇总表（dws_order_summary，保留原有） +-- - 1张分位点历史表（dws_index_percentile_history） +-- - 2个辅助函数（get_time_window, get_comparison_window） diff --git a/etl_billiards/database/seed_index_parameters.sql b/etl_billiards/database/seed_index_parameters.sql new file mode 100644 index 0000000..3b917da --- /dev/null +++ b/etl_billiards/database/seed_index_parameters.sql @@ -0,0 +1,118 @@ +-- ============================================================================= +-- 指数算法参数初始化脚本 +-- 版本: v1.0 +-- 创建日期: 2026-02-03 +-- 描述: 为客户召回指数和客户-助教亲密指数插入默认参数 +-- ============================================================================= + +-- 清空旧数据（如果需要重新初始化） +-- DELETE FROM billiards_dws.cfg_index_parameters WHERE index_type IN ('RECALL', 'INTIMACY'); + +-- ============================================================================= +-- 客户召回指数（RECALL）参数 +-- ============================================================================= + +INSERT INTO billiards_dws.cfg_index_parameters + (index_type, param_name, param_value, description, effective_from) +VALUES + -- 基础参数 + ('RECALL', 'lookback_days', 60, '回溯窗口（天）：分析近60天的数据', CURRENT_DATE), + ('RECALL', 'sigma_min', 2.0, '波动下限（天）：避免σ过小导致超期过敏', CURRENT_DATE), + + -- 半衰期参数 + ('RECALL', 'halflife_new', 7, '新客户半衰期（天）：7天后新客户加分衰减到一半', CURRENT_DATE), + ('RECALL', 'halflife_recharge', 10, '刚充值半衰期（天）：10天后充值加分衰减到一半', CURRENT_DATE), + + -- 权重参数 + ('RECALL', 'weight_overdue', 3.0, '超期紧急性权重：主导因子，建议3.0', CURRENT_DATE), + ('RECALL', 'weight_new', 1.0, '新客户权重：建议1.0', CURRENT_DATE), + ('RECALL', 'weight_recharge', 1.0, '刚充值权重：建议1.0', CURRENT_DATE), + ('RECALL', 'weight_hot', 1.0, '热度断档权重：建议1.0', CURRENT_DATE), + + -- 映射参数 + ('RECALL', 'percentile_lower', 5, '下锚分位数：5分位', CURRENT_DATE), + ('RECALL', 'percentile_upper', 95, '上锚分位数：95分位', CURRENT_DATE), + ('RECALL', 'ewma_alpha', 0.2, 'EWMA平滑系数：越小越平滑，建议0.2', CURRENT_DATE) +ON CONFLICT (index_type, param_name, effective_from) DO UPDATE SET + param_value = EXCLUDED.param_value, + description = EXCLUDED.description, + updated_at = NOW(); + + +-- ============================================================================= +-- 客户-助教亲密指数（INTIMACY）参数 +-- ============================================================================= + +INSERT INTO billiards_dws.cfg_index_parameters + (index_type, param_name, param_value, description, effective_from) +VALUES + -- 基础参数 + ('INTIMACY', 'lookback_days', 60, '回溯窗口（天）：分析近60天的数据', CURRENT_DATE), + ('INTIMACY', 'session_merge_hours', 4, '会话合并间隔（小时）：4小时内的服务算同次', CURRENT_DATE), + ('INTIMACY', 'recharge_attribute_hours', 1, '充值归因窗口（小时）：服务结束后1小时内', CURRENT_DATE), + ('INTIMACY', 'amount_base', 500, '金额压缩基准（元）：选门店常见充值档位', CURRENT_DATE), + ('INTIMACY', 'incentive_weight', 1.5, '附加课权重倍数：附加课=基础课的1.5倍', CURRENT_DATE), + + -- 半衰期参数 + ('INTIMACY', 'halflife_session', 14, '会话衰减半衰期（天）：14天后权重衰减到一半', CURRENT_DATE), + ('INTIMACY', 'halflife_last', 10, '最近一次半衰期（天）：10天后温度衰减到一半', CURRENT_DATE), + ('INTIMACY', 'halflife_recharge', 21, '充值衰减半衰期（天）：21天后充值贡献衰减到一半', CURRENT_DATE), + ('INTIMACY', 'halflife_short', 7, '短期激增检测半衰期（天）：用于Burst检测', CURRENT_DATE), + ('INTIMACY', 'halflife_long', 30, '长期激增检测半衰期（天）：用于Burst检测', CURRENT_DATE), + + -- 权重参数 + ('INTIMACY', 'weight_frequency', 2.0, '频次权重：建议2.0', CURRENT_DATE), + ('INTIMACY', 'weight_recency', 1.5, '最近一次权重：建议1.5', CURRENT_DATE), + ('INTIMACY', 'weight_recharge', 2.0, '归因充值权重：建议2.0', CURRENT_DATE), + ('INTIMACY', 'weight_duration', 0.5, '时长权重：次要因素，建议0.5', CURRENT_DATE), + ('INTIMACY', 'burst_gamma', 0.6, '激增放大系数γ：建议0.6', CURRENT_DATE), + + -- 映射参数 + ('INTIMACY', 'percentile_lower', 5, '下锚分位数：5分位', CURRENT_DATE), + ('INTIMACY', 'percentile_upper', 95, '上锚分位数：95分位', CURRENT_DATE), + ('INTIMACY', 'ewma_alpha', 0.2, 'EWMA平滑系数：越小越平滑，建议0.2', CURRENT_DATE) +ON CONFLICT (index_type, param_name, effective_from) DO UPDATE SET + param_value = EXCLUDED.param_value, + description = EXCLUDED.description, + updated_at = NOW(); + + +-- ============================================================================= +-- 验证 +-- ============================================================================= + +-- 检查参数数量 +DO $$ +DECLARE + recall_count INTEGER; + intimacy_count INTEGER; +BEGIN + SELECT COUNT(*) INTO recall_count + FROM billiards_dws.cfg_index_parameters + WHERE index_type = 'RECALL'; + + SELECT COUNT(*) INTO intimacy_count + FROM billiards_dws.cfg_index_parameters + WHERE index_type = 'INTIMACY'; + + RAISE NOTICE '召回指数参数数量: %', recall_count; + RAISE NOTICE '亲密指数参数数量: %', intimacy_count; + + IF recall_count < 10 THEN + RAISE WARNING '召回指数参数不完整，期望至少10个'; + END IF; + + IF intimacy_count < 15 THEN + RAISE WARNING '亲密指数参数不完整，期望至少15个'; + END IF; +END $$; + +-- 显示所有参数 +SELECT + index_type, + param_name, + param_value, + description, + effective_from +FROM billiards_dws.cfg_index_parameters +ORDER BY index_type, param_name; diff --git a/etl_billiards/docs/DWS 数据库处理需求.md b/etl_billiards/docs/DWS 数据库处理需求.md index 3d29384..ecb7b64 100644 --- a/etl_billiards/docs/DWS 数据库处理需求.md +++ b/etl_billiards/docs/DWS 数据库处理需求.md @@ -5,17 +5,91 @@ 本文档描述在ETL已完成的DWD层数据基础上对DWS层的数据处理： - 完成对DWS层数据库的处理，即数据库设计，成果为DDL的SQL语句。 - 数据读取处理到落库，即DWD读取，Python处理，SQL写入。 +- 在动手之前，先出一个任务计划文档，写明事实的具体技术方案细节。 文档更多聚焦业务描述，你需要使用专业技能，使用面向对象编程OOP思想，完成程序设计直至代码完成： - 参考.\README.md 了解现在项目现状。 -- 参考.\etl_billiards\docs\dwd_main_tables_dictionary.md 了解 DWD的schema的表和字段（若与数据库有出路，则以当前数据库为准。） +- 参考.\etl_billiards\docs 了解 DWD的schema的表和字段。 - SQL和Python代码需要详尽的，高密度的中文注释。 - 完成内容，需要详尽高密度的补充至.\README.md，以方便后续维护。 - DWS的表与表的字段 参考.\etl_billiards\docs\dwd_main_tables_dictionary.md 完成类似的数据库文档，方便后续维护。 - 注意中文编码需求。 -## 具体需求 -### 助教视角 -- 需要 +## 通用需求 +### 数据分层 +我希望使用互联网软件的业内通用方法，将数据按照更新时间分为4层，以符合业务层面的查询效率速度。 +- 第一层：回溯两天前到当前数据。 +- 第二层：回溯1个月前到当前数据。 +- 第三层：回溯3个月前到当前数据。 +- 第四层：全量数据。 +- 需要有配套的机制及时添加删除整理数据。 + +### 统计注意 +当统计一些数据时，注意口径，数据有效性标识。举例： +- 计算助教业绩/工资时，需要参考助教废除表，相关业务数据的影响。 +- 计算助教业绩/工资时，注意辨别 助教课 附加课影响。 + +## 业务需求 +### 系统设置 +- 助教新的绩效考核和工资结算方式更新为以下算法，影响工资结算和财务账务方面的统计核算，相关内容需要落库，以方便后续调整。还要标记执行时间（如哪个月执行哪个标准等），执行相关结算和计算逻辑。： +档位原因考虑 总业绩小时数阈值 专业课抽成（元/小时） 打赏课抽成 次月休假（天） +0档 淘汰压力 H <100 28 50% 3 +1档 及格档（重点激励） 100≤ H <130 18 40% 4 +2档 良好档（重点激励） 130≤ H <160 15 38% 4 +3档 优秀档 160≤ H <190 13 35% 5 +4档 卓越加速档（高端人才倾斜） 190≤ H <220 10 33% 6 +5档 冠军加速档（高端人才倾斜） H ≥220 8 30% 休假自由 + +*课程分为2种（dwd_assistant_service_log表的skill_name）： +基础课：又名 专业课 上桌 上钟，是为客户提供台球助教陪练的课程，按时长统计。精确到分钟。 +附加课：又名 超休 激励 打赏，是客户支付较为高昂的价格，买断整小时与助教外出。 +总业绩小时数阈值指基础课和附加课总和。 + +各级别助教（dim_assistant表的level）基础课，对客户收费：初级 98元/小时；中级 108元/小时；高级 118元/小时；星级 138元/小时； +附加课对客户收费统一为190元/小时。 + +充值提成： + + + + +冲刺奖 达成奖金 +当月 H ≥ 190：300 元 +当月 H ≥ 220：800 元（与上条不叠加，取高） + +额外奖金： +冲刺奖 达成奖金 +当月 H ≥ 190：300 元 +当月 H ≥ 220：800 元（与上条不叠加，取高） + +Top3 奖金： +第1名：1000 元 +第2名：600 元 +第3名：400 元 + +规则： +1、过档后，所有时长按新档位进行计算。 +举例，当前某中级助教已完成185小时，基础课占170小时，附加课15小时。则该月工资计算方法： +170*（108-13）+15*（1-0.35） + +2、本月新入职助教，定档方案： + 按照日均*30的总业绩小时数定档。 + 在该25日之后入职的新助教，最高定档至3档。 + 该折算仅用于定档，不适用于“冲刺奖”和“Top3奖”的计算口径。 + +### 助教维度 +以每个助教个体的视角 +- 我要知道我的业绩档位，历史月份与本月档位进度，档位影响的收入单价。及相邻月份的变化。 +- 我要知道我的有效业绩：历史月份与本月的 基础课课时，激励课课时，全部课课时。相邻月份的变化。 +- 我要知道我的收入：历史月份与本月的收入（注意助教等级，业绩档位，课程种类等因素的总和计算）。相邻月份的变化。 +- 我要知道我的客户情况：过去7天、10天、15天、30天、60天、90天 的跨度进行统计，我服务过（基础课+附加课）的客户数据，并关联每次服务的 时间 时长 台桌 分类 等详细信息。 + +### 客户维度 +统计每个客户的信息 +- 我要知道每个客户：过去7天、10天、15天、30天、60天、90天 的跨度进行统计，来店消费情况，并关联每次服务的 时间 食品饮品 时长 台桌 分类 助教服务 等详细信息。 + + +### 财务维度 +财务维度的需求（已经落到原型图需求级别了），见财务页面需求.md diff --git a/etl_billiards/docs/DWS_任务计划_v1.md b/etl_billiards/docs/DWS_任务计划_v1.md new file mode 100644 index 0000000..0a7e0c5 --- /dev/null +++ b/etl_billiards/docs/DWS_任务计划_v1.md @@ -0,0 +1,1122 @@ +# DWS 数据层实施任务计划 v1.0 + +> **版本**: 1.0 +> **创建日期**: 2026-02-01 +> **项目路径**: `C:\dev\LLTQ\ETL\feiqiu-ETL` +> **目标**: 在已完成的 DWD 层数据基础上，完成 DWS（数据汇总层）的数据库设计与 ETL 实现 + +--- + +## 一、项目背景与现状分析 + +### 1.1 已有基础设施 + +| 层级 | 状态 | Schema | 说明 | +|------|------|--------|------| +| ODS（操作数据层） | ✅ 完成 | `billiards_ods` | 原始数据落地，20+ 张表 | +| DWD（明细数据层） | ✅ 完成 | `billiards_dwd` | 维度表 9 张 + 事实表 12 张 | +| DWS（汇总数据层） | 🔄 部分完成 | `billiards_dws` | 仅有 `dws_order_summary` 1 张 | + +### 1.2 现有代码架构 + +``` +etl_billiards/ +├── tasks/ +│ ├── base_task.py # BaseTask 基类（E/T/L 模板方法） +│ ├── init_dws_schema_task.py # DWS Schema 初始化任务 +│ └── dws_build_order_summary_task.py # 订单汇总表构建任务 +├── database/ +│ └── schema_dws.sql # DWS DDL（当前仅 dws_order_summary） +├── scripts/ +│ └── build_dws_order_summary.py # 订单汇总构建脚本 +└── docs/ + ├── dwd_main_tables_dictionary.md # DWD 数据字典 + └── 财务页面需求.md # 财务需求原型 +``` + +### 1.3 核心 DWD 表依赖关系 + +**维度表**: +- `dim_assistant`: 助教维度（level 字段标识等级：初级/中级/高级/星级） +- `dim_member`: 会员维度 +- `dim_table`: 台桌维度 +- `dim_site`: 门店维度 + +**事实表**: +- `dwd_assistant_service_log`: 助教服务记录（核心：基础课+附加课统计） +- `dwd_assistant_trash_event`: 助教废除记录（影响业绩有效性） +- `dwd_settlement_head`: 结算单头表 +- `dwd_table_fee_log`: 台费流水 +- `dwd_store_goods_sale`: 商品销售 +- `dwd_recharge_order`: 充值订单 +- `dwd_member_balance_change`: 会员余额变动 +- `dwd_payment`: 支付记录 +- `dwd_refund`: 退款记录 + +--- + +## 二、需求分析与表设计 + +### 2.1 数据分层策略（时间分区） + +根据需求，将数据按更新时间分为 4 层以优化查询效率： + +| 分层 | 时间范围 | 用途 | 实现方式 | +|------|----------|------|----------| +| L1（热数据） | 最近 2 天 | 实时监控/当日报表 | 物化视图 + 定时刷新 | +| L2（近期数据） | 最近 1 个月 | 周报/月报 | 分区表 | +| L3（中期数据） | 最近 3 个月 | 季度分析 | 分区表 | +| L4（全量数据） | 历史全部 | 年度汇总/深度分析 | 原始表 | + +**实现机制**: +1. 使用 PostgreSQL 表分区（Range Partition by 月份） +2. 通过视图自动路由到对应时间层 +3. 配套清理任务定期归档/删除过期数据 + +### 2.2 DWS 表清单（完整设计） + +#### 2.2.1 配置表（系统设置） + +| 表名 | 类型 | 说明 | +|------|------|------| +| `cfg_performance_tier` | 配置 | 助教绩效档位配置（6档：0-5档） | +| `cfg_assistant_level_price` | 配置 | 助教等级定价（初级/中级/高级/星级） | +| `cfg_bonus_rules` | 配置 | 奖金规则配置（冲刺奖/Top3奖） | +| `cfg_tier_effective_period` | 配置 | 档位配置生效时间范围 | + +#### 2.2.2 助教维度汇总表 + +| 表名 | 类型 | 主要维度 | 说明 | +|------|------|----------|------| +| `dws_assistant_monthly_summary` | 汇总 | 助教×月份 | 月度业绩汇总（课时/收入/档位） | +| `dws_assistant_daily_detail` | 明细 | 助教×日期 | 日度业绩明细（便于月中进度追踪） | +| `dws_assistant_customer_stats` | 汇总 | 助教×时间窗口 | 服务客户统计（7/10/15/30/60/90天） | +| `dws_assistant_salary_calc` | 计算 | 助教×月份 | 月度工资计算结果 | + +#### 2.2.3 客户维度汇总表 + +| 表名 | 类型 | 主要维度 | 说明 | +|------|------|----------|------| +| `dws_member_consumption_summary` | 汇总 | 会员×时间窗口 | 消费情况统计（7/10/15/30/60/90天） | +| `dws_member_visit_detail` | 明细 | 会员×订单 | 来店消费明细（含服务详情） | + +#### 2.2.4 财务维度汇总表 + +| 表名 | 类型 | 主要维度 | 说明 | +|------|------|----------|------| +| `dws_finance_daily_summary` | 汇总 | 门店×日期 | 日度财务汇总 | +| `dws_finance_income_structure` | 汇总 | 门店×日期×区域 | 收入结构（按区域/类型） | +| `dws_finance_discount_detail` | 明细 | 门店×日期 | 优惠明细（团购/大客户/赠送卡等） | +| `dws_finance_recharge_summary` | 汇总 | 门店×日期 | 充值与预收汇总 | +| `dws_finance_expense_summary` | 汇总 | 门店×日期 | 支出结构汇总 | +| `dws_assistant_finance_analysis` | 汇总 | 门店×日期×等级 | 助教收支分析 | + +#### 2.2.5 现有表保留 + +| 表名 | 状态 | 说明 | +|------|------|------| +| `dws_order_summary` | ✅ 保留 | 订单级汇总（已实现） | + +--- + +## 三、表结构详细设计 + +### 3.1 配置表 DDL + +#### 3.1.1 `cfg_performance_tier` - 绩效档位配置 + +```sql +-- ============================================================ +-- 表名: cfg_performance_tier +-- 用途: 助教绩效档位配置表，定义6个档位的阈值与分成规则 +-- 业务规则: +-- - 过档后，所有时长按新档位进行计算 +-- - 总业绩小时数 = 基础课 + 附加课 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_performance_tier ( + tier_id SERIAL PRIMARY KEY, -- 档位主键 + tier_code INTEGER NOT NULL UNIQUE, -- 档位代码：0-5 + tier_name VARCHAR(50) NOT NULL, -- 档位名称 + tier_reason VARCHAR(100), -- 档位设计原因 + min_hours NUMERIC(10,2) NOT NULL, -- 最小小时数阈值（含） + max_hours NUMERIC(10,2), -- 最大小时数阈值（不含），NULL表示无上限 + base_deduction NUMERIC(10,2) NOT NULL, -- 基础课抽成（元/小时） + bonus_ratio NUMERIC(5,4) NOT NULL, -- 附加课/打赏课抽成比例（如0.50表示50%） + vacation_days INTEGER, -- 次月休假天数，NULL表示休假自由 + effective_from DATE NOT NULL DEFAULT '2026-01-01', -- 生效开始日期 + effective_to DATE DEFAULT '9999-12-31', -- 生效结束日期 + is_active BOOLEAN DEFAULT TRUE, -- 是否启用 + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +-- 初始化默认档位配置（2026年标准） +COMMENT ON TABLE billiards_dws.cfg_performance_tier IS '助教绩效档位配置表：定义各档位的业绩阈值与分成规则'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.tier_code IS '档位代码：0=淘汰压力，1=及格档，2=良好档，3=优秀档，4=卓越加速档，5=冠军加速档'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.base_deduction IS '基础课球房抽成金额（元/小时），助教实得=客户单价-此值'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.bonus_ratio IS '附加课/打赏课球房抽成比例，助教实得=收费*(1-此值)'; +``` + +#### 3.1.2 `cfg_assistant_level_price` - 助教等级定价 + +```sql +-- ============================================================ +-- 表名: cfg_assistant_level_price +-- 用途: 助教等级对应的客户收费标准 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_assistant_level_price ( + price_id SERIAL PRIMARY KEY, + level_code INTEGER NOT NULL, -- 等级代码（对应dim_assistant.level） + level_name VARCHAR(20) NOT NULL, -- 等级名称：初级/中级/高级/星级 + base_price NUMERIC(10,2) NOT NULL, -- 基础课客户收费（元/小时） + bonus_price NUMERIC(10,2) NOT NULL DEFAULT 190, -- 附加课客户收费（元/小时），统一190 + effective_from DATE NOT NULL DEFAULT '2026-01-01', + effective_to DATE DEFAULT '9999-12-31', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(level_code, effective_from) +); + +COMMENT ON TABLE billiards_dws.cfg_assistant_level_price IS '助教等级定价表：定义各等级对客户的收费标准'; +COMMENT ON COLUMN billiards_dws.cfg_assistant_level_price.level_code IS '等级代码：1=初级，2=中级，3=高级，4=星级，8=助教管理'; +``` + +#### 3.1.3 `cfg_bonus_rules` - 奖金规则配置 + +```sql +-- ============================================================ +-- 表名: cfg_bonus_rules +-- 用途: 冲刺奖、Top3奖等额外奖金规则配置 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_bonus_rules ( + rule_id SERIAL PRIMARY KEY, + rule_type VARCHAR(20) NOT NULL, -- 规则类型：SPRINT(冲刺奖)/TOP_RANK(排名奖) + rule_name VARCHAR(50) NOT NULL, -- 规则名称 + condition_type VARCHAR(20) NOT NULL, -- 条件类型：HOURS_GTE(时长>=)/RANK_EQ(排名=) + condition_value NUMERIC(10,2) NOT NULL, -- 条件值 + bonus_amount NUMERIC(10,2) NOT NULL, -- 奖金金额（元） + priority INTEGER DEFAULT 0, -- 优先级（同类型取高优先级） + is_stackable BOOLEAN DEFAULT FALSE, -- 是否可叠加 + effective_from DATE NOT NULL DEFAULT '2026-01-01', + effective_to DATE DEFAULT '9999-12-31', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +COMMENT ON TABLE billiards_dws.cfg_bonus_rules IS '奖金规则配置表：定义冲刺奖和Top3排名奖'; +COMMENT ON COLUMN billiards_dws.cfg_bonus_rules.is_stackable IS '是否可叠加：FALSE表示同类型奖金取高不叠加'; +``` + +### 3.2 助教维度汇总表 DDL + +#### 3.2.1 `dws_assistant_monthly_summary` - 月度业绩汇总 + +```sql +-- ============================================================ +-- 表名: dws_assistant_monthly_summary +-- 用途: 助教月度业绩汇总表，核心统计表 +-- 更新策略: 每日增量更新当月数据，月初全量刷新上月 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_monthly_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, -- 门店ID + assistant_id BIGINT NOT NULL, -- 助教ID + stat_month DATE NOT NULL, -- 统计月份（每月1日） + + -- 基础课统计 + base_service_count INTEGER DEFAULT 0, -- 基础课服务次数 + base_service_seconds INTEGER DEFAULT 0, -- 基础课服务秒数 + base_service_hours NUMERIC(10,2) DEFAULT 0, -- 基础课服务小时数（精确到分钟） + base_income_original NUMERIC(12,2) DEFAULT 0, -- 基础课原始收入（客户支付） + base_income_deduction NUMERIC(12,2) DEFAULT 0, -- 基础课球房抽成 + base_income_actual NUMERIC(12,2) DEFAULT 0, -- 基础课助教实得 + + -- 附加课统计 + bonus_service_count INTEGER DEFAULT 0, -- 附加课服务次数 + bonus_service_seconds INTEGER DEFAULT 0, -- 附加课服务秒数 + bonus_service_hours NUMERIC(10,2) DEFAULT 0, -- 附加课服务小时数 + bonus_income_original NUMERIC(12,2) DEFAULT 0, -- 附加课原始收入 + bonus_income_ratio_ded NUMERIC(12,2) DEFAULT 0, -- 附加课球房按比例抽成 + bonus_income_actual NUMERIC(12,2) DEFAULT 0, -- 附加课助教实得 + + -- 汇总统计 + total_service_count INTEGER DEFAULT 0, -- 总服务次数 + total_service_hours NUMERIC(10,2) DEFAULT 0, -- 总服务小时数 + total_income_original NUMERIC(12,2) DEFAULT 0, -- 总原始收入 + total_income_actual NUMERIC(12,2) DEFAULT 0, -- 总助教实得 + + -- 废除影响 + trashed_service_count INTEGER DEFAULT 0, -- 被废除服务次数 + trashed_service_seconds INTEGER DEFAULT 0, -- 被废除服务秒数 + trashed_amount NUMERIC(12,2) DEFAULT 0, -- 被废除金额 + + -- 档位信息 + tier_code INTEGER, -- 当前档位代码（0-5） + tier_name VARCHAR(50), -- 档位名称 + tier_base_deduction NUMERIC(10,2), -- 档位基础课抽成 + tier_bonus_ratio NUMERIC(5,4), -- 档位附加课抽成比例 + + -- 奖金 + sprint_bonus NUMERIC(10,2) DEFAULT 0, -- 冲刺奖金额 + rank_bonus NUMERIC(10,2) DEFAULT 0, -- 排名奖金额 + other_bonus NUMERIC(10,2) DEFAULT 0, -- 其他奖金 + total_bonus NUMERIC(10,2) DEFAULT 0, -- 奖金合计 + + -- 最终工资 + final_salary NUMERIC(12,2) DEFAULT 0, -- 最终应发工资 + + -- 助教快照信息 + assistant_level INTEGER, -- 助教等级 + assistant_level_name VARCHAR(20), -- 等级名称 + entry_date DATE, -- 入职日期 + is_new_employee BOOLEAN DEFAULT FALSE, -- 是否本月新入职 + work_days_in_month INTEGER, -- 本月在职天数 + + -- 排名信息 + monthly_rank INTEGER, -- 当月排名 + + -- 元数据 + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_month) +); + +-- 索引优化 +CREATE INDEX IF NOT EXISTS idx_dws_asst_monthly_site_month + ON billiards_dws.dws_assistant_monthly_summary(site_id, stat_month); +CREATE INDEX IF NOT EXISTS idx_dws_asst_monthly_assistant + ON billiards_dws.dws_assistant_monthly_summary(assistant_id, stat_month); +CREATE INDEX IF NOT EXISTS idx_dws_asst_monthly_month + ON billiards_dws.dws_assistant_monthly_summary(stat_month); + +COMMENT ON TABLE billiards_dws.dws_assistant_monthly_summary IS '助教月度业绩汇总表：统计每位助教每月的业绩、收入、档位、工资'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.base_service_hours IS '基础课服务小时数，精确到分钟（秒数/3600）'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.tier_code IS '根据total_service_hours计算的档位，参照cfg_performance_tier'; +``` + +#### 3.2.2 `dws_assistant_daily_detail` - 日度业绩明细 + +```sql +-- ============================================================ +-- 表名: dws_assistant_daily_detail +-- 用途: 助教日度业绩明细，用于月中进度追踪 +-- 更新策略: 每小时增量更新 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_daily_detail ( + detail_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, + stat_date DATE NOT NULL, -- 统计日期 + + -- 基础课日统计 + base_service_count INTEGER DEFAULT 0, + base_service_seconds INTEGER DEFAULT 0, + base_service_hours NUMERIC(10,4) DEFAULT 0, + base_income_original NUMERIC(12,2) DEFAULT 0, + + -- 附加课日统计 + bonus_service_count INTEGER DEFAULT 0, + bonus_service_seconds INTEGER DEFAULT 0, + bonus_service_hours NUMERIC(10,4) DEFAULT 0, + bonus_income_original NUMERIC(12,2) DEFAULT 0, + + -- 汇总 + total_service_count INTEGER DEFAULT 0, + total_service_hours NUMERIC(10,4) DEFAULT 0, + total_income_original NUMERIC(12,2) DEFAULT 0, + + -- 废除 + trashed_count INTEGER DEFAULT 0, + trashed_seconds INTEGER DEFAULT 0, + trashed_amount NUMERIC(12,2) DEFAULT 0, + + -- 累计（当月至今） + mtd_total_hours NUMERIC(10,2) DEFAULT 0, -- Month-To-Date 累计小时数 + mtd_tier_code INTEGER, -- 累计后档位 + + -- 服务客户统计 + unique_customers INTEGER DEFAULT 0, -- 当日服务不同客户数 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_date) +); + +CREATE INDEX IF NOT EXISTS idx_dws_asst_daily_site_date + ON billiards_dws.dws_assistant_daily_detail(site_id, stat_date); +CREATE INDEX IF NOT EXISTS idx_dws_asst_daily_assistant + ON billiards_dws.dws_assistant_daily_detail(assistant_id, stat_date); + +COMMENT ON TABLE billiards_dws.dws_assistant_daily_detail IS '助教日度业绩明细表：便于实时追踪月中业绩进度'; +``` + +#### 3.2.3 `dws_assistant_customer_stats` - 服务客户统计 + +```sql +-- ============================================================ +-- 表名: dws_assistant_customer_stats +-- 用途: 助教服务客户统计（多时间窗口） +-- 更新策略: 每日凌晨全量刷新 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_customer_stats ( + stats_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, + stat_date DATE NOT NULL, -- 统计基准日期 + window_days INTEGER NOT NULL, -- 时间窗口天数：7/10/15/30/60/90 + + -- 客户统计 + unique_customers INTEGER DEFAULT 0, -- 服务不同客户数 + total_service_count INTEGER DEFAULT 0, -- 总服务次数 + total_service_hours NUMERIC(10,2) DEFAULT 0, -- 总服务小时数 + + -- 按课程类型 + base_customers INTEGER DEFAULT 0, -- 基础课客户数 + base_service_count INTEGER DEFAULT 0, + bonus_customers INTEGER DEFAULT 0, -- 附加课客户数 + bonus_service_count INTEGER DEFAULT 0, + + -- 收入 + total_income NUMERIC(12,2) DEFAULT 0, + + -- 客户复购 + repeat_customers INTEGER DEFAULT 0, -- 复购客户数（服务>=2次） + repeat_rate NUMERIC(5,4) DEFAULT 0, -- 复购率 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_date, window_days) +); + +CREATE INDEX IF NOT EXISTS idx_dws_asst_cust_window + ON billiards_dws.dws_assistant_customer_stats(assistant_id, stat_date, window_days); + +COMMENT ON TABLE billiards_dws.dws_assistant_customer_stats IS '助教服务客户统计表：支持7/10/15/30/60/90天多时间窗口'; +``` + +### 3.3 客户维度汇总表 DDL + +#### 3.3.1 `dws_member_consumption_summary` - 消费情况统计 + +```sql +-- ============================================================ +-- 表名: dws_member_consumption_summary +-- 用途: 会员消费情况统计（多时间窗口） +-- 更新策略: 每日凌晨全量刷新 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_member_consumption_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + member_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + window_days INTEGER NOT NULL, -- 7/10/15/30/60/90 + + -- 到店统计 + visit_count INTEGER DEFAULT 0, -- 到店次数 + visit_days INTEGER DEFAULT 0, -- 到店天数 + + -- 台费消费 + table_fee_count INTEGER DEFAULT 0, -- 开台次数 + table_fee_seconds INTEGER DEFAULT 0, -- 台费时长（秒） + table_fee_hours NUMERIC(10,2) DEFAULT 0, -- 台费小时数 + table_fee_amount NUMERIC(12,2) DEFAULT 0, -- 台费金额 + + -- 助教服务 + assistant_service_count INTEGER DEFAULT 0, -- 助教服务次数 + assistant_service_hours NUMERIC(10,2) DEFAULT 0, -- 助教服务小时数 + assistant_service_amt NUMERIC(12,2) DEFAULT 0, -- 助教服务金额 + unique_assistants INTEGER DEFAULT 0, -- 服务过的不同助教数 + + -- 商品消费 + goods_order_count INTEGER DEFAULT 0, -- 商品订单数 + goods_item_count INTEGER DEFAULT 0, -- 商品项数 + goods_amount NUMERIC(12,2) DEFAULT 0, -- 商品金额 + + -- 汇总金额 + total_original_amount NUMERIC(12,2) DEFAULT 0, -- 原始消费总额 + total_discount_amount NUMERIC(12,2) DEFAULT 0, -- 优惠总额 + total_actual_amount NUMERIC(12,2) DEFAULT 0, -- 实际支付总额 + + -- 支付方式 + cash_payment NUMERIC(12,2) DEFAULT 0, -- 现金支付 + online_payment NUMERIC(12,2) DEFAULT 0, -- 线上支付 + card_payment NUMERIC(12,2) DEFAULT 0, -- 储值卡支付 + coupon_payment NUMERIC(12,2) DEFAULT 0, -- 团购券支付 + + -- 客户价值指标 + avg_visit_amount NUMERIC(10,2) DEFAULT 0, -- 单次到店平均消费 + avg_visit_duration_min INTEGER DEFAULT 0, -- 单次平均停留时长（分钟） + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, member_id, stat_date, window_days) +); + +CREATE INDEX IF NOT EXISTS idx_dws_member_cons_window + ON billiards_dws.dws_member_consumption_summary(member_id, stat_date, window_days); +CREATE INDEX IF NOT EXISTS idx_dws_member_cons_site + ON billiards_dws.dws_member_consumption_summary(site_id, stat_date, window_days); + +COMMENT ON TABLE billiards_dws.dws_member_consumption_summary IS '会员消费汇总表：支持7/10/15/30/60/90天多时间窗口'; +``` + +### 3.4 财务维度汇总表 DDL + +#### 3.4.1 `dws_finance_daily_summary` - 日度财务汇总 + +```sql +-- ============================================================ +-- 表名: dws_finance_daily_summary +-- 用途: 门店日度财务汇总（核心财务报表数据源） +-- 更新策略: 每小时增量更新 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_daily_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + + -- ========== 经营链：发生额 → 优惠 → 确认收入 ========== + -- 发生额（正价） + gross_revenue NUMERIC(14,2) DEFAULT 0, -- 发生额/正价总计 + gross_table_fee NUMERIC(14,2) DEFAULT 0, -- 台费发生额 + gross_assistant_base NUMERIC(14,2) DEFAULT 0, -- 助教基础课发生额 + gross_assistant_bonus NUMERIC(14,2) DEFAULT 0, -- 助教激励课发生额 + gross_goods NUMERIC(14,2) DEFAULT 0, -- 食品酒水发生额 + + -- 优惠明细 + discount_total NUMERIC(14,2) DEFAULT 0, -- 优惠总额 + discount_groupbuy NUMERIC(14,2) DEFAULT 0, -- 团购优惠 + discount_vip NUMERIC(14,2) DEFAULT 0, -- 大客户优惠 + discount_gift_card NUMERIC(14,2) DEFAULT 0, -- 赠送卡抵扣（台桌卡+酒水卡+抵用券） + discount_manual NUMERIC(14,2) DEFAULT 0, -- 手动调整 + discount_free NUMERIC(14,2) DEFAULT 0, -- 免单 + discount_rounding NUMERIC(14,2) DEFAULT 0, -- 抹零 + + -- 成交/确认收入 + confirmed_revenue NUMERIC(14,2) DEFAULT 0, -- 成交/确认收入（不含充值） + + -- ========== 支付方式构成 ========== + payment_stored_card NUMERIC(14,2) DEFAULT 0, -- 储值卡结算冲销 + payment_cash_online NUMERIC(14,2) DEFAULT 0, -- 现金/线上支付 + payment_groupbuy_deal NUMERIC(14,2) DEFAULT 0, -- 团购核销确认收入（成交价） + + -- ========== 现金流 ========== + -- 现金流入 + cash_inflow_consume NUMERIC(14,2) DEFAULT 0, -- 消费现金流入（现金+线上+平台回款-退款） + cash_inflow_recharge NUMERIC(14,2) DEFAULT 0, -- 充值到账（首充+续费） + cash_inflow_total NUMERIC(14,2) DEFAULT 0, -- 现金流入合计 + + -- 现金支出（需外部输入或其他系统对接） + cash_outflow_total NUMERIC(14,2) DEFAULT 0, -- 现金支出合计 + cash_outflow_rent NUMERIC(14,2) DEFAULT 0, -- 房租 + cash_outflow_utility NUMERIC(14,2) DEFAULT 0, -- 水电 + cash_outflow_purchase NUMERIC(14,2) DEFAULT 0, -- 进货成本 + cash_outflow_assistant NUMERIC(14,2) DEFAULT 0, -- 助教分成 + cash_outflow_salary NUMERIC(14,2) DEFAULT 0, -- 固定人员工资 + cash_outflow_platform NUMERIC(14,2) DEFAULT 0, -- 平台服务费 + cash_outflow_other NUMERIC(14,2) DEFAULT 0, -- 其他费用 + + -- 现金结余 + cash_balance NUMERIC(14,2) DEFAULT 0, -- 现金结余 + cash_balance_rate NUMERIC(5,4) DEFAULT 0, -- 结余率 + + -- ========== 充值与预收 ========== + recharge_first NUMERIC(14,2) DEFAULT 0, -- 首充金额 + recharge_renew NUMERIC(14,2) DEFAULT 0, -- 续费金额 + recharge_total NUMERIC(14,2) DEFAULT 0, -- 充值合计 + recharge_gift NUMERIC(14,2) DEFAULT 0, -- 充值赠送金额 + + -- 储值卡 + stored_card_consume NUMERIC(14,2) DEFAULT 0, -- 储值卡消耗 + stored_card_balance NUMERIC(14,2) DEFAULT 0, -- 储值卡余额（截止当日） + + -- 赠送卡 + gift_card_new NUMERIC(14,2) DEFAULT 0, -- 赠送卡新增 + gift_card_consume NUMERIC(14,2) DEFAULT 0, -- 赠送卡消耗 + gift_card_balance NUMERIC(14,2) DEFAULT 0, -- 赠送卡余额 + + -- ========== 订单统计 ========== + order_count INTEGER DEFAULT 0, -- 订单数 + order_avg_amount NUMERIC(10,2) DEFAULT 0, -- 客单价 + member_order_count INTEGER DEFAULT 0, -- 会员订单数 + member_order_rate NUMERIC(5,4) DEFAULT 0, -- 会员订单占比 + + -- ========== 退款统计 ========== + refund_count INTEGER DEFAULT 0, -- 退款笔数 + refund_amount NUMERIC(14,2) DEFAULT 0, -- 退款金额 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date) +); + +CREATE INDEX IF NOT EXISTS idx_dws_finance_daily_site_date + ON billiards_dws.dws_finance_daily_summary(site_id, stat_date); +CREATE INDEX IF NOT EXISTS idx_dws_finance_daily_date + ON billiards_dws.dws_finance_daily_summary(stat_date); + +COMMENT ON TABLE billiards_dws.dws_finance_daily_summary IS '门店日度财务汇总表：核心财务报表数据源'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.gross_revenue IS '发生额/正价：按各项目正价计算的理论销售额'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.confirmed_revenue IS '成交/确认收入：扣除优惠后的营业收入，不含充值'; +``` + +#### 3.4.2 `dws_finance_income_structure` - 收入结构（按区域） + +```sql +-- ============================================================ +-- 表名: dws_finance_income_structure +-- 用途: 收入结构按区域/类型细分 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_income_structure ( + structure_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + + -- 分类维度 + income_category VARCHAR(20) NOT NULL, -- 收入大类：TABLE/ASSISTANT_BASE/ASSISTANT_BONUS/GOODS + area_name VARCHAR(50), -- 区域名称：A区/B区/C区/团建区/麻将区/NULL(不分区) + + -- 金额 + gross_amount NUMERIC(14,2) DEFAULT 0, -- 发生额 + discount_amount NUMERIC(14,2) DEFAULT 0, -- 优惠金额 + confirmed_amount NUMERIC(14,2) DEFAULT 0, -- 确认收入 + + -- 数量/时长 + item_count INTEGER DEFAULT 0, -- 项目数/次数 + duration_seconds INTEGER DEFAULT 0, -- 时长（秒） + duration_hours NUMERIC(10,2) DEFAULT 0, -- 时长（小时） + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date, income_category, area_name) +); + +COMMENT ON TABLE billiards_dws.dws_finance_income_structure IS '收入结构表：按大类和区域细分的收入明细'; +``` + +#### 3.4.3 `dws_assistant_finance_analysis` - 助教收支分析 + +```sql +-- ============================================================ +-- 表名: dws_assistant_finance_analysis +-- 用途: 助教收支分析（财务视角） +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_finance_analysis ( + analysis_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + assistant_level INTEGER, -- 助教等级（NULL表示全部） + assistant_level_name VARCHAR(20), + + -- 基础课 + base_customer_payment NUMERIC(14,2) DEFAULT 0, -- 客户支付 + base_shop_commission NUMERIC(14,2) DEFAULT 0, -- 球房抽成 + base_hours NUMERIC(10,2) DEFAULT 0, -- 小时数 + base_avg_commission NUMERIC(10,2) DEFAULT 0, -- 均小时抽成 + + -- 激励课 + bonus_customer_payment NUMERIC(14,2) DEFAULT 0, + bonus_shop_commission NUMERIC(14,2) DEFAULT 0, + bonus_hours NUMERIC(10,2) DEFAULT 0, + bonus_avg_commission NUMERIC(10,2) DEFAULT 0, + + -- 汇总 + total_customer_payment NUMERIC(14,2) DEFAULT 0, + total_shop_commission NUMERIC(14,2) DEFAULT 0, + total_hours NUMERIC(10,2) DEFAULT 0, + + -- 助教数量 + assistant_count INTEGER DEFAULT 0, + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date, assistant_level) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_finance_analysis IS '助教收支分析表：财务视角的助教业绩与抽成统计'; +``` + +--- + +## 四、技术实现方案 + +### 4.1 代码架构设计 + +``` +etl_billiards/ +├── tasks/ +│ ├── dws/ # DWS 任务模块（新增） +│ │ ├── __init__.py +│ │ ├── base_dws_task.py # DWS任务基类 +│ │ ├── cfg_init_task.py # 配置表初始化任务 +│ │ ├── assistant_monthly_task.py # 助教月度汇总任务 +│ │ ├── assistant_daily_task.py # 助教日度明细任务 +│ │ ├── assistant_customer_task.py # 助教客户统计任务 +│ │ ├── member_consumption_task.py # 会员消费统计任务 +│ │ ├── finance_daily_task.py # 财务日度汇总任务 +│ │ ├── finance_structure_task.py # 收入结构任务 +│ │ └── assistant_finance_task.py # 助教收支分析任务 +│ └── init_dws_schema_task.py # 更新：执行完整DWS DDL +├── database/ +│ ├── schema_dws.sql # 更新：完整DWS DDL +│ └── seed_dws_config.sql # 新增：配置表初始数据 +├── scripts/ +│ └── build_dws_*.py # DWS 构建脚本 +└── docs/ + └── dws_tables_dictionary.md # 新增：DWS 数据字典 +``` + +### 4.2 类设计（OOP） + +```python +# ============================================================ +# 文件: tasks/dws/base_dws_task.py +# 用途: DWS 任务基类，封装通用逻辑 +# ============================================================ + +class BaseDwsTask(BaseTask): + """ + DWS 层任务基类 + + 特性： + 1. 支持增量/全量两种刷新模式 + 2. 支持 delete-before-insert 策略（幂等性保障） + 3. 内置时间窗口处理逻辑 + """ + + def __init__(self, config, db_connection, api_client, logger): + super().__init__(config, db_connection, api_client, logger) + self.target_schema = "billiards_dws" + self.source_schema = "billiards_dwd" + + @property + def target_table(self) -> str: + """子类需实现：返回目标表名""" + raise NotImplementedError + + @property + def delete_before_insert(self) -> bool: + """是否在插入前删除同范围数据，默认True""" + return True + + def get_delete_sql(self, context: TaskContext) -> tuple[str, list]: + """生成删除SQL，子类可覆盖""" + raise NotImplementedError + + def get_insert_sql(self, context: TaskContext) -> tuple[str, dict]: + """生成插入SQL，子类需实现""" + raise NotImplementedError +``` + +### 4.3 核心算法实现 + +#### 4.3.1 档位计算算法 + +```python +def calculate_tier(total_hours: float, is_new_employee: bool, + entry_date: date, stat_month: date, + work_days: int) -> dict: + """ + 计算助教档位 + + 规则： + 1. 过档后，所有时长按新档位计算 + 2. 新入职助教按 日均*30 折算定档 + 3. 25日后入职最高定3档 + + Args: + total_hours: 总业绩小时数 + is_new_employee: 是否本月新入职 + entry_date: 入职日期 + stat_month: 统计月份 + work_days: 本月在职天数 + + Returns: + {tier_code, tier_name, base_deduction, bonus_ratio} + """ + effective_hours = total_hours + max_tier = 5 + + if is_new_employee and work_days < 30: + # 新入职折算：日均 * 30 + daily_avg = total_hours / max(work_days, 1) + effective_hours = daily_avg * 30 + + # 25日后入职最高3档 + if entry_date.day >= 25: + max_tier = 3 + + # 查询档位配置 + tier = query_tier_config(effective_hours, max_tier) + return tier +``` + +#### 4.3.2 工资计算算法 + +```python +def calculate_salary(assistant_data: dict, tier: dict, + level_price: dict, bonus_rules: list) -> dict: + """ + 计算助教月度工资 + + 公式（以中级助教185小时为例）： + 基础课收入 = 基础课时 * (客户单价 - 档位抽成) + 附加课收入 = 附加课原价 * (1 - 档位抽成比例) + 冲刺奖 = 达到阈值取最高 + 排名奖 = 按排名取值 + + 最终工资 = 基础课收入 + 附加课收入 + 冲刺奖 + 排名奖 + """ + # 基础课收入 + base_income = ( + assistant_data['base_hours'] * + (level_price['base_price'] - tier['base_deduction']) + ) + + # 附加课收入 + bonus_income = ( + assistant_data['bonus_income_original'] * + (1 - tier['bonus_ratio']) + ) + + # 冲刺奖（取高不叠加） + sprint_bonus = calculate_sprint_bonus( + assistant_data['total_hours'], bonus_rules + ) + + # 排名奖 + rank_bonus = calculate_rank_bonus( + assistant_data['monthly_rank'], bonus_rules + ) + + final_salary = base_income + bonus_income + sprint_bonus + rank_bonus + + return { + 'base_income': base_income, + 'bonus_income': bonus_income, + 'sprint_bonus': sprint_bonus, + 'rank_bonus': rank_bonus, + 'final_salary': final_salary + } +``` + +### 4.4 SQL 构建示例 + +#### 4.4.1 助教月度汇总 SQL + +```sql +-- 助教月度业绩汇总构建SQL +WITH +-- 1. 有效服务记录（排除废除） +valid_services AS ( + SELECT + asl.site_id, + asl.site_assistant_id AS assistant_id, + DATE_TRUNC('month', asl.start_use_time)::date AS stat_month, + asl.skill_name, + -- 判断课程类型：基础课/附加课 + CASE + WHEN asl.skill_name IN ('超休', '激励', '打赏') THEN 'BONUS' + ELSE 'BASE' + END AS service_type, + asl.income_seconds, + asl.ledger_amount, + asl.projected_income, + asl.tenant_member_id + FROM billiards_dwd.dwd_assistant_service_log asl + WHERE COALESCE(asl.is_delete, 0) = 0 + AND asl.start_use_time >= %(start_date)s + AND asl.start_use_time < %(end_date)s + -- 排除已废除的记录 + AND NOT EXISTS ( + SELECT 1 FROM billiards_dwd.dwd_assistant_trash_event ate + WHERE ate.assistant_service_id = asl.assistant_service_id + ) +), + +-- 2. 按助教+月份+类型汇总 +service_summary AS ( + SELECT + site_id, + assistant_id, + stat_month, + service_type, + COUNT(*) AS service_count, + SUM(income_seconds) AS service_seconds, + SUM(income_seconds) / 3600.0 AS service_hours, + SUM(ledger_amount) AS income_original, + COUNT(DISTINCT tenant_member_id) AS unique_customers + FROM valid_services + GROUP BY site_id, assistant_id, stat_month, service_type +), + +-- 3. 废除统计 +trash_summary AS ( + SELECT + site_id, + assistant_id, -- 需要从关联表获取 + DATE_TRUNC('month', create_time)::date AS stat_month, + COUNT(*) AS trashed_count, + SUM(charge_minutes_raw * 60) AS trashed_seconds, + SUM(abolish_amount) AS trashed_amount + FROM billiards_dwd.dwd_assistant_trash_event + WHERE create_time >= %(start_date)s + AND create_time < %(end_date)s + GROUP BY site_id, assistant_id, DATE_TRUNC('month', create_time) +), + +-- 4. 助教维度信息 +assistant_info AS ( + SELECT + assistant_id, + site_id, + level, + CASE level + WHEN 1 THEN '初级' + WHEN 2 THEN '中级' + WHEN 3 THEN '高级' + WHEN 4 THEN '星级' + WHEN 8 THEN '助教管理' + ELSE '未知' + END AS level_name, + entry_time::date AS entry_date + FROM billiards_dwd.dim_assistant + WHERE SCD2_is_current = 1 +) + +-- 5. 最终汇总 +SELECT + ss_base.site_id, + ss_base.assistant_id, + ss_base.stat_month, + -- 基础课 + COALESCE(ss_base.service_count, 0) AS base_service_count, + COALESCE(ss_base.service_seconds, 0) AS base_service_seconds, + COALESCE(ss_base.service_hours, 0) AS base_service_hours, + COALESCE(ss_base.income_original, 0) AS base_income_original, + -- 附加课 + COALESCE(ss_bonus.service_count, 0) AS bonus_service_count, + COALESCE(ss_bonus.service_seconds, 0) AS bonus_service_seconds, + COALESCE(ss_bonus.service_hours, 0) AS bonus_service_hours, + COALESCE(ss_bonus.income_original, 0) AS bonus_income_original, + -- 汇总 + COALESCE(ss_base.service_count, 0) + COALESCE(ss_bonus.service_count, 0) AS total_service_count, + COALESCE(ss_base.service_hours, 0) + COALESCE(ss_bonus.service_hours, 0) AS total_service_hours, + COALESCE(ss_base.income_original, 0) + COALESCE(ss_bonus.income_original, 0) AS total_income_original, + -- 废除 + COALESCE(ts.trashed_count, 0) AS trashed_service_count, + COALESCE(ts.trashed_seconds, 0) AS trashed_service_seconds, + COALESCE(ts.trashed_amount, 0) AS trashed_amount, + -- 助教信息 + ai.level AS assistant_level, + ai.level_name AS assistant_level_name, + ai.entry_date +FROM service_summary ss_base +LEFT JOIN service_summary ss_bonus + ON ss_base.site_id = ss_bonus.site_id + AND ss_base.assistant_id = ss_bonus.assistant_id + AND ss_base.stat_month = ss_bonus.stat_month + AND ss_bonus.service_type = 'BONUS' +LEFT JOIN trash_summary ts + ON ss_base.site_id = ts.site_id + AND ss_base.assistant_id = ts.assistant_id + AND ss_base.stat_month = ts.stat_month +LEFT JOIN assistant_info ai + ON ss_base.assistant_id = ai.assistant_id +WHERE ss_base.service_type = 'BASE' +; +``` + +--- + +## 五、实施任务清单 + +### 5.1 阶段一：基础设施（预计 1-2 天） + +| 序号 | 任务 | 优先级 | 依赖 | 产出物 | +|------|------|--------|------|--------| +| 1.1 | 更新 `schema_dws.sql` DDL | P0 | - | DDL 文件 | +| 1.2 | 创建 `seed_dws_config.sql` 初始数据 | P0 | 1.1 | 种子数据文件 | +| 1.3 | 更新 `InitDwsSchemaTask` | P0 | 1.1,1.2 | 任务代码 | +| 1.4 | 创建 `BaseDwsTask` 基类 | P0 | - | 基类代码 | +| 1.5 | 注册新任务到 `task_registry.py` | P0 | 1.3,1.4 | 注册代码 | + +### 5.2 阶段二：配置表与助教维度（预计 2-3 天） + +| 序号 | 任务 | 优先级 | 依赖 | 产出物 | +|------|------|--------|------|--------| +| 2.1 | 实现 `CfgInitTask` 配置初始化 | P0 | 1.* | 任务代码 | +| 2.2 | 实现 `AssistantDailyTask` 日度明细 | P0 | 1.4 | 任务代码 | +| 2.3 | 实现 `AssistantMonthlyTask` 月度汇总 | P0 | 2.2 | 任务代码 | +| 2.4 | 实现档位计算与工资计算模块 | P0 | 2.3 | 算法模块 | +| 2.5 | 实现 `AssistantCustomerTask` 客户统计 | P1 | 2.2 | 任务代码 | + +### 5.3 阶段三：客户维度（预计 1-2 天） + +| 序号 | 任务 | 优先级 | 依赖 | 产出物 | +|------|------|--------|------|--------| +| 3.1 | 实现 `MemberConsumptionTask` | P0 | 1.4 | 任务代码 | +| 3.2 | 实现多时间窗口查询视图 | P1 | 3.1 | SQL 视图 | + +### 5.4 阶段四：财务维度（预计 2-3 天） + +| 序号 | 任务 | 优先级 | 依赖 | 产出物 | +|------|------|--------|------|--------| +| 4.1 | 实现 `FinanceDailySummaryTask` | P0 | 1.4 | 任务代码 | +| 4.2 | 实现 `FinanceIncomeStructureTask` | P0 | 4.1 | 任务代码 | +| 4.3 | 实现 `AssistantFinanceTask` | P0 | 2.3 | 任务代码 | +| 4.4 | 实现充值与预收统计逻辑 | P1 | 4.1 | 模块代码 | + +### 5.5 阶段五：数据分层与维护（预计 1-2 天） + +| 序号 | 任务 | 优先级 | 依赖 | 产出物 | +|------|------|--------|------|--------| +| 5.1 | 实现分区表管理任务 | P1 | 1.1 | 任务代码 | +| 5.2 | 实现数据清理/归档任务 | P1 | 5.1 | 任务代码 | +| 5.3 | 配置定时调度 | P1 | 2-4.* | 调度配置 | + +### 5.6 阶段六：文档与测试（预计 1-2 天） + +| 序号 | 任务 | 优先级 | 依赖 | 产出物 | +|------|------|--------|------|--------| +| 6.1 | 编写 `dws_tables_dictionary.md` | P0 | 1-5.* | 数据字典 | +| 6.2 | 更新 `README.md` | P0 | 1-5.* | README | +| 6.3 | 编写单元测试 | P1 | 2-4.* | 测试代码 | +| 6.4 | 编写集成测试 | P1 | 6.3 | 测试代码 | + +--- + +## 六、风险与注意事项 + +### 6.1 数据一致性 + +1. **废除记录处理**：计算助教业绩时必须排除已废除的服务记录 +2. **课程类型判断**：根据 `skill_name` 字段区分基础课和附加课，需确认枚举值完整性 +3. **新入职折算**：需要准确获取入职日期和在职天数 + +### 6.2 性能优化 + +1. **索引设计**：所有汇总表按常用查询维度建立复合索引 +2. **增量更新**：日常运行采用增量模式，仅全量刷新月初或异常恢复场景 +3. **分区策略**：大表按月份分区，历史数据可归档 + +### 6.3 编码注意 + +1. **中文注释**：所有 SQL 和 Python 代码需要详尽的中文注释 +2. **UTF-8 编码**：确保文件编码为 UTF-8，DDL 中使用 `COMMENT ON` 添加中文说明 +3. **日志输出**：关键步骤输出中文日志便于排查 + +### 6.4 业务规则确认 + +以下内容需与业务方确认： + +1. 附加课识别规则：`skill_name` 的具体枚举值（超休/激励/打赏等） +2. 助教等级映射：`dim_assistant.level` 的值域（1=初级/2=中级/3=高级/4=星级?） +3. 充值提成规则：当前需求文档为空，需补充 +4. 赠送卡分类：台费卡/酒水卡/抵用券的区分字段 + +--- + +## 七、验收标准 + +### 7.1 功能验收 + +- [ ] 所有 DWS 表可正常建表、查询 +- [ ] 助教月度汇总数据准确（与手工计算对比） +- [ ] 档位计算逻辑符合业务规则 +- [ ] 工资计算结果正确 +- [ ] 客户统计支持多时间窗口 +- [ ] 财务汇总数据与 DWD 明细一致 + +### 7.2 文档验收 + +- [ ] `dws_tables_dictionary.md` 完整记录所有表和字段 +- [ ] `README.md` 包含 DWS 使用说明 +- [ ] 代码注释覆盖率 > 80% + +### 7.3 性能验收 + +- [ ] 日增量刷新 < 5 分钟 +- [ ] 月全量刷新 < 30 分钟 +- [ ] 常用查询响应 < 1 秒 + +--- + +## 八、附录 + +### A. 配置表初始数据 + +```sql +-- 绩效档位初始数据 +INSERT INTO billiards_dws.cfg_performance_tier +(tier_code, tier_name, tier_reason, min_hours, max_hours, base_deduction, bonus_ratio, vacation_days) +VALUES +(0, '0档 淘汰压力', '淘汰压力', 0, 100, 28, 0.50, 3), +(1, '1档 及格档', '重点激励', 100, 130, 18, 0.40, 4), +(2, '2档 良好档', '重点激励', 130, 160, 15, 0.38, 4), +(3, '3档 优秀档', '优秀标准', 160, 190, 13, 0.35, 5), +(4, '4档 卓越加速档', '高端人才倾斜', 190, 220, 10, 0.33, 6), +(5, '5档 冠军加速档', '高端人才倾斜', 220, NULL, 8, 0.30, NULL); + +-- 助教等级定价初始数据 +INSERT INTO billiards_dws.cfg_assistant_level_price +(level_code, level_name, base_price, bonus_price) +VALUES +(1, '初级', 98, 190), +(2, '中级', 108, 190), +(3, '高级', 118, 190), +(4, '星级', 138, 190), +(8, '助教管理', 0, 0); + +-- 奖金规则初始数据 +INSERT INTO billiards_dws.cfg_bonus_rules +(rule_type, rule_name, condition_type, condition_value, bonus_amount, priority, is_stackable) +VALUES +('SPRINT', '冲刺奖-190小时', 'HOURS_GTE', 190, 300, 1, FALSE), +('SPRINT', '冲刺奖-220小时', 'HOURS_GTE', 220, 800, 2, FALSE), +('TOP_RANK', 'Top1奖金', 'RANK_EQ', 1, 1000, 1, TRUE), +('TOP_RANK', 'Top2奖金', 'RANK_EQ', 2, 600, 2, TRUE), +('TOP_RANK', 'Top3奖金', 'RANK_EQ', 3, 400, 3, TRUE); +``` + +### B. CLI 命令示例 + +```bash +# 初始化 DWS Schema +python -m cli.main --pipeline-flow INGEST_ONLY --tasks INIT_DWS_SCHEMA + +# 初始化配置数据 +python -m cli.main --pipeline-flow INGEST_ONLY --tasks DWS_INIT_CONFIG + +# 构建助教日度明细（指定日期） +python -m cli.main --pipeline-flow INGEST_ONLY --tasks DWS_ASSISTANT_DAILY \ + --window-start "2026-01-01" --window-end "2026-01-31" + +# 构建助教月度汇总 +python -m cli.main --pipeline-flow INGEST_ONLY --tasks DWS_ASSISTANT_MONTHLY \ + --window-start "2026-01-01" --window-end "2026-02-01" + +# 构建财务日度汇总 +python -m cli.main --pipeline-flow INGEST_ONLY --tasks DWS_FINANCE_DAILY \ + --window-start "2026-01-01" --window-end "2026-01-31" + +# 一键刷新所有 DWS 表 +python -m cli.main --pipeline-flow INGEST_ONLY --tasks DWS_REFRESH_ALL +``` + +--- + +**文档版本历史** + +| 版本 | 日期 | 作者 | 变更说明 | +|------|------|------|----------| +| 1.0 | 2026-02-01 | AI Assistant | 初始版本 | diff --git a/etl_billiards/docs/DWS_任务计划_v2.md b/etl_billiards/docs/DWS_任务计划_v2.md new file mode 100644 index 0000000..7ed12c3 --- /dev/null +++ b/etl_billiards/docs/DWS_任务计划_v2.md @@ -0,0 +1,1353 @@ +# DWS 数据层实施任务计划 v2.0 + +> **版本**: 2.0 +> **创建日期**: 2026-02-01 +> **更新日期**: 2026-02-01 +> **项目路径**: `C:\dev\LLTQ\ETL\feiqiu-ETL` +> **目标**: 在已完成的 DWD 层数据基础上，完成 DWS（数据汇总层）的数据库设计与 ETL 实现 + +--- + +## 一、项目背景与现状分析 + +### 1.1 已有基础设施 + +| 层级 | 状态 | Schema | 说明 | +|------|------|--------|------| +| ODS（操作数据层） | ✅ 完成 | `billiards_ods` | 原始数据落地，20+ 张表 | +| DWD（明细数据层） | ✅ 完成 | `billiards_dwd` | 维度表 9 张 + 事实表 12 张 | +| DWS（汇总数据层） | 🔄 部分完成 | `billiards_dws` | 仅有 `dws_order_summary` 1 张 | + +### 1.2 DWD 核心表与关键字段 + +#### 1.2.1 助教服务相关 + +**dwd_assistant_service_log** - 助教服务流水表 + +| 字段 | 说明 | 枚举值 | +|------|------|--------| +| `skill_id` | 技能ID（用于判断课程类型） | 2790683529513797=基础课, 2790683529513798=附加课/激励课, 3039912271463941=包厢课 | +| `skill_name` | 技能名称 | "基础课", "附加课", "包厢课" | +| `assistant_level` | 助教等级 | 8=助教管理, 10=初级, 20=中级, 30=高级, 40=星级 | +| `order_assistant_type` | 服务类型 | 1=基础课或包厢课, 2=附加课/激励课 | +| `ledger_unit_price` | 单价（元/小时） | 98/108/118/138/190 | + +#### 1.2.2 会员卡相关 + +**dim_member_card_account** - 会员卡账户表 + +| card_type_id | 卡类型 | 说明 | 分类 | +|--------------|--------|------|------| +| 2793249295533893 | 储值卡 | 充值获得，可抵扣任意费用 | **现金卡**（充值） | +| 2791990152417157 | 台费卡 | 充值赠送，仅可抵扣台费 | **赠送卡** | +| 2793266846533445 | 活动抵用券 | 充值赠送，不可抵扣助教费 | **赠送卡** | +| 2794699703437125 | 酒水卡 | 充值赠送，仅可抵扣酒水饮料食品 | **赠送卡** | +| 2793306611533637 | 月卡 | 充值获得，时长卡，仅可抵扣台费 | **时长卡** | +| 2791987095408517 | 年卡 | 充值获得，时长卡，仅可抵扣台费 | **时长卡** | + +#### 1.2.3 台区分布 + +**dim_table** - 台桌维度表 + +| 台区名称 | 台桌数 | 大类/索引 | +|----------|--------|-----------| +| A区 | 18 | 台球/打球/中八/追分 | +| B区 | 15 | 台球/打球/中八/追分 | +| C区 | 6 | 台球/打球/中八/追分 | +| VIP包厢 | 4 | 台球/打球/中八(V5为斯诺克) | +| 斯诺克区 | 4 | 台球/打球/斯诺克 | +| TV台 | 1 | 台球/打球/中八/追分 | +| 麻将房 | 5 | 麻将/麻将棋牌 | +| M7/M8/666/发财 | 6 | 麻将/麻将棋牌 | +| K包/k包活动区/幸会158 | 8 | K包/K歌/KTV | +| 补时长 | 7 | 补时长 | + +--- + +## 二、数据口径定义（数据来源矩阵） + +### 2.1 发生额/正价口径 + +| 项目 | DWD来源 | 计算公式 | 说明 | +|------|---------|----------|------| +| **台费正价** | dwd_settlement_head | `table_charge_money` | 结账单中的台费原价 | +| **商品正价** | dwd_settlement_head | `goods_money` | 结账单中的商品原价 | +| **助教基础课正价** | dwd_settlement_head | `assistant_pd_money` | 陪打费用（PD=陪打） | +| **助教激励课正价** | dwd_settlement_head | `assistant_cx_money` | 促销课费用（CX=促销/超休） | +| **团购台费正价** | dwd_groupbuy_redemption | `ledger_amount` | 团购券对应的台费正价 | +| **发生额合计** | dwd_settlement_head | `table_charge_money + goods_money + assistant_pd_money + assistant_cx_money` | 各项正价之和 | + +### 2.2 优惠分类口径 + +| 优惠类型 | DWD来源 | 计算公式 | 说明 | +|----------|---------|----------|------| +| **团购优惠** | dwd_groupbuy_redemption | `ledger_amount - coupon_money` | 正价与券面值差额 | +| **赠送卡抵扣** | dwd_member_balance_change | `SUM(change_amount) WHERE card_type_id IN (台费卡, 酒水卡, 活动抵用券) AND from_type=1` | 三类赠送卡消费合计 | +| **大客户优惠** | dwd_table_fee_log / dwd_store_goods_sale | `adjust_amount`（需抽样分析） | 手动调账产生的优惠 | +| **会员折扣** | dwd_settlement_head | `member_discount_amount` | 会员身份折扣 | +| **抹零** | dwd_settlement_head | `rounding_amount` | 抹零金额 | +| **其他优惠** | 待确认 | 抽样100单分析 | 订单折扣、台桌折扣、商品折扣、手动优惠的关系 | +| **优惠合计** | dwd_settlement_head | `consume_money - pay_amount` | 消费总额与实付差额 | + +### 2.3 支付方式口径 + +| 支付类型 | DWD来源 | 计算公式 | 说明 | +|----------|---------|----------|------| +| **储值卡支付** | dwd_settlement_head | `balance_amount + recharge_card_amount` | 余额+储值卡 | +| **赠送卡支付** | dwd_settlement_head | `gift_card_amount` | 礼品卡（赠送卡）支付 | +| **团购核销** | dwd_groupbuy_redemption | `coupon_money` | 团购券面值（核销价） | +| **现金/线上支付** | dwd_settlement_head | `pay_amount - balance_amount - recharge_card_amount - gift_card_amount - coupon_amount` | 扣除卡支付后的实付 | + +### 2.4 充值与赠送口径 + +| 类型 | DWD来源 | 筛选条件 | 说明 | +|------|---------|----------|------| +| **储值卡充值** | dwd_recharge_order | `member_card_type_name='储值卡' AND settle_type=5` | 现金充值到储值卡 | +| **首充金额** | dwd_recharge_order | `is_first=1` | 首次充值 | +| **续费金额** | dwd_recharge_order | `is_first=2` | 非首次充值 | +| **赠送卡新增** | dwd_member_balance_change | `card_type_id IN (台费卡,酒水卡,活动抵用券) AND from_type=4` | 活动赠送 | +| **赠送卡消费** | dwd_member_balance_change | `card_type_id IN (...) AND from_type=1 AND change_amount<0` | 消费扣款 | +| **各类卡余额** | dim_member_card_account | `SUM(balance) WHERE scd2_is_current=1 GROUP BY card_type_id` | 当前余额快照 | + +### 2.5 助教业绩口径 + +| 指标 | DWD来源 | 筛选条件 | 说明 | +|------|---------|----------|------| +| **基础课时长** | dwd_assistant_service_log | `skill_id=2790683529513797 AND is_delete=0` | 基础课秒数 | +| **附加课时长** | dwd_assistant_service_log | `skill_id=2790683529513798 AND is_delete=0` | 附加课/激励课秒数 | +| **包厢课时长** | dwd_assistant_service_log | `skill_id=3039912271463941 AND is_delete=0` | 包厢课秒数 | +| **有效服务** | dwd_assistant_service_log | 排除 dwd_assistant_trash_event 中已废除的记录 | 需LEFT JOIN排除 | +| **废除记录** | dwd_assistant_trash_event | 全量 | 废除的服务不计入业绩 | + +### 2.6 SCD2 维度取数说明 + +> **重要**：历史月份统计需要按业务时间点做 as-of join，而非直接使用 `scd2_is_current=1` + +```sql +-- 正确做法：按业务时间获取当时的维度版本 +SELECT asl.*, da.level, da.level_name +FROM dwd_assistant_service_log asl +LEFT JOIN dim_assistant da + ON asl.site_assistant_id = da.assistant_id + AND asl.start_use_time >= da.scd2_start_time + AND asl.start_use_time < COALESCE(da.scd2_end_time, '9999-12-31') +WHERE ... + +-- 错误做法：历史月份会套用当前等级 +SELECT asl.*, da.level +FROM dwd_assistant_service_log asl +LEFT JOIN dim_assistant da ON ... AND da.scd2_is_current = 1 -- 错误！ +``` + +--- + +## 三、DWS 表设计 + +### 3.1 表清单总览 + +#### 3.1.1 配置表（4张） + +| 表名 | 说明 | 数据来源 | +|------|------|----------| +| `cfg_performance_tier` | 绩效档位配置（6档） | 初始化+手动维护 | +| `cfg_assistant_level_price` | 助教等级定价 | 初始化+手动维护 | +| `cfg_bonus_rules` | 奖金规则配置 | 初始化+手动维护 | +| `cfg_area_category` | 台区分类映射 | 初始化+手动维护 | + +#### 3.1.2 助教维度表（5张） + +| 表名 | 说明 | 更新频率 | +|------|------|----------| +| `dws_assistant_monthly_summary` | 月度业绩汇总 | 每日增量 | +| `dws_assistant_daily_detail` | 日度业绩明细 | 每小时增量 | +| `dws_assistant_customer_stats` | 服务客户统计 | 每日全量 | +| `dws_assistant_salary_calc` | 月度工资计算 | 月初全量 | +| `dws_assistant_recharge_commission` | 充值提成记录 | **手动导入** | + +#### 3.1.3 客户维度表（2张） + +| 表名 | 说明 | 更新频率 | +|------|------|----------| +| `dws_member_consumption_summary` | 消费情况统计 | 每日全量 | +| `dws_member_visit_detail` | 来店消费明细 | 每日增量 | + +#### 3.1.4 财务维度表（7张） + +| 表名 | 说明 | 更新频率 | +|------|------|----------| +| `dws_finance_daily_summary` | 日度财务汇总 | 每小时增量 | +| `dws_finance_income_structure` | 收入结构 | 每日增量 | +| `dws_finance_discount_detail` | 优惠明细 | 每日增量 | +| `dws_finance_recharge_summary` | 充值与预收 | 每日增量 | +| `dws_finance_expense_summary` | 支出结构 | **手动导入** | +| `dws_assistant_finance_analysis` | 助教收支分析 | 每日增量 | +| `dws_platform_settlement` | 平台回款/服务费 | **手动导入** | + +### 3.2 配置表 DDL + +#### 3.2.1 `cfg_performance_tier` - 绩效档位配置 + +```sql +-- ============================================================ +-- 表名: cfg_performance_tier +-- 用途: 助教绩效档位配置表，定义6个档位的阈值与分成规则 +-- 业务规则: +-- - 过档后，所有时长按新档位进行计算 +-- - 总业绩小时数 = 基础课 + 附加课 + 包厢课 +-- 数据来源: 初始化脚本 + 手动维护 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_performance_tier ( + tier_id SERIAL PRIMARY KEY, -- 档位主键 + tier_code INTEGER NOT NULL UNIQUE, -- 档位代码：0-5 + tier_name VARCHAR(50) NOT NULL, -- 档位名称 + tier_reason VARCHAR(100), -- 档位设计原因 + min_hours NUMERIC(10,2) NOT NULL, -- 最小小时数阈值（含） + max_hours NUMERIC(10,2), -- 最大小时数阈值（不含），NULL=无上限 + base_deduction NUMERIC(10,2) NOT NULL, -- 基础课球房抽成（元/小时） + bonus_ratio NUMERIC(5,4) NOT NULL, -- 附加课球房抽成比例（如0.50表示50%） + vacation_days INTEGER, -- 次月休假天数，NULL=休假自由 + effective_from DATE NOT NULL DEFAULT '2026-01-01', -- 生效开始日期 + effective_to DATE DEFAULT '9999-12-31', -- 生效结束日期 + is_active BOOLEAN DEFAULT TRUE, -- 是否启用 + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +COMMENT ON TABLE billiards_dws.cfg_performance_tier IS '助教绩效档位配置表：定义各档位的业绩阈值与分成规则'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.tier_code IS '档位代码：0=淘汰压力，1=及格档，2=良好档，3=优秀档，4=卓越加速档，5=冠军加速档'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.base_deduction IS '基础课球房抽成金额（元/小时），助教实得=客户单价-此值'; +COMMENT ON COLUMN billiards_dws.cfg_performance_tier.bonus_ratio IS '附加课球房抽成比例，助教实得=客户支付*(1-此值)'; +``` + +#### 3.2.2 `cfg_assistant_level_price` - 助教等级定价 + +```sql +-- ============================================================ +-- 表名: cfg_assistant_level_price +-- 用途: 助教等级对应的客户收费标准 +-- 数据来源: 初始化脚本 + 手动维护 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_assistant_level_price ( + price_id SERIAL PRIMARY KEY, + level_code INTEGER NOT NULL, -- 等级代码（对应dim_assistant.assistant_level） + level_name VARCHAR(20) NOT NULL, -- 等级名称：初级/中级/高级/星级 + base_price NUMERIC(10,2) NOT NULL, -- 基础课客户收费（元/小时） + bonus_price NUMERIC(10,2) NOT NULL DEFAULT 190, -- 附加课客户收费（元/小时），统一190 + effective_from DATE NOT NULL DEFAULT '2026-01-01', + effective_to DATE DEFAULT '9999-12-31', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + UNIQUE(level_code, effective_from) +); + +COMMENT ON TABLE billiards_dws.cfg_assistant_level_price IS '助教等级定价表：定义各等级对客户的收费标准'; +COMMENT ON COLUMN billiards_dws.cfg_assistant_level_price.level_code IS '等级代码：8=助教管理，10=初级，20=中级，30=高级，40=星级'; +``` + +#### 3.2.3 `cfg_bonus_rules` - 奖金规则配置 + +```sql +-- ============================================================ +-- 表名: cfg_bonus_rules +-- 用途: 冲刺奖、Top3奖等额外奖金规则配置 +-- 业务规则: +-- - 冲刺奖：H>=190得300元，H>=220得800元（取高不叠加） +-- - Top3奖：第1名1000元，第2名600元，第3名400元 +-- 数据来源: 初始化脚本 + 手动维护 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_bonus_rules ( + rule_id SERIAL PRIMARY KEY, + rule_type VARCHAR(20) NOT NULL, -- 规则类型：SPRINT(冲刺奖)/TOP_RANK(排名奖) + rule_name VARCHAR(50) NOT NULL, -- 规则名称 + condition_type VARCHAR(20) NOT NULL, -- 条件类型：HOURS_GTE(时长>=)/RANK_EQ(排名=) + condition_value NUMERIC(10,2) NOT NULL, -- 条件值 + bonus_amount NUMERIC(10,2) NOT NULL, -- 奖金金额（元） + priority INTEGER DEFAULT 0, -- 优先级（同类型取高优先级） + is_stackable BOOLEAN DEFAULT FALSE, -- 是否可叠加 + effective_from DATE NOT NULL DEFAULT '2026-01-01', + effective_to DATE DEFAULT '9999-12-31', + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +COMMENT ON TABLE billiards_dws.cfg_bonus_rules IS '奖金规则配置表：定义冲刺奖和Top3排名奖'; +COMMENT ON COLUMN billiards_dws.cfg_bonus_rules.is_stackable IS '是否可叠加：FALSE表示同类型奖金取高不叠加'; +``` + +#### 3.2.4 `cfg_area_category` - 台区分类映射 + +```sql +-- ============================================================ +-- 表名: cfg_area_category +-- 用途: 台区名称到业务大类的映射，用于财务报表分类筛选 +-- 数据来源: 基于 dim_table.site_table_area_name 初始化 + 手动维护 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.cfg_area_category ( + category_id SERIAL PRIMARY KEY, + area_name VARCHAR(50) NOT NULL UNIQUE, -- 原始台区名称（来自dim_table） + category_l1 VARCHAR(30) NOT NULL, -- 一级分类：台球/麻将/K包/其他 + category_l2 VARCHAR(30), -- 二级分类：大厅/包厢/斯诺克等 + display_name VARCHAR(50), -- 显示名称（用于报表） + sort_order INTEGER DEFAULT 0, -- 排序序号 + is_active BOOLEAN DEFAULT TRUE, + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +COMMENT ON TABLE billiards_dws.cfg_area_category IS '台区分类映射表：将自由文本的台区名称映射到规范分类'; +``` + +### 3.3 助教维度表 DDL + +#### 3.3.1 `dws_assistant_monthly_summary` - 月度业绩汇总 + +```sql +-- ============================================================ +-- 表名: dws_assistant_monthly_summary +-- 用途: 助教月度业绩汇总表，核心统计表 +-- 更新策略: 每日增量更新当月数据，月初全量刷新上月 +-- 数据来源矩阵: +-- base_service_* → dwd_assistant_service_log WHERE skill_id=2790683529513797 +-- bonus_service_* → dwd_assistant_service_log WHERE skill_id=2790683529513798 +-- room_service_* → dwd_assistant_service_log WHERE skill_id=3039912271463941 +-- trashed_* → dwd_assistant_trash_event +-- tier_* → cfg_performance_tier +-- assistant_level → dim_assistant（按业务时间as-of join） +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_monthly_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, -- 门店ID + assistant_id BIGINT NOT NULL, -- 助教ID（dim_assistant.assistant_id） + stat_month DATE NOT NULL, -- 统计月份（每月1日） + + -- ========== 基础课统计 ========== + base_service_count INTEGER DEFAULT 0, -- 基础课服务次数 + base_service_seconds INTEGER DEFAULT 0, -- 基础课服务秒数 + base_service_hours NUMERIC(10,4) DEFAULT 0, -- 基础课服务小时数（秒数/3600，精确到分钟） + base_income_original NUMERIC(12,2) DEFAULT 0, -- 基础课原始收入（客户支付） + base_income_deduction NUMERIC(12,2) DEFAULT 0, -- 基础课球房抽成 + base_income_actual NUMERIC(12,2) DEFAULT 0, -- 基础课助教实得 + + -- ========== 附加课/激励课统计 ========== + bonus_service_count INTEGER DEFAULT 0, -- 附加课服务次数 + bonus_service_seconds INTEGER DEFAULT 0, -- 附加课服务秒数 + bonus_service_hours NUMERIC(10,4) DEFAULT 0, -- 附加课服务小时数 + bonus_income_original NUMERIC(12,2) DEFAULT 0, -- 附加课原始收入 + bonus_income_ratio_ded NUMERIC(12,2) DEFAULT 0, -- 附加课球房按比例抽成 + bonus_income_actual NUMERIC(12,2) DEFAULT 0, -- 附加课助教实得 + + -- ========== 包厢课统计 ========== + room_service_count INTEGER DEFAULT 0, -- 包厢课服务次数 + room_service_seconds INTEGER DEFAULT 0, -- 包厢课服务秒数 + room_service_hours NUMERIC(10,4) DEFAULT 0, -- 包厢课服务小时数 + room_income_original NUMERIC(12,2) DEFAULT 0, -- 包厢课原始收入 + room_income_actual NUMERIC(12,2) DEFAULT 0, -- 包厢课助教实得 + + -- ========== 汇总统计 ========== + total_service_count INTEGER DEFAULT 0, -- 总服务次数 + total_service_hours NUMERIC(10,4) DEFAULT 0, -- 总服务小时数（用于档位计算） + total_income_original NUMERIC(12,2) DEFAULT 0, -- 总原始收入 + total_income_actual NUMERIC(12,2) DEFAULT 0, -- 总助教实得 + + -- ========== 废除影响 ========== + trashed_service_count INTEGER DEFAULT 0, -- 被废除服务次数 + trashed_service_seconds INTEGER DEFAULT 0, -- 被废除服务秒数 + trashed_amount NUMERIC(12,2) DEFAULT 0, -- 被废除金额 + + -- ========== 档位信息（根据total_service_hours计算）========== + tier_code INTEGER, -- 当前档位代码（0-5） + tier_name VARCHAR(50), -- 档位名称 + tier_base_deduction NUMERIC(10,2), -- 档位基础课抽成 + tier_bonus_ratio NUMERIC(5,4), -- 档位附加课抽成比例 + + -- ========== 奖金 ========== + sprint_bonus NUMERIC(10,2) DEFAULT 0, -- 冲刺奖金额 + rank_bonus NUMERIC(10,2) DEFAULT 0, -- 排名奖金额 + recharge_commission NUMERIC(10,2) DEFAULT 0, -- 充值提成（关联dws_assistant_recharge_commission） + other_bonus NUMERIC(10,2) DEFAULT 0, -- 其他奖金 + total_bonus NUMERIC(10,2) DEFAULT 0, -- 奖金合计 + + -- ========== 最终工资 ========== + final_salary NUMERIC(12,2) DEFAULT 0, -- 最终应发工资 + + -- ========== 助教快照信息（按stat_month首日as-of join）========== + assistant_level INTEGER, -- 助教等级（8/10/20/30/40） + assistant_level_name VARCHAR(20), -- 等级名称 + assistant_level_price NUMERIC(10,2), -- 等级对应客户单价 + entry_date DATE, -- 入职日期 + is_new_employee BOOLEAN DEFAULT FALSE, -- 是否本月新入职 + work_days_in_month INTEGER, -- 本月在职天数 + + -- ========== 排名信息 ========== + monthly_rank INTEGER, -- 当月排名（按total_service_hours） + + -- ========== 元数据 ========== + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_month) +); + +-- 索引优化 +CREATE INDEX IF NOT EXISTS idx_dws_asst_monthly_site_month + ON billiards_dws.dws_assistant_monthly_summary(site_id, stat_month); +CREATE INDEX IF NOT EXISTS idx_dws_asst_monthly_assistant + ON billiards_dws.dws_assistant_monthly_summary(assistant_id, stat_month); +CREATE INDEX IF NOT EXISTS idx_dws_asst_monthly_month + ON billiards_dws.dws_assistant_monthly_summary(stat_month); + +COMMENT ON TABLE billiards_dws.dws_assistant_monthly_summary IS '助教月度业绩汇总表：统计每位助教每月的业绩、收入、档位、工资'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.total_service_hours IS '总服务小时数=基础课+附加课+包厢课，用于档位计算'; +COMMENT ON COLUMN billiards_dws.dws_assistant_monthly_summary.assistant_level IS '按stat_month首日通过as-of join获取的历史等级，非当前等级'; +``` + +#### 3.3.2 `dws_assistant_daily_detail` - 日度业绩明细 + +```sql +-- ============================================================ +-- 表名: dws_assistant_daily_detail +-- 用途: 助教日度业绩明细，用于月中进度追踪 +-- 更新策略: 每小时增量更新 +-- 数据来源: dwd_assistant_service_log + dwd_assistant_trash_event +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_daily_detail ( + detail_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, + stat_date DATE NOT NULL, -- 统计日期 + + -- 基础课日统计 + base_service_count INTEGER DEFAULT 0, + base_service_seconds INTEGER DEFAULT 0, + base_service_hours NUMERIC(10,4) DEFAULT 0, + base_income_original NUMERIC(12,2) DEFAULT 0, + + -- 附加课日统计 + bonus_service_count INTEGER DEFAULT 0, + bonus_service_seconds INTEGER DEFAULT 0, + bonus_service_hours NUMERIC(10,4) DEFAULT 0, + bonus_income_original NUMERIC(12,2) DEFAULT 0, + + -- 包厢课日统计 + room_service_count INTEGER DEFAULT 0, + room_service_seconds INTEGER DEFAULT 0, + room_service_hours NUMERIC(10,4) DEFAULT 0, + room_income_original NUMERIC(12,2) DEFAULT 0, + + -- 汇总 + total_service_count INTEGER DEFAULT 0, + total_service_hours NUMERIC(10,4) DEFAULT 0, + total_income_original NUMERIC(12,2) DEFAULT 0, + + -- 废除 + trashed_count INTEGER DEFAULT 0, + trashed_seconds INTEGER DEFAULT 0, + trashed_amount NUMERIC(12,2) DEFAULT 0, + + -- 累计（当月至今） + mtd_total_hours NUMERIC(10,4) DEFAULT 0, -- Month-To-Date 累计小时数 + mtd_tier_code INTEGER, -- 累计后档位 + + -- 服务客户统计 + unique_customers INTEGER DEFAULT 0, -- 当日服务不同客户数 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_date) +); + +CREATE INDEX IF NOT EXISTS idx_dws_asst_daily_site_date + ON billiards_dws.dws_assistant_daily_detail(site_id, stat_date); +CREATE INDEX IF NOT EXISTS idx_dws_asst_daily_assistant + ON billiards_dws.dws_assistant_daily_detail(assistant_id, stat_date); + +COMMENT ON TABLE billiards_dws.dws_assistant_daily_detail IS '助教日度业绩明细表：便于实时追踪月中业绩进度'; +``` + +#### 3.3.3 `dws_assistant_customer_stats` - 服务客户统计 + +```sql +-- ============================================================ +-- 表名: dws_assistant_customer_stats +-- 用途: 助教服务客户统计（多时间窗口） +-- 更新策略: 每日凌晨全量刷新 +-- 数据来源: dwd_assistant_service_log +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_customer_stats ( + stats_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, + stat_date DATE NOT NULL, -- 统计基准日期 + window_days INTEGER NOT NULL, -- 时间窗口天数：7/10/15/30/60/90 + + -- 客户统计 + unique_customers INTEGER DEFAULT 0, -- 服务不同客户数 + total_service_count INTEGER DEFAULT 0, -- 总服务次数 + total_service_hours NUMERIC(10,2) DEFAULT 0, -- 总服务小时数 + + -- 按课程类型 + base_customers INTEGER DEFAULT 0, -- 基础课客户数 + base_service_count INTEGER DEFAULT 0, + bonus_customers INTEGER DEFAULT 0, -- 附加课客户数 + bonus_service_count INTEGER DEFAULT 0, + + -- 收入 + total_income NUMERIC(12,2) DEFAULT 0, + + -- 客户复购 + repeat_customers INTEGER DEFAULT 0, -- 复购客户数（服务>=2次） + repeat_rate NUMERIC(5,4) DEFAULT 0, -- 复购率 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_date, window_days) +); + +CREATE INDEX IF NOT EXISTS idx_dws_asst_cust_window + ON billiards_dws.dws_assistant_customer_stats(assistant_id, stat_date, window_days); + +COMMENT ON TABLE billiards_dws.dws_assistant_customer_stats IS '助教服务客户统计表：支持7/10/15/30/60/90天多时间窗口'; +``` + +#### 3.3.4 `dws_assistant_salary_calc` - 月度工资计算 + +```sql +-- ============================================================ +-- 表名: dws_assistant_salary_calc +-- 用途: 助教月度工资计算详情，记录计算过程便于核对 +-- 更新策略: 月初全量计算上月工资 +-- 数据来源: dws_assistant_monthly_summary + cfg_* 配置表 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_salary_calc ( + calc_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, + stat_month DATE NOT NULL, -- 统计月份 + + -- ========== 输入参数 ========== + assistant_level INTEGER, -- 助教等级 + level_base_price NUMERIC(10,2), -- 等级基础课单价 + tier_code INTEGER, -- 最终档位 + tier_base_deduction NUMERIC(10,2), -- 档位抽成 + tier_bonus_ratio NUMERIC(5,4), -- 附加课抽成比例 + + -- ========== 时长输入 ========== + base_hours NUMERIC(10,4), -- 基础课小时数 + bonus_hours NUMERIC(10,4), -- 附加课小时数 + room_hours NUMERIC(10,4), -- 包厢课小时数 + total_hours NUMERIC(10,4), -- 总小时数 + + -- ========== 计算过程 ========== + -- 基础课收入 = base_hours * (level_base_price - tier_base_deduction) + base_calc_unit_income NUMERIC(10,2), -- 基础课单小时助教收入 + base_calc_income NUMERIC(12,2), -- 基础课收入 + + -- 附加课收入 = bonus_原始收入 * (1 - tier_bonus_ratio) + bonus_original_income NUMERIC(12,2), -- 附加课原始收入 + bonus_calc_income NUMERIC(12,2), -- 附加课助教收入 + + -- 包厢课收入 + room_calc_income NUMERIC(12,2), -- 包厢课收入 + + -- ========== 奖金计算 ========== + sprint_rule_matched VARCHAR(50), -- 匹配的冲刺奖规则 + sprint_bonus NUMERIC(10,2) DEFAULT 0, -- 冲刺奖 + rank_value INTEGER, -- 排名 + rank_rule_matched VARCHAR(50), -- 匹配的排名奖规则 + rank_bonus NUMERIC(10,2) DEFAULT 0, -- 排名奖 + recharge_commission NUMERIC(10,2) DEFAULT 0, -- 充值提成 + other_bonus NUMERIC(10,2) DEFAULT 0, -- 其他奖金 + + -- ========== 新入职折算 ========== + is_new_employee BOOLEAN DEFAULT FALSE, -- 是否新入职 + entry_date DATE, -- 入职日期 + work_days INTEGER, -- 在职天数 + projected_hours NUMERIC(10,4), -- 折算后小时数（用于定档） + max_tier_limit INTEGER, -- 最高档位限制（25日后入职限3档） + + -- ========== 最终结果 ========== + total_service_income NUMERIC(12,2), -- 服务收入合计 + total_bonus NUMERIC(10,2), -- 奖金合计 + final_salary NUMERIC(12,2), -- 最终应发工资 + + -- ========== 元数据 ========== + calc_time TIMESTAMPTZ DEFAULT now(), -- 计算时间 + calc_version INTEGER DEFAULT 1, -- 计算版本 + remark TEXT, -- 备注 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, assistant_id, stat_month, calc_version) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_salary_calc IS '助教月度工资计算详情表：记录计算过程便于核对和审计'; +``` + +#### 3.3.5 `dws_assistant_recharge_commission` - 充值提成记录 + +```sql +-- ============================================================ +-- 表名: dws_assistant_recharge_commission +-- 用途: 助教充值提成记录（手动导入） +-- 更新策略: 通过Excel等方式手动导入 +-- 数据来源: 外部Excel文件 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_recharge_commission ( + commission_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, -- 助教ID + stat_month DATE NOT NULL, -- 统计月份 + + -- 充值关联 + recharge_order_id BIGINT, -- 充值订单ID（如有） + member_card_id BIGINT, -- 储值卡ID + member_name VARCHAR(100), -- 会员名称 + member_phone VARCHAR(50), -- 会员电话 + + -- 充值信息 + recharge_time TIMESTAMPTZ, -- 充值时间 + recharge_amount NUMERIC(12,2) NOT NULL, -- 充值金额 + commission_rate NUMERIC(5,4), -- 提成比例 + commission_amount NUMERIC(10,2) NOT NULL, -- 提成金额 + + -- 导入信息 + import_batch VARCHAR(50), -- 导入批次号 + import_time TIMESTAMPTZ DEFAULT now(), -- 导入时间 + import_source VARCHAR(100), -- 来源文件名 + remark TEXT, + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_dws_asst_recharge_month + ON billiards_dws.dws_assistant_recharge_commission(assistant_id, stat_month); + +COMMENT ON TABLE billiards_dws.dws_assistant_recharge_commission IS '助教充值提成记录表：通过Excel等方式手动导入'; +``` + +### 3.4 客户维度表 DDL + +#### 3.4.1 `dws_member_consumption_summary` - 消费情况统计 + +```sql +-- ============================================================ +-- 表名: dws_member_consumption_summary +-- 用途: 会员消费情况统计（多时间窗口） +-- 更新策略: 每日凌晨全量刷新 +-- 数据来源矩阵: +-- visit_count → dwd_settlement_head（按order_settle_id去重） +-- table_fee_* → dwd_table_fee_log +-- assistant_* → dwd_assistant_service_log +-- goods_* → dwd_store_goods_sale +-- 金额字段 → dwd_settlement_head +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_member_consumption_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + member_id BIGINT NOT NULL, -- dim_member.member_id + stat_date DATE NOT NULL, -- 统计基准日期 + window_days INTEGER NOT NULL, -- 7/10/15/30/60/90 + + -- 到店统计 + visit_count INTEGER DEFAULT 0, -- 到店次数（结账单数） + visit_days INTEGER DEFAULT 0, -- 到店天数 + + -- 台费消费 + table_fee_count INTEGER DEFAULT 0, -- 开台次数 + table_fee_seconds INTEGER DEFAULT 0, -- 台费时长（秒） + table_fee_hours NUMERIC(10,2) DEFAULT 0, -- 台费小时数 + table_fee_amount NUMERIC(12,2) DEFAULT 0, -- 台费金额（正价） + table_fee_actual NUMERIC(12,2) DEFAULT 0, -- 台费实付 + + -- 助教服务 + assistant_service_count INTEGER DEFAULT 0, -- 助教服务次数 + assistant_service_hours NUMERIC(10,2) DEFAULT 0, -- 助教服务小时数 + assistant_service_amt NUMERIC(12,2) DEFAULT 0, -- 助教服务金额 + unique_assistants INTEGER DEFAULT 0, -- 服务过的不同助教数 + + -- 商品消费 + goods_order_count INTEGER DEFAULT 0, -- 商品订单数 + goods_item_count INTEGER DEFAULT 0, -- 商品项数 + goods_amount NUMERIC(12,2) DEFAULT 0, -- 商品金额 + + -- 汇总金额（来源：dwd_settlement_head） + total_consume_money NUMERIC(12,2) DEFAULT 0, -- 消费总额（consume_money） + total_discount_amount NUMERIC(12,2) DEFAULT 0, -- 优惠总额 + total_pay_amount NUMERIC(12,2) DEFAULT 0, -- 实付总额（pay_amount） + + -- 支付方式 + cash_payment NUMERIC(12,2) DEFAULT 0, -- 现金/线上支付 + stored_card_payment NUMERIC(12,2) DEFAULT 0, -- 储值卡支付 + gift_card_payment NUMERIC(12,2) DEFAULT 0, -- 赠送卡支付 + coupon_payment NUMERIC(12,2) DEFAULT 0, -- 团购券支付 + + -- 客户价值指标 + avg_visit_amount NUMERIC(10,2) DEFAULT 0, -- 单次到店平均消费 + avg_visit_duration_min INTEGER DEFAULT 0, -- 单次平均停留时长（分钟） + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, member_id, stat_date, window_days) +); + +CREATE INDEX IF NOT EXISTS idx_dws_member_cons_window + ON billiards_dws.dws_member_consumption_summary(member_id, stat_date, window_days); +CREATE INDEX IF NOT EXISTS idx_dws_member_cons_site + ON billiards_dws.dws_member_consumption_summary(site_id, stat_date, window_days); + +COMMENT ON TABLE billiards_dws.dws_member_consumption_summary IS '会员消费汇总表：支持7/10/15/30/60/90天多时间窗口'; +``` + +#### 3.4.2 `dws_member_visit_detail` - 来店消费明细 + +```sql +-- ============================================================ +-- 表名: dws_member_visit_detail +-- 用途: 会员来店消费明细表，记录每次到店的详细消费 +-- 更新策略: 每日增量更新 +-- 数据来源: dwd_settlement_head + 各明细表JOIN +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_member_visit_detail ( + visit_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + member_id BIGINT NOT NULL, + order_settle_id BIGINT NOT NULL, -- 结账单ID + order_trade_no BIGINT, -- 订单号 + + -- 时间信息 + visit_date DATE NOT NULL, -- 到店日期 + start_time TIMESTAMPTZ, -- 开台时间 + end_time TIMESTAMPTZ, -- 结账时间 + duration_minutes INTEGER, -- 停留时长（分钟） + + -- 台桌信息 + table_id BIGINT, -- 台桌ID + table_name VARCHAR(50), -- 台桌名称 + area_name VARCHAR(50), -- 台区名称 + area_category VARCHAR(30), -- 台区分类（映射后） + + -- 台费明细 + table_fee_seconds INTEGER DEFAULT 0, + table_fee_amount NUMERIC(12,2) DEFAULT 0, -- 台费正价 + table_fee_actual NUMERIC(12,2) DEFAULT 0, -- 台费实付 + + -- 助教服务明细 + assistant_count INTEGER DEFAULT 0, -- 助教服务人次 + assistant_names TEXT, -- 助教姓名列表（逗号分隔） + assistant_hours NUMERIC(10,2) DEFAULT 0, -- 助教总时长 + assistant_amount NUMERIC(12,2) DEFAULT 0, -- 助教费用 + + -- 商品明细 + goods_count INTEGER DEFAULT 0, -- 商品种类数 + goods_names TEXT, -- 商品名称列表（Top5） + goods_amount NUMERIC(12,2) DEFAULT 0, -- 商品金额 + + -- 金额汇总 + consume_money NUMERIC(12,2) DEFAULT 0, -- 消费总额 + discount_amount NUMERIC(12,2) DEFAULT 0, -- 优惠总额 + pay_amount NUMERIC(12,2) DEFAULT 0, -- 实付金额 + + -- 支付方式 + payment_method_desc VARCHAR(100), -- 支付方式描述 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, member_id, order_settle_id) +); + +CREATE INDEX IF NOT EXISTS idx_dws_member_visit_member + ON billiards_dws.dws_member_visit_detail(member_id, visit_date); +CREATE INDEX IF NOT EXISTS idx_dws_member_visit_date + ON billiards_dws.dws_member_visit_detail(site_id, visit_date); + +COMMENT ON TABLE billiards_dws.dws_member_visit_detail IS '会员来店消费明细表：记录每次到店的详细消费情况'; +``` + +### 3.5 财务维度表 DDL + +#### 3.5.1 `dws_finance_daily_summary` - 日度财务汇总 + +```sql +-- ============================================================ +-- 表名: dws_finance_daily_summary +-- 用途: 门店日度财务汇总（核心财务报表数据源） +-- 更新策略: 每小时增量更新 +-- 数据来源矩阵: +-- gross_* → dwd_settlement_head (table_charge_money, goods_money, assistant_pd_money, assistant_cx_money) +-- discount_* → dwd_settlement_head + dwd_member_balance_change + dwd_groupbuy_redemption +-- payment_* → dwd_settlement_head (balance_amount, recharge_card_amount, gift_card_amount, coupon_amount) +-- recharge_* → dwd_recharge_order +-- refund_* → dwd_refund +-- stored_card_* → dwd_member_balance_change WHERE card_type_id=储值卡 +-- gift_card_* → dwd_member_balance_change WHERE card_type_id IN (台费卡,酒水卡,活动抵用券) +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_daily_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + + -- ========== 经营链：发生额 → 优惠 → 确认收入 ========== + -- 发生额（正价）- 来源：dwd_settlement_head + gross_revenue NUMERIC(14,2) DEFAULT 0, -- 发生额/正价总计 + gross_table_fee NUMERIC(14,2) DEFAULT 0, -- 台费发生额 ← table_charge_money + gross_assistant_base NUMERIC(14,2) DEFAULT 0, -- 助教基础课发生额 ← assistant_pd_money + gross_assistant_bonus NUMERIC(14,2) DEFAULT 0, -- 助教激励课发生额 ← assistant_cx_money + gross_goods NUMERIC(14,2) DEFAULT 0, -- 食品酒水发生额 ← goods_money + + -- 优惠明细 + discount_total NUMERIC(14,2) DEFAULT 0, -- 优惠总额 = gross_revenue - confirmed_revenue + discount_groupbuy NUMERIC(14,2) DEFAULT 0, -- 团购优惠 ← dwd_groupbuy_redemption(ledger_amount - coupon_money) + discount_vip NUMERIC(14,2) DEFAULT 0, -- 大客户优惠（预留，需抽样分析） + discount_gift_card NUMERIC(14,2) DEFAULT 0, -- 赠送卡抵扣 ← dwd_member_balance_change(台费卡+酒水卡+活动抵用券消费) + discount_member NUMERIC(14,2) DEFAULT 0, -- 会员折扣 ← member_discount_amount + discount_manual NUMERIC(14,2) DEFAULT 0, -- 手动调整 ← adjust_amount + discount_free NUMERIC(14,2) DEFAULT 0, -- 免单（预留） + discount_rounding NUMERIC(14,2) DEFAULT 0, -- 抹零 ← rounding_amount + + -- 成交/确认收入 + confirmed_revenue NUMERIC(14,2) DEFAULT 0, -- 成交/确认收入（不含充值）← pay_amount汇总 + + -- ========== 支付方式构成 ========== + payment_stored_card NUMERIC(14,2) DEFAULT 0, -- 储值卡结算冲销 ← balance_amount + recharge_card_amount + payment_gift_card NUMERIC(14,2) DEFAULT 0, -- 赠送卡支付 ← gift_card_amount + payment_cash_online NUMERIC(14,2) DEFAULT 0, -- 现金/线上支付 = pay_amount - 卡支付 + payment_groupbuy_deal NUMERIC(14,2) DEFAULT 0, -- 团购核销确认收入 ← dwd_groupbuy_redemption.coupon_money + + -- ========== 现金流 ========== + -- 现金流入 + cash_inflow_consume NUMERIC(14,2) DEFAULT 0, -- 消费现金流入（现金+线上-退款） + cash_inflow_recharge NUMERIC(14,2) DEFAULT 0, -- 充值到账 ← dwd_recharge_order.pay_amount + cash_inflow_platform NUMERIC(14,2) DEFAULT 0, -- 平台回款（预留，手动导入） + cash_inflow_total NUMERIC(14,2) DEFAULT 0, -- 现金流入合计 + + -- 现金支出（预留，通过 dws_finance_expense_summary 手动导入） + cash_outflow_total NUMERIC(14,2) DEFAULT 0, -- 现金支出合计 + + -- 现金结余 + cash_balance NUMERIC(14,2) DEFAULT 0, -- 现金结余 = inflow - outflow + cash_balance_rate NUMERIC(5,4) DEFAULT 0, -- 结余率 = balance / inflow + + -- ========== 充值与预收 ========== + recharge_first NUMERIC(14,2) DEFAULT 0, -- 首充金额 ← is_first=1 + recharge_renew NUMERIC(14,2) DEFAULT 0, -- 续费金额 ← is_first=2 + recharge_total NUMERIC(14,2) DEFAULT 0, -- 充值合计 + recharge_count INTEGER DEFAULT 0, -- 充值笔数 + + -- 储值卡（card_type_id=2793249295533893） + stored_card_new NUMERIC(14,2) DEFAULT 0, -- 储值卡充值（from_type=3） + stored_card_consume NUMERIC(14,2) DEFAULT 0, -- 储值卡消耗（from_type=1，change_amount<0） + stored_card_balance_eod NUMERIC(14,2) DEFAULT 0, -- 储值卡余额（截止当日） + + -- 赠送卡（台费卡+酒水卡+活动抵用券） + gift_card_new NUMERIC(14,2) DEFAULT 0, -- 赠送卡新增（from_type=4） + gift_card_consume NUMERIC(14,2) DEFAULT 0, -- 赠送卡消耗 + gift_card_balance_eod NUMERIC(14,2) DEFAULT 0, -- 赠送卡余额 + + -- 细分赠送卡 + gift_card_table_new NUMERIC(14,2) DEFAULT 0, -- 台费卡新增 + gift_card_table_consume NUMERIC(14,2) DEFAULT 0, -- 台费卡消耗 + gift_card_table_balance NUMERIC(14,2) DEFAULT 0, -- 台费卡余额 + gift_card_drink_new NUMERIC(14,2) DEFAULT 0, -- 酒水卡新增 + gift_card_drink_consume NUMERIC(14,2) DEFAULT 0, -- 酒水卡消耗 + gift_card_drink_balance NUMERIC(14,2) DEFAULT 0, -- 酒水卡余额 + gift_card_coupon_new NUMERIC(14,2) DEFAULT 0, -- 活动抵用券新增 + gift_card_coupon_consume NUMERIC(14,2) DEFAULT 0, -- 活动抵用券消耗 + gift_card_coupon_balance NUMERIC(14,2) DEFAULT 0, -- 活动抵用券余额 + + -- ========== 订单统计 ========== + order_count INTEGER DEFAULT 0, -- 订单数 + order_avg_amount NUMERIC(10,2) DEFAULT 0, -- 客单价 + member_order_count INTEGER DEFAULT 0, -- 会员订单数 + member_order_rate NUMERIC(5,4) DEFAULT 0, -- 会员订单占比 + + -- ========== 退款统计 ========== + refund_count INTEGER DEFAULT 0, -- 退款笔数 + refund_amount NUMERIC(14,2) DEFAULT 0, -- 退款金额 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date) +); + +CREATE INDEX IF NOT EXISTS idx_dws_finance_daily_site_date + ON billiards_dws.dws_finance_daily_summary(site_id, stat_date); +CREATE INDEX IF NOT EXISTS idx_dws_finance_daily_date + ON billiards_dws.dws_finance_daily_summary(stat_date); + +COMMENT ON TABLE billiards_dws.dws_finance_daily_summary IS '门店日度财务汇总表：核心财务报表数据源'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.gross_revenue IS '发生额/正价：table_charge_money+goods_money+assistant_pd_money+assistant_cx_money'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.confirmed_revenue IS '成交/确认收入：扣除优惠后的营业收入，不含充值'; +COMMENT ON COLUMN billiards_dws.dws_finance_daily_summary.discount_gift_card IS '赠送卡抵扣：台费卡+酒水卡+活动抵用券的消费金额'; +``` + +#### 3.5.2 `dws_finance_income_structure` - 收入结构 + +```sql +-- ============================================================ +-- 表名: dws_finance_income_structure +-- 用途: 收入结构按区域/类型细分 +-- 更新策略: 每日增量更新 +-- 数据来源: dwd_table_fee_log + dwd_assistant_service_log + dwd_store_goods_sale +-- JOIN cfg_area_category 获取标准分类 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_income_structure ( + structure_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + + -- 分类维度 + income_category VARCHAR(20) NOT NULL, -- 收入大类：TABLE/ASSISTANT_BASE/ASSISTANT_BONUS/GOODS + area_name VARCHAR(50), -- 原始台区名称（dim_table.site_table_area_name） + area_category_l1 VARCHAR(30), -- 一级分类（cfg_area_category.category_l1） + area_category_l2 VARCHAR(30), -- 二级分类 + + -- 金额（三列式） + gross_amount NUMERIC(14,2) DEFAULT 0, -- 发生额 + discount_amount NUMERIC(14,2) DEFAULT 0, -- 优惠金额 + confirmed_amount NUMERIC(14,2) DEFAULT 0, -- 确认收入 + + -- 数量/时长 + item_count INTEGER DEFAULT 0, -- 项目数/次数 + duration_seconds INTEGER DEFAULT 0, -- 时长（秒） + duration_hours NUMERIC(10,2) DEFAULT 0, -- 时长（小时） + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date, income_category, COALESCE(area_name, '')) +); + +CREATE INDEX IF NOT EXISTS idx_dws_finance_structure_date + ON billiards_dws.dws_finance_income_structure(site_id, stat_date); + +COMMENT ON TABLE billiards_dws.dws_finance_income_structure IS '收入结构表：按大类和区域细分的收入明细'; +``` + +#### 3.5.3 `dws_finance_discount_detail` - 优惠明细 + +```sql +-- ============================================================ +-- 表名: dws_finance_discount_detail +-- 用途: 优惠类型明细统计 +-- 更新策略: 每日增量更新 +-- 数据来源矩阵: +-- GROUPBUY → dwd_groupbuy_redemption +-- GIFT_CARD_* → dwd_member_balance_change +-- MEMBER_DISCOUNT → dwd_settlement_head.member_discount_amount +-- MANUAL_ADJUST → dwd_settlement_head.adjust_amount +-- ROUNDING → dwd_settlement_head.rounding_amount +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_discount_detail ( + detail_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + + -- 优惠类型 + discount_type VARCHAR(30) NOT NULL, -- 优惠类型代码 + discount_type_name VARCHAR(50) NOT NULL, -- 优惠类型名称 + + -- 金额 + discount_amount NUMERIC(14,2) DEFAULT 0, -- 优惠金额 + order_count INTEGER DEFAULT 0, -- 涉及订单数 + + -- 占比 + discount_rate NUMERIC(5,4) DEFAULT 0, -- 占总优惠比例 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date, discount_type) +); + +-- 优惠类型枚举： +-- GROUPBUY: 团购优惠 +-- GIFT_CARD_TABLE: 台费卡抵扣 +-- GIFT_CARD_DRINK: 酒水卡抵扣 +-- GIFT_CARD_COUPON: 活动抵用券抵扣 +-- MEMBER_DISCOUNT: 会员折扣 +-- VIP_DISCOUNT: 大客户优惠 +-- MANUAL_ADJUST: 手动调整 +-- FREE_ORDER: 免单 +-- ROUNDING: 抹零 + +COMMENT ON TABLE billiards_dws.dws_finance_discount_detail IS '优惠明细统计表：按优惠类型细分的优惠金额统计'; +``` + +#### 3.5.4 `dws_finance_recharge_summary` - 充值与预收汇总 + +```sql +-- ============================================================ +-- 表名: dws_finance_recharge_summary +-- 用途: 充值与会员卡余额汇总 +-- 更新策略: 每日增量更新 +-- 数据来源: +-- 充值 → dwd_recharge_order +-- 余额变动 → dwd_member_balance_change +-- 期末余额 → dim_member_card_account +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_recharge_summary ( + summary_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + card_type VARCHAR(20) NOT NULL, -- 卡类型：STORED/TABLE_FEE/DRINK/COUPON/MONTH/YEAR + card_type_name VARCHAR(30) NOT NULL, -- 卡类型名称 + + -- 充值/新增 + recharge_count INTEGER DEFAULT 0, -- 充值/新增笔数 + recharge_amount NUMERIC(14,2) DEFAULT 0, -- 充值/新增金额 + first_recharge_count INTEGER DEFAULT 0, -- 首充笔数 + first_recharge_amount NUMERIC(14,2) DEFAULT 0, -- 首充金额 + + -- 消耗 + consume_count INTEGER DEFAULT 0, -- 消费笔数 + consume_amount NUMERIC(14,2) DEFAULT 0, -- 消费金额 + + -- 退款/调整 + refund_amount NUMERIC(14,2) DEFAULT 0, -- 退款金额 + adjust_amount NUMERIC(14,2) DEFAULT 0, -- 调整金额 + + -- 期末余额 + balance_eod NUMERIC(14,2) DEFAULT 0, -- 期末余额 + card_count INTEGER DEFAULT 0, -- 有效卡数量 + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date, card_type) +); + +COMMENT ON TABLE billiards_dws.dws_finance_recharge_summary IS '充值与会员卡汇总表：按卡类型统计充值、消耗、余额'; +``` + +#### 3.5.5 `dws_finance_expense_summary` - 支出结构（手动导入） + +```sql +-- ============================================================ +-- 表名: dws_finance_expense_summary +-- 用途: 支出结构汇总（手动导入） +-- 更新策略: 通过Excel等方式手动导入 +-- 数据来源: 外部Excel文件 +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_finance_expense_summary ( + expense_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_month DATE NOT NULL, -- 统计月份（每月1日） + + -- 支出类型 + expense_type VARCHAR(30) NOT NULL, -- 支出类型代码 + expense_type_name VARCHAR(50) NOT NULL, -- 支出类型名称 + + -- 金额 + expense_amount NUMERIC(14,2) NOT NULL, -- 支出金额 + + -- 导入信息 + import_batch VARCHAR(50), -- 导入批次号 + import_time TIMESTAMPTZ DEFAULT now(), -- 导入时间 + import_source VARCHAR(100), -- 来源文件名 + remark TEXT, + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_month, expense_type) +); + +-- 支出类型枚举： +-- RENT: 房租 +-- UTILITY: 水电 +-- PROPERTY: 物业费 +-- PURCHASE: 进货成本 +-- CONSUMABLE: 耗材 +-- REIMBURSE: 报销 +-- ASSISTANT_COMMISSION: 助教分成 +-- ASSISTANT_BONUS: 助教奖惩 +-- SALARY: 固定人员工资 +-- PLATFORM_MEITUAN: 美团服务费 +-- PLATFORM_DOUYIN: 抖音服务费 +-- PLATFORM_HUILAIMI: 汇来米平台服务费 +-- OTHER: 其他费用 + +COMMENT ON TABLE billiards_dws.dws_finance_expense_summary IS '支出结构汇总表：通过Excel等方式手动导入'; +``` + +#### 3.5.6 `dws_assistant_finance_analysis` - 助教收支分析 + +```sql +-- ============================================================ +-- 表名: dws_assistant_finance_analysis +-- 用途: 助教收支分析（财务视角） +-- 更新策略: 每日增量更新 +-- 数据来源: dwd_assistant_service_log + cfg_performance_tier + cfg_assistant_level_price +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_assistant_finance_analysis ( + analysis_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_date DATE NOT NULL, + assistant_level INTEGER, -- 助教等级（NULL表示全部） + assistant_level_name VARCHAR(20), + + -- 基础课 + base_customer_payment NUMERIC(14,2) DEFAULT 0, -- 客户支付 + base_shop_commission NUMERIC(14,2) DEFAULT 0, -- 球房抽成 + base_hours NUMERIC(10,2) DEFAULT 0, -- 小时数 + base_avg_commission NUMERIC(10,2) DEFAULT 0, -- 均小时抽成 + base_service_count INTEGER DEFAULT 0, -- 服务次数 + + -- 激励课 + bonus_customer_payment NUMERIC(14,2) DEFAULT 0, + bonus_shop_commission NUMERIC(14,2) DEFAULT 0, + bonus_hours NUMERIC(10,2) DEFAULT 0, + bonus_avg_commission NUMERIC(10,2) DEFAULT 0, + bonus_service_count INTEGER DEFAULT 0, + + -- 包厢课 + room_customer_payment NUMERIC(14,2) DEFAULT 0, + room_shop_commission NUMERIC(14,2) DEFAULT 0, + room_hours NUMERIC(10,2) DEFAULT 0, + room_service_count INTEGER DEFAULT 0, + + -- 汇总 + total_customer_payment NUMERIC(14,2) DEFAULT 0, + total_shop_commission NUMERIC(14,2) DEFAULT 0, + total_hours NUMERIC(10,2) DEFAULT 0, + + -- 助教数量 + assistant_count INTEGER DEFAULT 0, + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_date, COALESCE(assistant_level, 0)) +); + +COMMENT ON TABLE billiards_dws.dws_assistant_finance_analysis IS '助教收支分析表：财务视角的助教业绩与抽成统计'; +``` + +#### 3.5.7 `dws_platform_settlement` - 平台回款/服务费（手动导入） + +```sql +-- ============================================================ +-- 表名: dws_platform_settlement +-- 用途: 平台回款与服务费记录（手动导入） +-- 更新策略: 通过Excel等方式手动导入 +-- 数据来源: 外部Excel文件（美团/抖音/汇来米等平台结算单） +-- ============================================================ +CREATE TABLE IF NOT EXISTS billiards_dws.dws_platform_settlement ( + settlement_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + stat_month DATE NOT NULL, -- 结算月份 + + -- 平台信息 + platform_code VARCHAR(20) NOT NULL, -- 平台代码：MEITUAN/DOUYIN/HUILAIMI/OTHER + platform_name VARCHAR(50) NOT NULL, -- 平台名称 + + -- 核销统计 + redemption_count INTEGER DEFAULT 0, -- 核销笔数 + redemption_amount NUMERIC(14,2) DEFAULT 0, -- 核销券面值 + + -- 回款 + settlement_amount NUMERIC(14,2) DEFAULT 0, -- 平台回款金额 + settlement_date DATE, -- 回款日期 + + -- 服务费 + service_fee NUMERIC(14,2) DEFAULT 0, -- 平台服务费 + service_fee_rate NUMERIC(5,4), -- 服务费率 + + -- 差价 + price_diff NUMERIC(14,2) DEFAULT 0, -- 团购差价（正价-核销价） + + -- 导入信息 + import_batch VARCHAR(50), + import_time TIMESTAMPTZ DEFAULT now(), + import_source VARCHAR(100), + remark TEXT, + + created_at TIMESTAMPTZ DEFAULT now(), + updated_at TIMESTAMPTZ DEFAULT now(), + + UNIQUE(site_id, stat_month, platform_code) +); + +COMMENT ON TABLE billiards_dws.dws_platform_settlement IS '平台回款与服务费表：通过Excel等方式手动导入'; +``` + +--- + +## 四、数据分层策略 + +### 4.1 时间分层设计 + +| 分层 | 时间范围 | 用途 | 实现方式 | 财务支持 | +|------|----------|------|----------|----------| +| L1 热数据 | 最近2天 | 实时监控/当日报表 | 物化视图 + 每小时刷新 | ✓ | +| L2 近期 | 最近1月 | 周报/月报 | 分区表 | ✓ | +| L3 中期 | 最近3月 | 季度分析 | 分区表 | ✓ | +| L4 全量 | 最近6月 | 半年分析 | 分区表 | ✓ 财务特殊需求 | +| L5 历史 | 全部 | 年度汇总/深度分析 | 归档表 | ✓ | + +### 4.2 财务时间筛选支持 + +| UI筛选项 | 实现方式 | 环比支持 | +|----------|----------|----------| +| 本月 | 当前月 | vs 上月 | +| 上个月 | 上月全月 | vs 上上月 | +| 前3个月不加本月 | 过去3个完整月 | vs 更早3个月 | +| 前3个月+本月 | 过去3月+本月 | vs 更早4个月 | +| 最近半年不含本月 | 过去6个完整月 | vs 更早6个月 | +| 本季度含本月 | 当前季度 | vs 上季度 | +| 上个季度 | 上季度全季 | vs 上上季度 | +| 本周 | 当前周 | vs 上周 | +| 上周 | 上周全周 | vs 上上周 | + +--- + +## 五、实施任务清单 + +### 5.1 阶段一：基础设施（预计 1-2 天） + +| 序号 | 任务 | 产出物 | +|------|------|--------| +| 1.1 | 更新 `schema_dws.sql` 完整DDL | DDL文件 | +| 1.2 | 创建 `seed_dws_config.sql` 配置初始数据 | 种子数据 | +| 1.3 | 创建 `cfg_area_category` 台区映射数据 | 映射数据 | +| 1.4 | 更新 `InitDwsSchemaTask` | 任务代码 | +| 1.5 | 创建 `BaseDwsTask` 基类 | 基类代码 | + +### 5.2 阶段二：助教维度（预计 2-3 天） + +| 序号 | 任务 | 产出物 | +|------|------|--------| +| 2.1 | 实现 `AssistantDailyTask` | 日度明细 | +| 2.2 | 实现 `AssistantMonthlyTask` | 月度汇总 | +| 2.3 | 实现档位计算模块（含新入职折算） | 算法模块 | +| 2.4 | 实现工资计算模块 | 算法模块 | +| 2.5 | 实现 `AssistantCustomerTask` | 客户统计 | +| 2.6 | 实现充值提成导入功能 | 导入脚本 | + +### 5.3 阶段三：客户维度（预计 1-2 天） + +| 序号 | 任务 | 产出物 | +|------|------|--------| +| 3.1 | 实现 `MemberConsumptionTask` | 消费统计 | +| 3.2 | 实现 `MemberVisitDetailTask` | 来店明细 | + +### 5.4 阶段四：财务维度（预计 2-3 天） + +| 序号 | 任务 | 产出物 | +|------|------|--------| +| 4.1 | 实现 `FinanceDailySummaryTask` | 日度汇总 | +| 4.2 | 实现 `FinanceIncomeStructureTask` | 收入结构 | +| 4.3 | 实现 `FinanceDiscountDetailTask` | 优惠明细 | +| 4.4 | 实现 `FinanceRechargeSummaryTask` | 充值统计 | +| 4.5 | 实现 `AssistantFinanceTask` | 助教收支 | +| 4.6 | 实现支出/平台结算导入功能 | 导入脚本 | + +### 5.5 阶段五：文档与测试（预计 1-2 天） + +| 序号 | 任务 | 产出物 | +|------|------|--------| +| 5.1 | 编写 `dws_tables_dictionary.md` | 数据字典 | +| 5.2 | 更新 `README.md` | README | +| 5.3 | 抽样100单分析优惠口径 | 分析报告 | +| 5.4 | 编写单元测试 | 测试代码 | + +--- + +## 六、配置表初始数据 + +```sql +-- ============================================================ +-- 文件: seed_dws_config.sql +-- 用途: DWS 配置表初始数据 +-- ============================================================ + +-- 绩效档位配置 +INSERT INTO billiards_dws.cfg_performance_tier +(tier_code, tier_name, tier_reason, min_hours, max_hours, base_deduction, bonus_ratio, vacation_days) +VALUES +(0, '0档 淘汰压力', '淘汰压力', 0, 100, 28, 0.50, 3), +(1, '1档 及格档', '重点激励', 100, 130, 18, 0.40, 4), +(2, '2档 良好档', '重点激励', 130, 160, 15, 0.38, 4), +(3, '3档 优秀档', '优秀标准', 160, 190, 13, 0.35, 5), +(4, '4档 卓越加速档', '高端人才倾斜', 190, 220, 10, 0.33, 6), +(5, '5档 冠军加速档', '高端人才倾斜', 220, NULL, 8, 0.30, NULL); + +-- 助教等级定价（注意：level_code对应dim_assistant.assistant_level） +INSERT INTO billiards_dws.cfg_assistant_level_price +(level_code, level_name, base_price, bonus_price) +VALUES +(10, '初级', 98, 190), +(20, '中级', 108, 190), +(30, '高级', 118, 190), +(40, '星级', 138, 190), +(8, '助教管理', 0, 0); + +-- 奖金规则 +INSERT INTO billiards_dws.cfg_bonus_rules +(rule_type, rule_name, condition_type, condition_value, bonus_amount, priority, is_stackable) +VALUES +('SPRINT', '冲刺奖-190小时', 'HOURS_GTE', 190, 300, 1, FALSE), +('SPRINT', '冲刺奖-220小时', 'HOURS_GTE', 220, 800, 2, FALSE), +('TOP_RANK', 'Top1奖金', 'RANK_EQ', 1, 1000, 1, TRUE), +('TOP_RANK', 'Top2奖金', 'RANK_EQ', 2, 600, 2, TRUE), +('TOP_RANK', 'Top3奖金', 'RANK_EQ', 3, 400, 3, TRUE); + +-- 台区分类映射 +INSERT INTO billiards_dws.cfg_area_category +(area_name, category_l1, category_l2, display_name, sort_order) +VALUES +('A区', '台球', '大厅', 'A区', 1), +('B区', '台球', '大厅', 'B区', 2), +('C区', '台球', '大厅', 'C区', 3), +('斯诺克区', '台球', '斯诺克', '斯诺克区', 4), +('VIP包厢', '台球', '包厢', 'VIP包厢', 5), +('TV台', '台球', '大厅', 'TV台', 6), +('麻将房', '麻将', '麻将房', '麻将房', 10), +('M7', '麻将', '麻将房', 'M7', 11), +('M8', '麻将', '麻将房', 'M8', 12), +('666', '麻将', '麻将房', '666', 13), +('发财', '麻将', '麻将房', '发财', 14), +('K包', 'K包', 'KTV', 'K包', 20), +('k包活动区', 'K包', 'KTV', 'K包活动区', 21), +('幸会158', 'K包', 'KTV', '幸会158', 22), +('补时长', '其他', '补时长', '补时长', 99); +``` + +--- + +## 七、待确认事项 + +### 7.1 需抽样分析 + +1. **优惠口径分析**：抽取100个订单样本，分析 `adjust_amount` / `member_discount_amount` / `coupon_amount` 等字段的关系，明确"大客户优惠"和"其他优惠"的区分规则。 + +### 7.2 已确认事项 + +| 事项 | 结论 | +|------|------| +| 课程类型判断 | 使用 `skill_id` 而非 `skill_name`：基础课=2790683529513797，附加课=2790683529513798，包厢课=3039912271463941 | +| 助教等级枚举 | 8=助教管理，10=初级，20=中级，30=高级，40=星级 | +| 赠送卡定义 | 台费卡+酒水卡+活动抵用券 | +| 储值卡定义 | card_type_id=2793249295533893 的"储值卡" | +| 支出数据 | 数据库预留结构，后期通过Excel手动导入 | +| 平台回款 | 数据库预留结构，后期通过Excel手动导入 | +| 充值提成 | 数据库预留结构，后期通过Excel手动导入 | +| 历史等级取数 | 使用 as-of join 按业务时间获取当时的助教等级 | + +--- + +**文档版本历史** + +| 版本 | 日期 | 变更说明 | +|------|------|----------| +| 1.0 | 2026-02-01 | 初始版本 | +| 2.0 | 2026-02-01 | 根据反馈更新：补全DDL、明确数据口径、增加数据来源矩阵、更新skill_id枚举、增加手动导入表结构 | diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_assistant_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_assistant_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_assistant_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_assistant_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_groupbuy_package_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_groupbuy_package_ex.md similarity index 91% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_groupbuy_package_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_groupbuy_package_ex.md index e52a97c..8c95ff2 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_groupbuy_package_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_groupbuy_package_ex.md @@ -37,10 +37,11 @@ | 18 | effective_status | INTEGER | YES | | 生效状态。**枚举值**: 1(24)=有效, 3(10)=失效 **[待确认]** | | 19 | max_selectable_categories | INTEGER | YES | | 最大可选分类数（当前数据全为 0） | | 20 | creator_name | VARCHAR(100) | YES | | 创建人。**样本值**: "店长:郑丽珊", "管理员：郑丽珊" | -| 21 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 22 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 23 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 24 | scd2_version | INTEGER | YES | | 版本号 | +| 21 | tenant_coupon_sale_order_item_id | BIGINT | YES | | 租户券销售订单项 ID | +| 22 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 23 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 24 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 25 | scd2_version | INTEGER | YES | | 版本号 | ## 样本数据 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_member_card_account_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_member_card_account_ex.md similarity index 90% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_member_card_account_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_member_card_account_ex.md index 2fc48ed..420ea99 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_member_card_account_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_member_card_account_ex.md @@ -69,10 +69,15 @@ | 50 | goodscategoryid | TEXT | YES | | 可用商品分类 ID 列表（当前数据全为空） | | 51 | pdassisnatlevel | TEXT | YES | | 陪打助教等级限制。**当前值**: "{}" | | 52 | cxassisnatlevel | TEXT | YES | | 促销助教等级限制。**当前值**: "{}" | -| 53 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 54 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 55 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 56 | scd2_version | INTEGER | YES | | 版本号 | +| 53 | able_share_member_discount | BOOLEAN | YES | | 是否可共享会员折扣 | +| 54 | electricity_deduct_radio | NUMERIC(18,4) | YES | | 电费扣减比例 | +| 55 | electricity_discount | NUMERIC(18,4) | YES | | 电费折扣 | +| 56 | electricity_card_deduct | BOOLEAN | YES | | 电费卡扣 | +| 57 | recharge_freeze_balance | NUMERIC(18,2) | YES | | 充值冻结余额 | +| 58 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 59 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 60 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 61 | scd2_version | INTEGER | YES | | 版本号 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_member_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_member_ex.md similarity index 81% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_member_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_member_ex.md index 915c4fe..4b1fed9 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_member_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_member_ex.md @@ -24,10 +24,13 @@ | 5 | growth_value | NUMERIC(18,2) | YES | | 成长值 | | 6 | user_status | INTEGER | YES | | 用户状态。**枚举值**: 1(556)=正常 | | 7 | status | INTEGER | YES | | 账户状态。**枚举值**: 1(490)=正常, 3(66)=**[含义待确认]** | -| 8 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 9 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 10 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 11 | scd2_version | INTEGER | YES | | 版本号 | +| 8 | person_tenant_org_id | BIGINT | YES | | 人员租户组织 ID | +| 9 | person_tenant_org_name | TEXT | YES | | 人员租户组织名称 | +| 10 | register_source | TEXT | YES | | 注册来源 | +| 11 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 12 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 13 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 14 | scd2_version | INTEGER | YES | | 版本号 | ## 样本数据 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_site_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_site_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_site_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_site_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_store_goods_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_store_goods_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_store_goods_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_store_goods_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_table_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_table_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_table_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_table_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dim_tenant_goods_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_tenant_goods_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dim_tenant_goods_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dim_tenant_goods_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_assistant_service_log_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_assistant_service_log_ex.md similarity index 98% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_assistant_service_log_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_assistant_service_log_ex.md index bab0466..af486a8 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_assistant_service_log_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_assistant_service_log_ex.md @@ -47,6 +47,7 @@ | 28 | get_grade_times | INTEGER | YES | | 评分次数（当前数据全为 0） | | 29 | grade_status | INTEGER | YES | | 评分状态。**枚举值**: 0(216)=未评分, 1(4787)=已评分 **[待确认]** | | 30 | composite_grade_time | TIMESTAMPTZ | YES | | 评分时间 | +| 31 | assistant_team_name | TEXT | YES | | 助教团队名称 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_assistant_trash_event_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_assistant_trash_event_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_assistant_trash_event_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_assistant_trash_event_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_groupbuy_redemption_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_groupbuy_redemption_ex.md similarity index 85% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_groupbuy_redemption_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_groupbuy_redemption_ex.md index 147fe5a..620089e 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_groupbuy_redemption_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_groupbuy_redemption_ex.md @@ -38,6 +38,13 @@ | 19 | salesman_role_id | BIGINT | YES | | 销售员角色 ID（当前数据全为 0） | | 20 | salesman_org_id | BIGINT | YES | | 销售员组织 ID（当前数据全为 0） | | 21 | ledger_group_name | VARCHAR(128) | YES | | 账本分组名称（当前数据全为 NULL） | +| 22 | table_share_money | NUMERIC(18,2) | YES | | 台费分摊金额 | +| 23 | table_service_share_money | NUMERIC(18,2) | YES | | 台费服务分摊金额 | +| 24 | goods_share_money | NUMERIC(18,2) | YES | | 商品分摊金额 | +| 25 | good_service_share_money | NUMERIC(18,2) | YES | | 商品服务分摊金额 | +| 26 | assistant_share_money | NUMERIC(18,2) | YES | | 助教分摊金额 | +| 27 | assistant_service_share_money | NUMERIC(18,2) | YES | | 助教服务分摊金额 | +| 28 | recharge_share_money | NUMERIC(18,2) | YES | | 充值分摊金额 | ## 台区核销分布 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_member_balance_change_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_member_balance_change_ex.md similarity index 97% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_member_balance_change_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_member_balance_change_ex.md index c0274be..fcc1376 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_member_balance_change_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_member_balance_change_ex.md @@ -23,6 +23,7 @@ | 4 | refund_amount | NUMERIC(18,2) | YES | | 退款金额 | | 5 | operator_id | BIGINT | YES | | 操作员 ID | | 6 | operator_name | VARCHAR(64) | YES | | 操作员名称。**枚举值**: "收银员:郑丽珊"(4101), "店长:郑丽珊"(223), "管理员：郑丽珊"(153), "店长:蒋雨轩"(124), "店长:谢晓洪"(115), "店长:黄月柳"(29) | +| 7 | principal_data | TEXT | YES | | 本金变动数据 | ## 操作员分布 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_platform_coupon_redemption_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_platform_coupon_redemption_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_platform_coupon_redemption_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_platform_coupon_redemption_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_recharge_order_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_recharge_order_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_recharge_order_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_recharge_order_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_refund_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_refund_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_refund_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_refund_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_settlement_head_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_settlement_head_ex.md similarity index 98% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_settlement_head_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_settlement_head_ex.md index 3c9b1db..5f89933 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_settlement_head_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_settlement_head_ex.md @@ -47,6 +47,7 @@ | 28 | order_remark | VARCHAR(255) | YES | | 订单备注。**样本值**: "五折"(42), "轩哥"(24), "陈德韩"(7), "免台费"(3) | | 29 | operator_id | BIGINT | YES | | 操作员 ID | | 30 | salesman_user_id | BIGINT | YES | | 销售员用户 ID（当前数据全为 0） | +| 31 | settle_list | JSONB | YES | | 结算明细列表（JSON数组） | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_store_goods_sale_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_store_goods_sale_ex.md similarity index 100% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_store_goods_sale_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_store_goods_sale_ex.md diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_table_fee_adjust_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_table_fee_adjust_ex.md similarity index 86% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_table_fee_adjust_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_table_fee_adjust_ex.md index 987920e..2cb4455 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_table_fee_adjust_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_table_fee_adjust_ex.md @@ -25,6 +25,11 @@ | 6 | operator_name | VARCHAR(64) | YES | | 操作员名称。**枚举值**: "收银员:郑丽珊"(2849) | | 7 | applicant_id | BIGINT | YES | | 申请人 ID | | 8 | operator_id | BIGINT | YES | | 操作员 ID | +| 9 | area_type_id | BIGINT | YES | | 区域类型 ID | +| 10 | site_table_area_id | BIGINT | YES | | 门店台区 ID | +| 11 | site_table_area_name | TEXT | YES | | 门店台区名称 | +| 12 | site_name | TEXT | YES | | 门店名称 | +| 13 | tenant_name | TEXT | YES | | 租户名称 | ## 样本数据 diff --git a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_table_fee_log_ex.md b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_table_fee_log_ex.md similarity index 97% rename from etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_table_fee_log_ex.md rename to etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_table_fee_log_ex.md index 58c0619..0bf893d 100644 --- a/etl_billiards/docs/bd_manual/Ex/BD_manual_dwd_table_fee_log_ex.md +++ b/etl_billiards/docs/bd_manual/DWD/Ex/BD_manual_dwd_table_fee_log_ex.md @@ -29,6 +29,7 @@ | 10 | operator_id | BIGINT | YES | | 操作员 ID。**枚举值**: 3个不同ID | | 11 | salesman_user_id | BIGINT | YES | | 销售员用户 ID（当前数据全为 0） | | 12 | salesman_org_id | BIGINT | YES | | 销售员组织 ID（当前数据全为 0） | +| 13 | order_consumption_type | INTEGER | YES | | 订单消费类型 | ## 样本数据 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_billiards_dwd.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_billiards_dwd.md similarity index 100% rename from etl_billiards/docs/bd_manual/main/BD_manual_billiards_dwd.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_billiards_dwd.md diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_assistant.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_assistant.md similarity index 100% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_assistant.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_assistant.md diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_goods_category.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_goods_category.md similarity index 89% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_goods_category.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_goods_category.md index 55f7c45..ff5e686 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_goods_category.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_goods_category.md @@ -19,10 +19,10 @@ |------|--------|------|------|------|------| | 1 | category_id | BIGINT | NO | PK | 分类唯一标识 | | 2 | tenant_id | BIGINT | YES | | 租户 ID（当前值: 2790683160709957） | -| 3 | category_name | VARCHAR | YES | | 分类名称。**样本值**: "槟榔", "皮头" 等 | -| 4 | alias_name | VARCHAR | YES | | 分类别名（当前数据大部分为空） | +| 3 | category_name | VARCHAR(50) | YES | | 分类名称。**样本值**: "槟榔", "皮头" 等 | +| 4 | alias_name | VARCHAR(50) | YES | | 分类别名（当前数据大部分为空） | | 5 | parent_category_id | BIGINT | YES | | 父级分类 ID（0=一级分类）→ 自关联 | -| 6 | business_name | VARCHAR | YES | | 业务大类名称。**样本值**: "酒水", "器材" 等 | +| 6 | business_name | VARCHAR(50) | YES | | 业务大类名称。**样本值**: "酒水", "器材" 等 | | 7 | tenant_goods_business_id | BIGINT | YES | | 业务大类 ID | | 8 | category_level | INTEGER | YES | | 分类层级。**枚举值**: 1=一级大类, 2=二级子类 | | 9 | is_leaf | INTEGER | YES | | 是否叶子节点。**枚举值**: 0=非叶子, 1=叶子 | diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_groupbuy_package.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_groupbuy_package.md similarity index 72% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_groupbuy_package.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_groupbuy_package.md index 5508425..bb72c87 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_groupbuy_package.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_groupbuy_package.md @@ -20,23 +20,25 @@ | 1 | groupbuy_package_id | BIGINT | NO | PK | 团购套餐 ID | | 2 | tenant_id | BIGINT | YES | | 租户 ID（当前值: 2790683160709957） | | 3 | site_id | BIGINT | YES | | 门店 ID → dim_site（当前值: 2790685415443269） | -| 4 | package_name | VARCHAR | YES | | 套餐名称。**样本值**: "中八、斯诺克包厢两小时", "斯诺克两小时"等 | +| 4 | package_name | VARCHAR(200) | YES | | 套餐名称。**样本值**: "中八、斯诺克包厢两小时", "斯诺克两小时"等 | | 5 | package_template_id | BIGINT | YES | | 套餐模板 ID | | 6 | selling_price | NUMERIC(10,2) | YES | | 售卖价格（每笔订单不同，从核销记录中dwd_groupbuy_redemption获取） | | 7 | coupon_face_value | NUMERIC(10,2) | YES | | 券面值（每笔订单不同，从核销记录中dwd_groupbuy_redemption获取） | | 8 | duration_seconds | INTEGER | YES | | 套餐时长（秒）。**样本值**: 3600=1小时, 7200=2小时, 14400=4小时 等 | | 9 | start_time | TIMESTAMPTZ | YES | | 套餐生效开始时间 | | 10 | end_time | TIMESTAMPTZ | YES | | 套餐生效结束时间 | -| 11 | table_area_name | VARCHAR | YES | | 适用台区名称。**枚举值**: "A区", "VIP包厢", "斯诺克区", "B区", "麻将房", "888" | +| 11 | table_area_name | VARCHAR(100) | YES | | 适用台区名称。**枚举值**: "A区", "VIP包厢", "斯诺克区", "B区", "麻将房", "888" | | 12 | is_enabled | INTEGER | YES | | 启用状态。**枚举值**: 1=启用, 2=停用 | | 13 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | | 14 | create_time | TIMESTAMPTZ | YES | | 创建时间 | -| 15 | tenant_table_area_id_list | VARCHAR | YES | | 租户级台区 ID 列表 | -| 16 | card_type_ids | VARCHAR | YES | | 允许使用的卡类型 ID 列表（当前数据为 "0"） | -| 17 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 18 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 19 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 20 | scd2_version | INTEGER | YES | | 版本号 | +| 15 | tenant_table_area_id_list | VARCHAR(512) | YES | | 租户级台区 ID 列表 | +| 16 | card_type_ids | VARCHAR(255) | YES | | 允许使用的卡类型 ID 列表（当前数据为 "0"） | +| 17 | sort | INTEGER | YES | | 排序 | +| 18 | is_first_limit | BOOLEAN | YES | | 是否首单限制 | +| 19 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 20 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 21 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 22 | scd2_version | INTEGER | YES | | 版本号 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_member.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_member.md similarity index 83% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_member.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_member.md index d47d27d..409cfb4 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_member.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_member.md @@ -27,10 +27,12 @@ | 8 | member_card_grade_name | TEXT | YES | | 卡等级名称。**枚举值**: "储值卡", "台费卡", "年卡", "活动抵用券", "月卡" | | 9 | create_time | TIMESTAMPTZ | YES | | 创建时间 | | 10 | update_time | TIMESTAMPTZ | YES | | 更新时间 | -| 11 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 12 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 13 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 14 | scd2_version | INTEGER | YES | | 版本号 | +| 11 | pay_money_sum | NUMERIC(18,2) | YES | | 累计支付金额 | +| 12 | recharge_money_sum | NUMERIC(18,2) | YES | | 累计充值金额 | +| 13 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 14 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 15 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 16 | scd2_version | INTEGER | YES | | 版本号 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_member_card_account.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_member_card_account.md similarity index 89% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_member_card_account.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_member_card_account.md index 74ff6ab..03fa275 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_member_card_account.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_member_card_account.md @@ -34,10 +34,12 @@ | 15 | last_consume_time | TIMESTAMPTZ | YES | | 最近消费时间 | | 16 | status | INTEGER | YES | | 卡状态。**枚举值**: 1=正常, 4=过期 | | 17 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | -| 18 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 19 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 20 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 21 | scd2_version | INTEGER | YES | | 版本号 | +| 18 | principal_balance | NUMERIC(18,2) | YES | | 本金余额 | +| 19 | member_grade | INTEGER | YES | | 会员等级 | +| 20 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 21 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 22 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 23 | scd2_version | INTEGER | YES | | 版本号 | ## 卡种分布 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_site.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_site.md similarity index 100% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_site.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_site.md diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_store_goods.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_store_goods.md similarity index 89% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_store_goods.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_store_goods.md index 98a5de1..b06b8d5 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_store_goods.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_store_goods.md @@ -37,10 +37,12 @@ | 18 | enable_status | INTEGER | YES | | 启用状态。**枚举值**: 1=启用 | | 19 | send_state | INTEGER | YES | | 配送状态。暂无作用 | | 20 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | -| 21 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 22 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 23 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 24 | scd2_version | INTEGER | YES | | 版本号 | +| 21 | commodity_code | TEXT | YES | | 商品编码 | +| 22 | not_sale | INTEGER | YES | | 是否停售 | +| 23 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 24 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 25 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 26 | scd2_version | INTEGER | YES | | 版本号 | ## 样本数据 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_table.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_table.md similarity index 89% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_table.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_table.md index 9b83c84..66dbf7b 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_table.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_table.md @@ -24,10 +24,11 @@ | 5 | site_table_area_name | TEXT | YES | | 台区名称。**样本值**: "A区", "B区", "补时长", "C区", "麻将房", "K包", "VIP包厢", "斯诺克区", "666", "k包活动区", "M7" 等 | | 6 | tenant_table_area_id | BIGINT | YES | | 租户级台区 ID | | 7 | table_price | NUMERIC(18,2) | YES | | 台桌单价（当前数据全为 0.00） | -| 8 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 9 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 10 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 11 | scd2_version | INTEGER | YES | | 版本号 | +| 8 | order_id | BIGINT | YES | | 订单 ID | +| 9 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 10 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 11 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 12 | scd2_version | INTEGER | YES | | 版本号 | ## 台区分布 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dim_tenant_goods.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_tenant_goods.md similarity index 70% rename from etl_billiards/docs/bd_manual/main/BD_manual_dim_tenant_goods.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_tenant_goods.md index 7dc5bb5..61323ae 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dim_tenant_goods.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dim_tenant_goods.md @@ -20,21 +20,22 @@ | 1 | tenant_goods_id | BIGINT | NO | PK | 租户商品 ID（SKU） | | 2 | tenant_id | BIGINT | YES | | 租户 ID | | 3 | supplier_id | BIGINT | YES | | 供应商 ID（当前数据全为 0） | -| 4 | category_name | VARCHAR | YES | | 分类名称（二级分类）。**样本值**: "零食", "饮料", "香烟"等 | +| 4 | category_name | VARCHAR(64) | YES | | 分类名称（二级分类）。**样本值**: "零食", "饮料", "香烟"等 | | 5 | goods_category_id | BIGINT | YES | | 一级分类 ID | | 6 | goods_second_category_id | BIGINT | YES | | 二级分类 ID | -| 7 | goods_name | VARCHAR | YES | | 商品名称。**样本值**: "海之言", "西梅多多饮品", "美汁源果粒橙", "三诺橙汁"等 | -| 8 | goods_number | VARCHAR | YES | | 商品编号（序号） | -| 9 | unit | VARCHAR | YES | | 商品单位。**枚举值**: "包", "瓶", "个", "份"等 | +| 7 | goods_name | VARCHAR(128) | YES | | 商品名称。**样本值**: "海之言", "西梅多多饮品", "美汁源果粒橙", "三诺橙汁"等 | +| 8 | goods_number | VARCHAR(64) | YES | | 商品编号（序号） | +| 9 | unit | VARCHAR(16) | YES | | 商品单位。**枚举值**: "包", "瓶", "个", "份"等 | | 10 | market_price | NUMERIC(18,2) | YES | | 市场价/吊牌价（元） | | 11 | goods_state | INTEGER | YES | | 商品状态。**枚举值**: 1=上架, 2=下架 | | 12 | create_time | TIMESTAMPTZ | YES | | 创建时间 | | 13 | update_time | TIMESTAMPTZ | YES | | 更新时间 | | 14 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | -| 15 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | -| 16 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | -| 17 | scd2_is_current | INTEGER | YES | | 当前版本标记 | -| 18 | scd2_version | INTEGER | YES | | 版本号 | +| 15 | not_sale | INTEGER | YES | | 是否停售 | +| 16 | scd2_start_time | TIMESTAMPTZ | NO | PK | SCD2 版本生效时间 | +| 17 | scd2_end_time | TIMESTAMPTZ | YES | | SCD2 版本失效时间 | +| 18 | scd2_is_current | INTEGER | YES | | 当前版本标记 | +| 19 | scd2_version | INTEGER | YES | | 版本号 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_assistant_service_log.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_assistant_service_log.md similarity index 87% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_assistant_service_log.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_assistant_service_log.md index 736e991..6d915e4 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_assistant_service_log.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_assistant_service_log.md @@ -28,16 +28,16 @@ | 9 | site_table_id | BIGINT | YES | | 台桌 ID → dim_table（0=非台桌服务） | | 10 | tenant_member_id | BIGINT | YES | | 会员 ID → dim_member（0=散客） | | 11 | system_member_id | BIGINT | YES | | 系统会员 ID（0=散客） | -| 12 | assistant_no | VARCHAR | YES | | 助教工号。**样本值**: "2", "9"等 | -| 13 | nickname | VARCHAR | YES | | 助教昵称。**样本值**: "佳怡", "婉婉", "七七"等 | +| 12 | assistant_no | VARCHAR(64) | YES | | 助教工号。**样本值**: "2", "9"等 | +| 13 | nickname | VARCHAR(64) | YES | | 助教昵称。**样本值**: "佳怡", "婉婉", "七七"等 | | 14 | site_assistant_id | BIGINT | YES | | 助教 ID → dim_assistant | | 15 | user_id | BIGINT | YES | | 助教用户 ID | | 16 | assistant_team_id | BIGINT | YES | | 助教团队 ID。**枚举值**: 2792011585884037=1组, 2959085810992645=2组 | | 17 | person_org_id | BIGINT | YES | | 人事组织 ID | | 18 | assistant_level | INTEGER | YES | | 助教等级。**枚举值**: 8=助教管理, 10=初级, 20=中级, 30=高级, 40=星级 | -| 19 | level_name | VARCHAR | YES | | 等级名称。**枚举值**: "助教管理", "初级", "中级", "高级", "星级" | +| 19 | level_name | VARCHAR(64) | YES | | 等级名称。**枚举值**: "助教管理", "初级", "中级", "高级", "星级" | | 20 | skill_id | BIGINT | YES | | 技能 ID **枚举值**: 2790683529513797 = 基础课 , 2790683529513798 = 附加课/激励课, 3039912271463941 = 包厢课 | -| 21 | skill_name | VARCHAR | YES | | 技能名称。 **枚举值**: "基础课","附加课","包厢课"| +| 21 | skill_name | VARCHAR(64) | YES | | 技能名称。 **枚举值**: "基础课","附加课","包厢课"| | 22 | ledger_unit_price | NUMERIC(10,2) | YES | | 单价（元/小时），**样本值**: 98.00/108.00/190.00 等 | | 23 | ledger_amount | NUMERIC(10,2) | YES | | 计费金额 | | 24 | projected_income | NUMERIC(10,2) | YES | | 预估收入 | @@ -49,6 +49,7 @@ | 30 | start_use_time | TIMESTAMPTZ | YES | | 服务开始时间 | | 31 | last_use_time | TIMESTAMPTZ | YES | | 服务结束时间 | | 32 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | +| 33 | real_service_money | NUMERIC(18,2) | YES | | 实际服务费金额 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_assistant_trash_event.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_assistant_trash_event.md similarity index 82% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_assistant_trash_event.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_assistant_trash_event.md index 8f83e24..f9fdd8e 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_assistant_trash_event.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_assistant_trash_event.md @@ -21,12 +21,13 @@ | 2 | site_id | BIGINT | YES | | 门店 ID | | 3 | table_id | BIGINT | YES | | 台桌 ID → dim_table | | 4 | table_area_id | BIGINT | YES | | 台区 ID | -| 5 | assistant_no | VARCHAR | YES | | 助教工号/昵称。**样本值**: "七七", "乔西", "球球"等 | -| 6 | assistant_name | VARCHAR | YES | | 助教名称，与 assistant_no 相同 | +| 5 | assistant_no | VARCHAR(32) | YES | | 助教工号/昵称。**样本值**: "七七", "乔西", "球球"等 | +| 6 | assistant_name | VARCHAR(64) | YES | | 助教名称，与 assistant_no 相同 | | 7 | charge_minutes_raw | INTEGER | YES | | 原计费时长（秒）。**样本值**: 0, 3600=1h, 10800=3h 等 | | 8 | abolish_amount | NUMERIC(18,2) | YES | | 作废金额（元）。**样本值**: 0.00, 190.00, 570.00 等 | -| 9 | trash_reason | VARCHAR | YES | | 作废原因（当前数据全为 NULL） | +| 9 | trash_reason | VARCHAR(255) | YES | | 作废原因（当前数据全为 NULL） | | 10 | create_time | TIMESTAMPTZ | YES | | 创建时间 | +| 11 | tenant_id | BIGINT | YES | | 租户 ID | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_groupbuy_redemption.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_groupbuy_redemption.md similarity index 89% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_groupbuy_redemption.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_groupbuy_redemption.md index 44a442f..ee0ea4f 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_groupbuy_redemption.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_groupbuy_redemption.md @@ -35,11 +35,13 @@ | 16 | ledger_amount | NUMERIC(18,2) | YES | | 账本金额（元）。**样本值**: 48.00, 96.00, 68.00 等 | | 17 | coupon_money | NUMERIC(18,2) | YES | | 券面额（元）。**样本值**: 48.00, 116.00, 96.00, 68.00 等 | | 18 | promotion_seconds | INTEGER | YES | | 促销时长（秒）。**样本值**: 3600=1h, 7200=2h, 14400=4h 等 | -| 19 | coupon_code | VARCHAR | YES | | 券码 | +| 19 | coupon_code | VARCHAR(64) | YES | | 券码 | | 20 | is_single_order | INTEGER | YES | | 是否独立订单。**枚举值**: 0=否, 1=是 | | 21 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | -| 22 | ledger_name | VARCHAR | YES | | 套餐名称。**样本值**: "全天A区中八一小时", "中八A区新人特惠一小时" 等 | +| 22 | ledger_name | VARCHAR(128) | YES | | 套餐名称。**样本值**: "全天A区中八一小时", "中八A区新人特惠一小时" 等 | | 23 | create_time | TIMESTAMPTZ | YES | | 创建时间 | +| 24 | member_discount_money | NUMERIC(18,2) | YES | | 会员折扣金额 | +| 25 | coupon_sale_id | BIGINT | YES | | 优惠券销售 ID | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_member_balance_change.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_member_balance_change.md similarity index 82% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_member_balance_change.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_member_balance_change.md index 3e0cf77..d475315 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_member_balance_change.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_member_balance_change.md @@ -25,9 +25,9 @@ | 6 | system_member_id | BIGINT | YES | | 系统会员 ID | | 7 | tenant_member_card_id | BIGINT | YES | | 会员卡 ID → dim_member_card_account | | 8 | card_type_id | BIGINT | YES | | 卡类型 ID | -| 9 | card_type_name | VARCHAR | YES | | 卡类型名称。**枚举值**: "储值卡", "活动抵用券", "台费卡", "酒水卡", "年卡", "月卡" | -| 10 | member_name | VARCHAR | YES | | 会员名称快照 | -| 11 | member_mobile | VARCHAR | YES | | 会员手机号快照 | +| 9 | card_type_name | VARCHAR(32) | YES | | 卡类型名称。**枚举值**: "储值卡", "活动抵用券", "台费卡", "酒水卡", "年卡", "月卡" | +| 10 | member_name | VARCHAR(64) | YES | | 会员名称快照 | +| 11 | member_mobile | VARCHAR(20) | YES | | 会员手机号快照 | | 12 | balance_before | NUMERIC(18,2) | YES | | 变动前余额 | | 13 | change_amount | NUMERIC(18,2) | YES | | 变动金额（正=充值/赠送，负=消费） | | 14 | balance_after | NUMERIC(18,2) | YES | | 变动后余额 | @@ -35,7 +35,10 @@ | 16 | payment_method | INTEGER | YES | | 支付方式，暂未启用。 | | 17 | change_time | TIMESTAMPTZ | YES | | 变动时间 | | 18 | is_delete | INTEGER | YES | | 删除标记 | -| 19 | remark | VARCHAR | YES | | 备注。**样本值**: "注销会员", "充值退款" 等 | +| 19 | remark | VARCHAR(255) | YES | | 备注。**样本值**: "注销会员", "充值退款" 等 | +| 20 | principal_before | NUMERIC(18,2) | YES | | 变动前本金 | +| 21 | principal_after | NUMERIC(18,2) | YES | | 变动后本金 | +| 22 | principal_change_amount | NUMERIC(18,2) | YES | | 本金变动金额（正=增加，负=减少） | ## 卡类型余额变动分布 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_payment.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_payment.md similarity index 97% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_payment.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_payment.md index ef7d31d..cbb2a31 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_payment.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_payment.md @@ -28,6 +28,7 @@ | 9 | create_time | TIMESTAMPTZ | YES | | 创建时间 | | 10 | pay_time | TIMESTAMPTZ | YES | | 支付时间 | | 11 | pay_date | DATE | YES | | 支付日期 | +| 12 | tenant_id | BIGINT | YES | | 租户 ID | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_platform_coupon_redemption.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_platform_coupon_redemption.md similarity index 86% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_platform_coupon_redemption.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_platform_coupon_redemption.md index 4bd4e32..d336187 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_platform_coupon_redemption.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_platform_coupon_redemption.md @@ -20,9 +20,9 @@ | 1 | platform_coupon_redemption_id | BIGINT | NO | PK | 核销 ID | | 2 | tenant_id | BIGINT | YES | | 租户 ID | | 3 | site_id | BIGINT | YES | | 门店 ID | -| 4 | coupon_code | VARCHAR | YES | | 券码 | +| 4 | coupon_code | VARCHAR(64) | YES | | 券码 | | 5 | coupon_channel | INTEGER | YES | | 券渠道。**枚举值**: 1=美团, 2=抖音 | -| 6 | coupon_name | VARCHAR | YES | | 券名称。**样本值**: "【全天可用】中八桌球一小时（A区）", "【全天可用】中八桌球两小时（A区）" 等 | +| 6 | coupon_name | VARCHAR(200) | YES | | 券名称。**样本值**: "【全天可用】中八桌球一小时（A区）", "【全天可用】中八桌球两小时（A区）" 等 | | 7 | sale_price | NUMERIC(10,2) | YES | | 售卖价（元）。**样本值**: 29.90, 69.90, 59.90, 39.90, 19.90 等 | | 8 | coupon_money | NUMERIC(10,2) | YES | | 券面额（元）。**样本值**: 48.00, 96.00, 116.00, 68.00 等 | | 9 | coupon_free_time | INTEGER | YES | | 券赠送时长（当前数据全为 0） | @@ -31,8 +31,8 @@ | 12 | group_package_id | BIGINT | YES | | 团购套餐 ID（当前数据全为 0） | | 13 | site_order_id | BIGINT | YES | | 门店订单 ID | | 14 | table_id | BIGINT | YES | | 台桌 ID → dim_table | -| 15 | certificate_id | VARCHAR | YES | | 凭证 ID | -| 16 | verify_id | VARCHAR | YES | | 核验 ID（仅抖音券有值） | +| 15 | certificate_id | VARCHAR(64) | YES | | 凭证 ID | +| 16 | verify_id | VARCHAR(64) | YES | | 核验 ID（仅抖音券有值） | | 17 | use_status | INTEGER | YES | | 使用状态。**枚举值**: 1=已使用, 2=已撤销 | | 18 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | | 19 | create_time | TIMESTAMPTZ | YES | | 创建时间 | diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_recharge_order.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_recharge_order.md similarity index 100% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_recharge_order.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_recharge_order.md diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_refund.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_refund.md similarity index 100% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_refund.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_refund.md diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_settlement_head.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_settlement_head.md similarity index 78% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_settlement_head.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_settlement_head.md index 351cb61..c0270cd 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_settlement_head.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_settlement_head.md @@ -20,19 +20,19 @@ | 1 | order_settle_id | BIGINT | NO | PK | 结账单 ID | | 2 | tenant_id | BIGINT | YES | | 租户 ID | | 3 | site_id | BIGINT | YES | | 门店 ID → dim_site | -| 4 | site_name | VARCHAR | YES | | 门店名称。**当前值**: "朗朗桌球" | +| 4 | site_name | VARCHAR(100) | YES | | 门店名称。**当前值**: "朗朗桌球" | | 5 | table_id | BIGINT | YES | | 台桌 ID → dim_table（0=非台桌订单，如商城订单） | -| 6 | settle_name | VARCHAR | YES | | 结账名称。**样本值**: "商城订单", "A区 A3", "A区 A4", "斯诺克区 S1" | +| 6 | settle_name | VARCHAR(100) | YES | | 结账名称。**样本值**: "商城订单", "A区 A3", "A区 A4", "斯诺克区 S1" | | 7 | order_trade_no | BIGINT | YES | | 订单号 | | 8 | create_time | TIMESTAMPTZ | YES | | 创建时间 | | 9 | pay_time | TIMESTAMPTZ | YES | | 支付时间 | | 10 | settle_type | INTEGER | YES | | 结账类型。**枚举值**: 1=台桌结账, 3=商城订单, 6=退货订单, 7=退款订单 | | 11 | revoke_order_id | BIGINT | YES | | 撤销订单 ID（当前数据全为 0） | | 12 | member_id | BIGINT | YES | | 会员 ID → dim_member（0=散客，占比约 82.8%） | -| 13 | member_name | VARCHAR | YES | | 会员名称 | -| 14 | member_phone | VARCHAR | YES | | 会员电话 | +| 13 | member_name | VARCHAR(100) | YES | | 会员名称 | +| 14 | member_phone | VARCHAR(50) | YES | | 会员电话 | | 15 | member_card_account_id | BIGINT | YES | | 会员卡账户 ID（当前数据全为 0） | -| 16 | member_card_type_name | VARCHAR | YES | | 卡类型名称（当前数据全为空） | +| 16 | member_card_type_name | VARCHAR(100) | YES | | 卡类型名称（当前数据全为空） | | 17 | is_bind_member | BOOLEAN | YES | | 是否绑定会员。**枚举值**: False=否 | | 18 | member_discount_amount | NUMERIC(18,2) | YES | | 会员折扣金额 | | 19 | consume_money | NUMERIC(18,2) | YES | | 消费总金额（元） | @@ -40,7 +40,7 @@ | 21 | goods_money | NUMERIC(18,2) | YES | | 商品金额 | | 22 | real_goods_money | NUMERIC(18,2) | YES | | 实收商品金额 | | 23 | assistant_pd_money | NUMERIC(18,2) | YES | | 助教陪打费用 | -| 24 | assistant_cx_money | NUMERIC(18,2) | YES | | 助教促销费用 | +| 24 | assistant_cx_money | NUMERIC(18,2) | YES | | 助教超休费用 | | 25 | adjust_amount | NUMERIC(18,2) | YES | | 调整金额 | | 26 | pay_amount | NUMERIC(18,2) | YES | | 实付金额 | | 27 | balance_amount | NUMERIC(18,2) | YES | | 余额支付金额 | @@ -49,6 +49,11 @@ | 30 | coupon_amount | NUMERIC(18,2) | YES | | 券抵扣金额 | | 31 | rounding_amount | NUMERIC(18,2) | YES | | 抹零金额 | | 32 | point_amount | NUMERIC(18,2) | YES | | 积分抵扣等值金额 | +| 33 | electricity_money | NUMERIC(18,2) | YES | | 电费金额 | +| 34 | real_electricity_money | NUMERIC(18,2) | YES | | 实际电费金额 | +| 35 | electricity_adjust_money | NUMERIC(18,2) | YES | | 电费调整金额 | +| 36 | pl_coupon_sale_amount | NUMERIC(18,2) | YES | | 平台券销售额 | +| 37 | mervou_sales_amount | NUMERIC(18,2) | YES | | 商户券销售额 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_store_goods_sale.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_store_goods_sale.md similarity index 88% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_store_goods_sale.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_store_goods_sale.md index b099697..01a42dc 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_store_goods_sale.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_store_goods_sale.md @@ -29,8 +29,8 @@ | 10 | tenant_goods_category_id | BIGINT | YES | | 商品分类 ID | | 11 | tenant_goods_business_id | BIGINT | YES | | 业务大类 ID | | 12 | site_table_id | BIGINT | YES | | 台桌 ID（0=商城订单，非台桌消费） | -| 13 | ledger_name | VARCHAR | YES | | 商品名称。**样本值**: "哇哈哈矿泉水", "东方树叶", "可乐" 等 | -| 14 | ledger_group_name | VARCHAR | YES | | 商品分类。**样本值**: "酒水", "零食", "香烟" 等 | +| 13 | ledger_name | VARCHAR(200) | YES | | 商品名称。**样本值**: "哇哈哈矿泉水", "东方树叶", "可乐" 等 | +| 14 | ledger_group_name | VARCHAR(100) | YES | | 商品分类。**样本值**: "酒水", "零食", "香烟" 等 | | 15 | ledger_unit_price | NUMERIC(18,2) | YES | | 单价（元） | | 16 | ledger_count | INTEGER | YES | | 购买数量。**样本值**: 1, 2, 3, 4 等 | | 17 | ledger_amount | NUMERIC(18,2) | YES | | 销售金额（元） | @@ -40,6 +40,7 @@ | 21 | ledger_status | INTEGER | YES | | 账本状态。**枚举值**: 1=已结算 | | 22 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | | 23 | create_time | TIMESTAMPTZ | YES | | 创建时间 | +| 24 | coupon_share_money | NUMERIC(18,2) | YES | | 优惠券分摊金额 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_table_fee_adjust.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_table_fee_adjust.md similarity index 84% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_table_fee_adjust.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_table_fee_adjust.md index 41d9a42..1eafd24 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_table_fee_adjust.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_table_fee_adjust.md @@ -24,11 +24,15 @@ | 5 | site_id | BIGINT | YES | | 门店 ID | | 6 | table_id | BIGINT | YES | | 台桌 ID → dim_table | | 7 | table_area_id | BIGINT | YES | | 台区 ID | -| 8 | table_area_name | VARCHAR | YES | | 台区名称（当前数据全为 NULL） | +| 8 | table_area_name | VARCHAR(64) | YES | | 台区名称（当前数据全为 NULL） | | 9 | tenant_table_area_id | BIGINT | YES | | 租户台区 ID | | 10 | ledger_amount | NUMERIC(18,2) | YES | | 调整金额（元） | | 11 | ledger_status | INTEGER | YES | | 账本状态。**枚举值**: 0=待确认, 1=已确认 | | 12 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | +| 13 | table_name | TEXT | YES | | 台桌名称 | +| 14 | table_price | NUMERIC(18,2) | YES | | 台桌价格 | +| 15 | charge_free | BOOLEAN | YES | | 是否免费 | +| 16 | adjust_time | TIMESTAMPTZ | YES | | 调整时间 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_table_fee_log.md b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_table_fee_log.md similarity index 87% rename from etl_billiards/docs/bd_manual/main/BD_manual_dwd_table_fee_log.md rename to etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_table_fee_log.md index f6c4335..5b6195a 100644 --- a/etl_billiards/docs/bd_manual/main/BD_manual_dwd_table_fee_log.md +++ b/etl_billiards/docs/bd_manual/DWD/main/BD_manual_dwd_table_fee_log.md @@ -25,10 +25,10 @@ | 6 | site_id | BIGINT | YES | | 门店 ID | | 7 | site_table_id | BIGINT | YES | | 台桌 ID → dim_table | | 8 | site_table_area_id | BIGINT | YES | | 台区 ID | -| 9 | site_table_area_name | VARCHAR | YES | | 台区名称。**枚举值**: "A区", "B区", "斯诺克区", "麻将房", "C区", "补时长", "VIP包厢" 等 | +| 9 | site_table_area_name | VARCHAR(64) | YES | | 台区名称。**枚举值**: "A区", "B区", "斯诺克区", "麻将房", "C区", "补时长", "VIP包厢" 等 | | 10 | tenant_table_area_id | BIGINT | YES | | 租户级台区 ID | | 11 | member_id | BIGINT | YES | | 会员 ID（0=散客，占比约 82.4%） | -| 12 | ledger_name | VARCHAR | YES | | 台桌名称。**样本值**: "A3", "A5", "A4", "S1", "B5", "M3" 等 | +| 12 | ledger_name | VARCHAR(64) | YES | | 台桌名称。**样本值**: "A3", "A5", "A4", "S1", "B5", "M3" 等 | | 13 | ledger_unit_price | NUMERIC(18,2) | YES | | 单价（元/小时），如 48.00/58.00/68.00 | | 14 | ledger_count | INTEGER | YES | | 计费时长（秒）。**样本值**: 3600=1h, 7200=2h, 10800=3h 等 | | 15 | ledger_amount | NUMERIC(18,2) | YES | | 计费金额（元） | @@ -44,6 +44,8 @@ | 25 | ledger_status | INTEGER | YES | | 账本状态。**枚举值**: 1=已结算 | | 26 | is_single_order | INTEGER | YES | | 是否独立订单。**枚举值**: 0=合并订单, 1=独立订单 | | 27 | is_delete | INTEGER | YES | | 删除标记。**枚举值**: 0=未删除 | +| 28 | activity_discount_amount | NUMERIC(18,2) | YES | | 活动折扣金额 | +| 29 | real_service_money | NUMERIC(18,2) | YES | | 实际服务费金额 | ## 使用说明 diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_area_category.md b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_area_category.md new file mode 100644 index 0000000..426ab6c --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_area_category.md @@ -0,0 +1,74 @@ +# cfg_area_category 台区分类映射表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | cfg_area_category | +| 主键 | category_id | +| 数据来源 | 手工维护/seed脚本（基于dim_table实际数据） | +| 说明 | 将dim_table.site_table_area_name映射到财务报表区域分类 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 | +|------|--------|------|------|------|------| +| 1 | category_id | SERIAL | NO | PK | 分类ID（自增） | +| 2 | source_area_name | VARCHAR(100) | NO | UK | 源区域名称（来自dim_table.site_table_area_name） | +| 3 | category_code | VARCHAR(20) | NO | | 分类代码。**枚举值**: BILLIARD, BILLIARD_VIP, SNOOKER, MAHJONG, KTV, SPECIAL, OTHER | +| 4 | category_name | VARCHAR(50) | NO | | 分类名称 | +| 5 | match_type | VARCHAR(10) | NO | | 匹配类型。**枚举值**: EXACT（精确）, LIKE（模糊）, DEFAULT（兜底） | +| 6 | match_priority | INTEGER | NO | | 匹配优先级（数字越小优先级越高） | +| 7 | is_active | BOOLEAN | NO | | 是否启用 | +| 8 | description | TEXT | YES | | 说明 | +| 9 | created_at | TIMESTAMPTZ | NO | | 创建时间 | +| 10 | updated_at | TIMESTAMPTZ | NO | | 更新时间 | + +## 分类映射示例 + +| 源区域名称 | 分类代码 | 分类名称 | +|------------|----------|----------| +| A区 | BILLIARD | 台球散台 | +| B区 | BILLIARD | 台球散台 | +| C区 | BILLIARD | 台球散台 | +| TV台 | BILLIARD | 台球散台 | +| VIP包厢 | BILLIARD_VIP | 台球VIP | +| 斯诺克区 | SNOOKER | 斯诺克 | +| 麻将房 | MAHJONG | 麻将棋牌 | +| M7 | MAHJONG | 麻将棋牌 | +| M8 | MAHJONG | 麻将棋牌 | +| 666 | MAHJONG | 麻将棋牌 | +| 发财 | MAHJONG | 麻将棋牌 | +| K包 | KTV | K歌娱乐 | +| k包活动区 | KTV | K歌娱乐 | +| 幸会158 | KTV | K歌娱乐 | +| 补时长 | SPECIAL | 补时长 | + +## 使用说明 + +**取值方式** + +```sql +-- 将台区名称映射到分类 +SELECT + dt.site_table_area_name, + COALESCE(ac.category_code, 'OTHER') AS category_code, + COALESCE(ac.category_name, '其他') AS category_name +FROM billiards_dwd.dim_table dt +LEFT JOIN billiards_dws.cfg_area_category ac + ON dt.site_table_area_name = ac.source_area_name + AND ac.is_active = TRUE +WHERE dt.scd2_is_current = 1; + +-- 按分类汇总收入 +SELECT + COALESCE(ac.category_name, '其他') AS category_name, + SUM(tfl.ledger_amount) AS total_amount +FROM billiards_dwd.dwd_table_fee_log tfl +LEFT JOIN billiards_dwd.dim_table dt ON dt.table_id = tfl.site_table_id +LEFT JOIN billiards_dws.cfg_area_category ac ON dt.site_table_area_name = ac.source_area_name +GROUP BY COALESCE(ac.category_name, '其他'); +``` diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_assistant_level_price.md b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_assistant_level_price.md new file mode 100644 index 0000000..926336a --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_assistant_level_price.md @@ -0,0 +1,56 @@ +# cfg_assistant_level_price 助教等级定价表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | cfg_assistant_level_price | +| 主键 | price_id | +| 数据来源 | 手工维护/seed脚本 | +| 说明 | 助教等级对应的基础课和附加课单价配置 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 | +|------|--------|------|------|------|------| +| 1 | price_id | SERIAL | NO | PK | 定价ID（自增） | +| 2 | level_code | INTEGER | NO | | 等级代码。**枚举值**: 8=助教管理, 10=初级, 20=中级, 30=高级, 40=星级 | +| 3 | level_name | VARCHAR(20) | NO | | 等级名称 | +| 4 | base_course_price | NUMERIC(10,2) | NO | | 基础课单价（元/小时） | +| 5 | bonus_course_price | NUMERIC(10,2) | NO | | 附加课单价（元/小时），固定50元 | +| 6 | effective_from | DATE | NO | | 生效起始日期（含） | +| 7 | effective_to | DATE | NO | | 生效截止日期（含） | +| 8 | description | TEXT | YES | | 说明 | +| 9 | created_at | TIMESTAMPTZ | NO | | 创建时间 | +| 10 | updated_at | TIMESTAMPTZ | NO | | 更新时间 | + +## 定价配置示例 + +| 等级代码 | 等级名称 | 基础课单价 | 附加课单价 | +|----------|----------|------------|------------| +| 8 | 助教管理 | 98元/小时 | 50元/小时 | +| 10 | 初级 | 98元/小时 | 50元/小时 | +| 20 | 中级 | 108元/小时 | 50元/小时 | +| 30 | 高级 | 118元/小时 | 50元/小时 | +| 40 | 星级 | 138元/小时 | 50元/小时 | + +## 使用说明 + +**取值方式** + +SCD2口径：助教等级来自dim_assistant，取数时需按有效期as-of join + +```sql +-- 获取助教在指定日期的等级定价 +SELECT p.* +FROM billiards_dws.cfg_assistant_level_price p +JOIN billiards_dwd.dim_assistant a ON p.level_code = a.level +WHERE a.assistant_id = 123 + AND a.scd2_start_time <= '2026-01-15' + AND (a.scd2_end_time IS NULL OR a.scd2_end_time > '2026-01-15') + AND p.effective_from <= '2026-01-15' + AND p.effective_to >= '2026-01-15'; +``` diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_bonus_rules.md b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_bonus_rules.md new file mode 100644 index 0000000..0d46302 --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_bonus_rules.md @@ -0,0 +1,73 @@ +# cfg_bonus_rules 奖金规则配置表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | cfg_bonus_rules | +| 主键 | rule_id | +| 数据来源 | 手工维护/seed脚本 | +| 说明 | 冲刺奖金（按小时阈值）和Top3排名奖金配置 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 | +|------|--------|------|------|------|------| +| 1 | rule_id | SERIAL | NO | PK | 规则ID（自增） | +| 2 | rule_type | VARCHAR(20) | NO | | 规则类型。**枚举值**: SPRINT（冲刺奖金）, TOP_RANK（Top排名奖金） | +| 3 | rule_code | VARCHAR(30) | NO | | 规则代码。**枚举值**: SPRINT_190, SPRINT_220, TOP_1, TOP_2, TOP_3 | +| 4 | rule_name | VARCHAR(50) | NO | | 规则名称 | +| 5 | threshold_hours | NUMERIC(10,2) | YES | | 小时数阈值（冲刺奖金用） | +| 6 | rank_position | INTEGER | YES | | 排名位置（Top奖金用） | +| 7 | bonus_amount | NUMERIC(12,2) | NO | | 奖金金额（元） | +| 8 | is_cumulative | BOOLEAN | NO | | 是否可累计（冲刺奖金为FALSE，取最高档） | +| 9 | priority | INTEGER | NO | | 优先级（数字越大优先级越高） | +| 10 | effective_from | DATE | NO | | 生效起始日期（含） | +| 11 | effective_to | DATE | NO | | 生效截止日期（含） | +| 12 | description | TEXT | YES | | 说明 | +| 13 | created_at | TIMESTAMPTZ | NO | | 创建时间 | +| 14 | updated_at | TIMESTAMPTZ | NO | | 更新时间 | + +## 奖金规则示例 + +### 冲刺奖金（不累计，取最高档） +| 规则代码 | 小时阈值 | 奖金金额 | 优先级 | +|----------|----------|----------|--------| +| SPRINT_190 | 190小时 | 300元 | 1 | +| SPRINT_220 | 220小时 | 800元 | 2 | + +### Top3排名奖金（独立发放） +| 规则代码 | 排名 | 奖金金额 | +|----------|------|----------| +| TOP_1 | 第1名 | 1000元 | +| TOP_2 | 第2名 | 600元 | +| TOP_3 | 第3名 | 400元 | + +## 使用说明 + +**取值方式** + +```sql +-- 获取冲刺奖金（取最高档） +SELECT * FROM billiards_dws.cfg_bonus_rules +WHERE rule_type = 'SPRINT' + AND threshold_hours <= 200 -- 实际小时数 + AND effective_from <= '2026-01-01' + AND effective_to >= '2026-01-01' +ORDER BY priority DESC +LIMIT 1; + +-- 获取Top3排名奖金 +SELECT * FROM billiards_dws.cfg_bonus_rules +WHERE rule_type = 'TOP_RANK' + AND rank_position = 1 -- 排名 + AND effective_from <= '2026-01-01' + AND effective_to >= '2026-01-01'; +``` + +**排名口径说明** +- Top3排名按有效业绩小时数（effective_hours）降序排列 +- 如遇并列则都算（如2个第一，则记为2个第一，下一个是第三） diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_performance_tier.md b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_performance_tier.md new file mode 100644 index 0000000..265ffcc --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_performance_tier.md @@ -0,0 +1,71 @@ +# cfg_performance_tier 绩效档位配置表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | cfg_performance_tier | +| 主键 | tier_id | +| 数据来源 | 手工维护/seed脚本 | +| 说明 | 助教绩效档位配置，包含6档阈值、抽成比例、假期天数 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 | +|------|--------|------|------|------|------| +| 1 | tier_id | SERIAL | NO | PK | 档位ID（自增） | +| 2 | tier_code | VARCHAR(20) | NO | | 档位代码。**枚举值**: T0, T1, T2, T3, T4, T5, NEW | +| 3 | tier_name | VARCHAR(50) | NO | | 档位名称 | +| 4 | tier_level | INTEGER | NO | | 档位等级（数字越大档位越高） | +| 5 | min_hours | NUMERIC(10,2) | NO | | 最低业绩小时数阈值（>=） | +| 6 | max_hours | NUMERIC(10,2) | YES | | 最高业绩小时数阈值（<，NULL表示无上限） | +| 7 | base_deduction | NUMERIC(10,2) | NO | | 专业课抽成（元/小时），球房从基础课扣除 | +| 8 | bonus_deduction_ratio | NUMERIC(5,4) | NO | | 打赏课抽成比例（0-1） | +| 9 | vacation_days | INTEGER | NO | | 次月可休假天数 | +| 10 | vacation_unlimited | BOOLEAN | NO | | 是否休假自由（5档为TRUE） | +| 11 | is_new_hire_tier | BOOLEAN | NO | | 是否为新入职专用档位 | +| 12 | effective_from | DATE | NO | | 生效起始日期（含） | +| 13 | effective_to | DATE | NO | | 生效截止日期（含） | +| 14 | description | TEXT | YES | | 档位说明 | +| 15 | created_at | TIMESTAMPTZ | NO | | 创建时间 | +| 16 | updated_at | TIMESTAMPTZ | NO | | 更新时间 | + +## 档位配置示例 + +| 档位代码 | 档位名称 | 小时数范围 | 专业课抽成 | 打赏课抽成 | 假期 | +|----------|----------|------------|------------|------------|------| +| T0 | 0档 | 0-100 | 23元/小时 | 45% | 4天 | +| T1 | 1档 | 100-130 | 20元/小时 | 42% | 5天 | +| T2 | 2档 | 130-160 | 17元/小时 | 40% | 6天 | +| T3 | 3档 | 160-190 | 13元/小时 | 35% | 7天 | +| T4 | 4档 | 190-230 | 8元/小时 | 30% | 8天 | +| T5 | 5档 | 230+ | 0元/小时 | 0% | 自由 | +| NEW | 新入职 | 任意 | 23元/小时 | 45% | 4天 | + +## 使用说明 + +**取值方式** + +按月份匹配生效的配置： +```sql +-- 获取指定月份的档位配置 +SELECT * FROM billiards_dws.cfg_performance_tier +WHERE effective_from <= '2026-01-01' + AND effective_to >= '2026-01-01' +ORDER BY min_hours; + +-- 根据有效业绩小时数匹配档位 +SELECT * FROM billiards_dws.cfg_performance_tier +WHERE effective_from <= '2026-01-01' + AND effective_to >= '2026-01-01' + AND min_hours <= 185 -- 有效小时数 + AND (max_hours IS NULL OR max_hours > 185) +LIMIT 1; +``` + +**薪资计算公式** +- 基础课收入 = 基础课小时数 × (客户支付价格 - base_deduction) +- 附加课收入 = 附加课小时数 × 190 × (1 - bonus_deduction_ratio) diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_skill_type.md b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_skill_type.md new file mode 100644 index 0000000..0a0d0af --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_cfg_skill_type.md @@ -0,0 +1,62 @@ +# cfg_skill_type 技能→课程类型映射表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | cfg_skill_type | +| 主键 | skill_type_id | +| 数据来源 | 手工维护/seed脚本 | +| 说明 | 将skill_id映射到课程类型（基础课/附加课），避免依赖skill_name文本匹配 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 | +|------|--------|------|------|------|------| +| 1 | skill_type_id | SERIAL | NO | PK | 映射ID（自增） | +| 2 | skill_id | BIGINT | NO | UK | 技能ID（来自dwd_assistant_service_log.skill_id） | +| 3 | skill_name | VARCHAR(50) | YES | | 技能名称（仅用于展示和校验） | +| 4 | course_type_code | VARCHAR(10) | NO | | 课程类型代码。**枚举值**: BASE（基础课）, BONUS（附加课） | +| 5 | course_type_name | VARCHAR(20) | NO | | 课程类型名称 | +| 6 | is_active | BOOLEAN | NO | | 是否启用 | +| 7 | description | TEXT | YES | | 说明 | +| 8 | created_at | TIMESTAMPTZ | NO | | 创建时间 | +| 9 | updated_at | TIMESTAMPTZ | NO | | 更新时间 | + +## 技能映射示例 + +| skill_id | skill_name | 课程类型代码 | 课程类型名称 | +|----------|------------|--------------|--------------| +| 2791903611396869 | 陪打/PD | BASE | 基础课 | +| 2807440316432197 | 超休/CX | BONUS | 附加课 | + +## 使用说明 + +**取值方式** + +```sql +-- 将服务记录分类为基础课/附加课 +SELECT + asl.*, + COALESCE(st.course_type_code, 'BASE') AS course_type_code, + COALESCE(st.course_type_name, '基础课') AS course_type_name +FROM billiards_dwd.dwd_assistant_service_log asl +LEFT JOIN billiards_dws.cfg_skill_type st + ON asl.skill_id = st.skill_id + AND st.is_active = TRUE; + +-- 按课程类型汇总小时数 +SELECT + COALESCE(st.course_type_code, 'BASE') AS course_type, + SUM(asl.income_seconds) / 3600.0 AS total_hours +FROM billiards_dwd.dwd_assistant_service_log asl +LEFT JOIN billiards_dws.cfg_skill_type st ON asl.skill_id = st.skill_id +GROUP BY COALESCE(st.course_type_code, 'BASE'); +``` + +**说明** +- 基础课（陪打/PD）: 按等级定价，客户支付98-138元/小时 +- 附加课（超休/CX）: 固定客户支付190元/小时，助教收入50元/小时 diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_customer_stats.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_customer_stats.md new file mode 100644 index 0000000..38358bb --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_customer_stats.md @@ -0,0 +1,98 @@ +# dws_assistant_customer_stats 助教服务客户统计表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_assistant_customer_stats | +| 主键 | id | +| 唯一键 | (site_id, assistant_id, member_id, stat_date) | +| 数据来源 | dwd_assistant_service_log | +| 更新频率 | 每日更新 | +| 说明 | 以"助教+客户"为粒度，统计服务关系和滚动窗口指标 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | assistant_id | BIGINT | NO | 助教ID | +| 5 | assistant_nickname | VARCHAR(50) | YES | 助教花名 | +| 6 | member_id | BIGINT | NO | 客户ID（member_id=0散客不入此表） | +| 7 | member_nickname | VARCHAR(100) | YES | 客户昵称 | +| 8 | member_mobile | VARCHAR(20) | YES | 客户手机号（脱敏） | +| 9 | stat_date | DATE | NO | 统计基准日期 | +| 10 | first_service_date | DATE | YES | 首次服务日期 | +| 11 | last_service_date | DATE | YES | 最近服务日期 | +| 12 | total_service_count | INTEGER | NO | 累计服务次数 | +| 13 | total_service_hours | NUMERIC(10,2) | NO | 累计服务小时数 | +| 14 | total_service_amount | NUMERIC(12,2) | NO | 累计服务金额 | +| 15-20 | service_count_7d/10d/15d/30d/60d/90d | INTEGER | NO | 近N天服务次数 | +| 21-26 | service_hours_7d/10d/15d/30d/60d/90d | NUMERIC(10,2) | NO | 近N天服务小时数 | +| 27-32 | service_amount_7d/10d/15d/30d/60d/90d | NUMERIC(12,2) | NO | 近N天服务金额 | +| 33 | days_since_last | INTEGER | YES | 距离最近服务的天数 | +| 34 | is_active_7d | BOOLEAN | NO | 近7天是否活跃 | +| 35 | is_active_30d | BOOLEAN | NO | 近30天是否活跃 | +| 36 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 37 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 滚动窗口计算 +```sql +-- 统计每个助教-客户组合的滚动窗口指标 +WITH service_data AS ( + SELECT + site_id, + site_assistant_id AS assistant_id, + tenant_member_id AS member_id, + DATE(create_time) AS service_date, + COUNT(*) AS service_count, + SUM(income_seconds) / 3600.0 AS service_hours, + SUM(ledger_amount) AS service_amount + FROM billiards_dwd.dwd_assistant_service_log + WHERE is_delete = 0 + AND tenant_member_id != 0 -- 排除散客 + GROUP BY site_id, site_assistant_id, tenant_member_id, DATE(create_time) +) +SELECT + assistant_id, + member_id, + :stat_date AS stat_date, + MIN(service_date) AS first_service_date, + MAX(service_date) AS last_service_date, + SUM(service_count) AS total_service_count, + SUM(CASE WHEN service_date >= :stat_date - 6 THEN service_count ELSE 0 END) AS service_count_7d, + SUM(CASE WHEN service_date >= :stat_date - 29 THEN service_count ELSE 0 END) AS service_count_30d, + -- ... 其他窗口 +FROM service_data +GROUP BY assistant_id, member_id; +``` + +## 使用说明 + +**散客处理** +- member_id=0 的散客不进入此表统计 +- 仅统计有会员身份的客户 + +**活跃度判断** +```sql +-- 近7天活跃 = 近7天有服务记录 +is_active_7d = (service_count_7d > 0) +-- 近30天活跃 = 近30天有服务记录 +is_active_30d = (service_count_30d > 0) +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-21 ~ 至今 | +| 依赖表 | dwd_assistant_service_log, dim_member | +| 注意事项 | 滚动窗口需要足够的历史数据支撑 | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_daily_detail.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_daily_detail.md new file mode 100644 index 0000000..3d8abcc --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_daily_detail.md @@ -0,0 +1,109 @@ +# dws_assistant_daily_detail 助教日度业绩明细表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_assistant_daily_detail | +| 主键 | id | +| 唯一键 | (site_id, assistant_id, stat_date) | +| 数据来源 | dwd_assistant_service_log + dwd_assistant_trash_event | +| 更新频率 | 每小时增量更新 | +| 说明 | 以"助教+日期"为粒度，汇总每日业绩明细 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | assistant_id | BIGINT | NO | 助教ID（dim_assistant.assistant_id） | +| 5 | assistant_nickname | VARCHAR(50) | YES | 助教花名（冗余，便于查询展示） | +| 6 | stat_date | DATE | NO | 统计日期 | +| 7 | assistant_level_code | INTEGER | YES | 助教等级代码（SCD2口径：取stat_date当日生效的等级） | +| 8 | assistant_level_name | VARCHAR(20) | YES | 助教等级名称 | +| 9 | total_service_count | INTEGER | NO | 总服务次数 | +| 10 | base_service_count | INTEGER | NO | 基础课服务次数 | +| 11 | bonus_service_count | INTEGER | NO | 附加课服务次数 | +| 12 | total_seconds | INTEGER | NO | 总计费时长（秒） | +| 13 | base_seconds | INTEGER | NO | 基础课计费时长（秒） | +| 14 | bonus_seconds | INTEGER | NO | 附加课计费时长（秒） | +| 15 | total_hours | NUMERIC(10,2) | NO | 总计费小时数 | +| 16 | base_hours | NUMERIC(10,2) | NO | 基础课小时数 | +| 17 | bonus_hours | NUMERIC(10,2) | NO | 附加课小时数 | +| 18 | total_ledger_amount | NUMERIC(12,2) | NO | 总计费金额（元） | +| 19 | base_ledger_amount | NUMERIC(12,2) | NO | 基础课计费金额 | +| 20 | bonus_ledger_amount | NUMERIC(12,2) | NO | 附加课计费金额 | +| 21 | unique_customers | INTEGER | NO | 服务客户数（去重） | +| 22 | unique_tables | INTEGER | NO | 服务台桌数（去重） | +| 23 | trashed_seconds | INTEGER | NO | 被废除的服务时长（秒） | +| 24 | trashed_count | INTEGER | NO | 被废除的服务次数 | +| 25 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 26 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 主要来源：dwd_assistant_service_log +```sql +SELECT + site_id, + DATE(create_time) AS stat_date, + site_assistant_id AS assistant_id, + nickname AS assistant_nickname, + COUNT(*) AS total_service_count, + SUM(income_seconds) AS total_seconds, + SUM(ledger_amount) AS total_ledger_amount, + COUNT(DISTINCT tenant_member_id) AS unique_customers, + COUNT(DISTINCT site_table_id) AS unique_tables +FROM billiards_dwd.dwd_assistant_service_log +WHERE is_delete = 0 +GROUP BY site_id, DATE(create_time), site_assistant_id, nickname; +``` + +### 废除记录：dwd_assistant_trash_event +```sql +SELECT + site_id, + DATE(create_time) AS stat_date, + assistant_no, + assistant_name, + SUM(charge_minutes_raw * 60) AS trashed_seconds, + COUNT(*) AS trashed_count +FROM billiards_dwd.dwd_assistant_trash_event +GROUP BY site_id, DATE(create_time), assistant_no, assistant_name; +``` + +## 使用说明 + +**时间分层查询** +```sql +-- 近2天 +SELECT * FROM billiards_dws.dws_assistant_daily_detail +WHERE stat_date >= CURRENT_DATE - 1; + +-- 近1月 +SELECT * FROM billiards_dws.dws_assistant_daily_detail +WHERE stat_date >= CURRENT_DATE - INTERVAL '1 month'; + +-- 月度汇总 +SELECT + assistant_id, + DATE_TRUNC('month', stat_date) AS stat_month, + SUM(total_hours) AS total_hours, + SUM(base_hours) AS base_hours, + SUM(bonus_hours) AS bonus_hours +FROM billiards_dws.dws_assistant_daily_detail +GROUP BY assistant_id, DATE_TRUNC('month', stat_date); +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-21 ~ 至今 | +| 依赖表 | dwd_assistant_service_log, dwd_assistant_trash_event, dim_assistant | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_finance_analysis.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_finance_analysis.md new file mode 100644 index 0000000..92052eb --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_finance_analysis.md @@ -0,0 +1,88 @@ +# dws_assistant_finance_analysis 助教收支分析表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_assistant_finance_analysis | +| 主键 | id | +| 唯一键 | (site_id, stat_date, assistant_id) | +| 数据来源 | dwd_assistant_service_log + dws_assistant_salary_calc | +| 更新频率 | 每日更新 | +| 说明 | 以"日期+助教"为粒度，分析助教产出的收入和成本 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | stat_date | DATE | NO | 统计日期 | +| 5 | assistant_id | BIGINT | NO | 助教ID | +| 6 | assistant_nickname | VARCHAR(50) | YES | 助教花名 | +| 7 | revenue_total | NUMERIC(14,2) | NO | 助教产出收入（ledger_amount汇总） | +| 8 | revenue_base | NUMERIC(14,2) | NO | 基础课收入 | +| 9 | revenue_bonus | NUMERIC(14,2) | NO | 附加课收入 | +| 10 | cost_daily | NUMERIC(14,2) | NO | 日均工资成本（月工资/工作天数） | +| 11 | gross_profit | NUMERIC(14,2) | NO | 毛利 = 收入 - 成本 | +| 12 | gross_margin | NUMERIC(5,4) | NO | 毛利率 | +| 13 | service_count | INTEGER | NO | 服务次数 | +| 14 | service_hours | NUMERIC(10,2) | NO | 服务小时数 | +| 15 | unique_customers | INTEGER | NO | 服务客户数 | +| 16 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 17 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 收入来源：dwd_assistant_service_log +```sql +SELECT + DATE(create_time) AS stat_date, + site_assistant_id AS assistant_id, + SUM(ledger_amount) AS revenue_total, + SUM(CASE WHEN skill_id = 2791903611396869 THEN ledger_amount ELSE 0 END) AS revenue_base, + SUM(CASE WHEN skill_id = 2807440316432197 THEN ledger_amount ELSE 0 END) AS revenue_bonus, + COUNT(*) AS service_count, + SUM(income_seconds) / 3600.0 AS service_hours, + COUNT(DISTINCT tenant_member_id) AS unique_customers +FROM billiards_dwd.dwd_assistant_service_log +WHERE is_delete = 0 +GROUP BY DATE(create_time), site_assistant_id; +``` + +### 成本来源：dws_assistant_salary_calc +```sql +-- 日均成本 = 月度应发工资 / 当月工作天数 +SELECT + assistant_id, + salary_month, + gross_salary / NULLIF(work_days, 0) AS cost_daily +FROM billiards_dws.dws_assistant_salary_calc sc +JOIN billiards_dws.dws_assistant_monthly_summary ms + ON sc.assistant_id = ms.assistant_id AND sc.salary_month = ms.stat_month; +``` + +## 使用说明 + +**毛利计算** +``` +gross_profit = revenue_total - cost_daily +gross_margin = gross_profit / NULLIF(revenue_total, 0) +``` + +**注意事项** +- cost_daily 基于月度工资分摊，非实际日薪 +- 当月数据在月末工资计算前 cost_daily 可能不准确 + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ⚠️ 部分可回溯 | +| 数据范围 | 2025-07-21 ~ 至今 | +| 依赖表 | dwd_assistant_service_log, dws_assistant_salary_calc | +| 限制 | cost_daily 依赖 salary_calc，需先完成薪资计算 | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_monthly_summary.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_monthly_summary.md new file mode 100644 index 0000000..35e209d --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_monthly_summary.md @@ -0,0 +1,110 @@ +# dws_assistant_monthly_summary 助教月度业绩汇总表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_assistant_monthly_summary | +| 主键 | id | +| 唯一键 | (site_id, assistant_id, stat_month) | +| 数据来源 | dws_assistant_daily_detail 聚合 + cfg_performance_tier | +| 更新频率 | 每日更新当月数据 | +| 说明 | 以"助教+月份"为粒度，汇总月度业绩及档位计算 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | assistant_id | BIGINT | NO | 助教ID | +| 5 | assistant_nickname | VARCHAR(50) | YES | 助教花名 | +| 6 | stat_month | DATE | NO | 统计月份（月第一天，如2026-01-01） | +| 7 | assistant_level_code | INTEGER | YES | 助教等级代码（月末时点） | +| 8 | assistant_level_name | VARCHAR(20) | YES | 助教等级名称 | +| 9 | hire_date | DATE | YES | 入职日期 | +| 10 | is_new_hire | BOOLEAN | NO | 是否新入职（入职日期 >= 月1日0点） | +| 11 | work_days | INTEGER | NO | 有服务天数 | +| 12 | total_service_count | INTEGER | NO | 总服务次数 | +| 13 | base_service_count | INTEGER | NO | 基础课服务次数 | +| 14 | bonus_service_count | INTEGER | NO | 附加课服务次数 | +| 15 | total_hours | NUMERIC(10,2) | NO | 总计费小时数 | +| 16 | base_hours | NUMERIC(10,2) | NO | 基础课小时数 | +| 17 | bonus_hours | NUMERIC(10,2) | NO | 附加课小时数 | +| 18 | effective_hours | NUMERIC(10,2) | NO | 有效业绩小时数（影响档位）= total_hours - trashed_hours | +| 19 | trashed_hours | NUMERIC(10,2) | NO | 被废除小时数 | +| 20 | total_ledger_amount | NUMERIC(12,2) | NO | 总计费金额 | +| 21 | base_ledger_amount | NUMERIC(12,2) | NO | 基础课计费金额 | +| 22 | bonus_ledger_amount | NUMERIC(12,2) | NO | 附加课计费金额 | +| 23 | unique_customers | INTEGER | NO | 月度服务客户数（去重） | +| 24 | unique_tables | INTEGER | NO | 月度服务台桌数（去重） | +| 25 | avg_service_seconds | NUMERIC(10,2) | NO | 平均单次服务时长（秒） | +| 26 | tier_id | INTEGER | YES | 匹配的档位ID | +| 27 | tier_code | VARCHAR(20) | YES | 档位代码（T0-T5/NEW） | +| 28 | tier_name | VARCHAR(50) | YES | 档位名称 | +| 29 | rank_by_hours | INTEGER | YES | 月度排名（按effective_hours降序） | +| 30 | rank_with_ties | INTEGER | YES | 考虑并列的排名 | +| 31 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 32 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 从日度明细聚合 +```sql +SELECT + site_id, + tenant_id, + assistant_id, + DATE_TRUNC('month', stat_date)::DATE AS stat_month, + COUNT(DISTINCT stat_date) AS work_days, + SUM(total_service_count) AS total_service_count, + SUM(base_service_count) AS base_service_count, + SUM(bonus_service_count) AS bonus_service_count, + SUM(total_hours) AS total_hours, + SUM(base_hours) AS base_hours, + SUM(bonus_hours) AS bonus_hours, + SUM(trashed_seconds) / 3600.0 AS trashed_hours +FROM billiards_dws.dws_assistant_daily_detail +GROUP BY site_id, tenant_id, assistant_id, DATE_TRUNC('month', stat_date); +``` + +### 档位匹配 +```sql +-- 根据有效业绩匹配档位 +SELECT * FROM billiards_dws.cfg_performance_tier +WHERE min_hours <= :effective_hours + AND (max_hours IS NULL OR max_hours > :effective_hours) + AND effective_from <= :stat_month + AND effective_to >= :stat_month + AND is_new_hire_tier = :is_new_hire +LIMIT 1; +``` + +## 使用说明 + +**新入职判断** +- 入职日期 >= 统计月1日0点 则为新入职 +- 新入职使用NEW档位配置 + +**排名计算** +```sql +-- rank_with_ties: 并列排名（如2个第一则都是1，下一个是3） +SELECT + assistant_id, + effective_hours, + RANK() OVER (ORDER BY effective_hours DESC) AS rank_with_ties +FROM billiards_dws.dws_assistant_monthly_summary +WHERE stat_month = '2026-01-01'; +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025年8月起（需要完整月数据） | +| 依赖表 | dws_assistant_daily_detail, cfg_performance_tier, dim_assistant | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_recharge_commission.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_recharge_commission.md new file mode 100644 index 0000000..1758404 --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_recharge_commission.md @@ -0,0 +1,84 @@ +# dws_assistant_recharge_commission 助教充值提成表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_assistant_recharge_commission | +| 主键 | id | +| 数据来源 | Excel手动导入 | +| 更新频率 | 按需导入 | +| 说明 | 以"助教+月份+充值订单"为粒度，记录充值提成 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | assistant_id | BIGINT | NO | 助教ID | +| 5 | assistant_nickname | VARCHAR(50) | YES | 助教花名 | +| 6 | commission_month | DATE | NO | 提成月份（月第一天） | +| 7 | recharge_order_id | BIGINT | YES | 充值订单ID | +| 8 | recharge_order_no | VARCHAR(50) | YES | 充值订单号 | +| 9 | recharge_amount | NUMERIC(12,2) | NO | 充值订单金额 | +| 10 | commission_amount | NUMERIC(12,2) | NO | 提成金额 | +| 11 | commission_ratio | NUMERIC(5,4) | YES | 提成比例 | +| 12 | import_batch_no | VARCHAR(50) | YES | 导入批次号 | +| 13 | import_file_name | VARCHAR(200) | YES | 导入文件名 | +| 14 | import_time | TIMESTAMPTZ | YES | 导入时间 | +| 15 | import_user | VARCHAR(50) | YES | 导入操作人 | +| 16 | remark | TEXT | YES | 备注 | +| 17 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 18 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## Excel导入模板 + +| 月份 | 助教号 | 助教花名 | 充值订单金额 | 提成金额 | 备注 | +|------|--------|----------|--------------|----------|------| +| 2026-01 | 1 | 小燕 | 5000.00 | 300.00 | ge | +| 2026-01 | 2 | 小明 | 3000.00 | 180.00 | 续充 | + +### 导入规则 +- **月份**: 必填，格式 2026-01 或 2026/01/01 +- **助教号**: 必填，数字（如 1, 2, 31） +- **助教花名**: 必填，与助教号组合确定唯一助教 +- **充值订单金额**: 选填，单位：元 +- **提成金额**: 必填，单位：元 +- **备注**: 选填 + +### 助教匹配逻辑 +```sql +-- 通过 assistant_no + nickname 查找 assistant_id +SELECT assistant_id +FROM billiards_dwd.dim_assistant +WHERE assistant_no = :assistant_no + AND nickname = :nickname + AND scd2_is_current = 1; +``` + +## 使用说明 + +**汇总到薪资计算** +```sql +-- 获取助教某月的充值提成总额 +SELECT + assistant_id, + commission_month, + SUM(commission_amount) AS total_commission +FROM billiards_dws.dws_assistant_recharge_commission +WHERE commission_month = '2026-01-01' +GROUP BY assistant_id, commission_month; +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ❌ 不可自动回溯 | +| 原因 | 数据来源为Excel手工导入，DWD层无此数据 | +| 处理 | 需要人工补录历史数据 | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_salary_calc.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_salary_calc.md new file mode 100644 index 0000000..cbc58dc --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_assistant_salary_calc.md @@ -0,0 +1,98 @@ +# dws_assistant_salary_calc 助教工资计算详情表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_assistant_salary_calc | +| 主键 | id | +| 唯一键 | (site_id, assistant_id, salary_month) | +| 数据来源 | dws_assistant_monthly_summary + cfg_* 配置表 | +| 更新频率 | 月初计算上月工资 | +| 说明 | 以"助教+月份"为粒度，计算月度工资明细 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | assistant_id | BIGINT | NO | 助教ID | +| 5 | assistant_nickname | VARCHAR(50) | YES | 助教花名 | +| 6 | salary_month | DATE | NO | 工资月份（月第一天） | +| 7 | assistant_level_code | INTEGER | YES | 助教等级代码 | +| 8 | assistant_level_name | VARCHAR(20) | YES | 助教等级名称 | +| 9 | hire_date | DATE | YES | 入职日期 | +| 10 | is_new_hire | BOOLEAN | NO | 是否新入职 | +| 11 | effective_hours | NUMERIC(10,2) | NO | 有效业绩小时数 | +| 12 | base_hours | NUMERIC(10,2) | NO | 基础课小时数 | +| 13 | bonus_hours | NUMERIC(10,2) | NO | 附加课小时数 | +| 14 | tier_id | INTEGER | YES | 档位ID | +| 15 | tier_code | VARCHAR(20) | YES | 档位代码 | +| 16 | tier_name | VARCHAR(50) | YES | 档位名称 | +| 17 | rank_with_ties | INTEGER | YES | 月度排名（考虑并列） | +| 18 | base_course_price | NUMERIC(10,2) | NO | 基础课客户支付价格 | +| 19 | bonus_course_price | NUMERIC(10,2) | NO | 附加课客户支付价格（固定190） | +| 20 | base_deduction | NUMERIC(10,2) | NO | 专业课抽成（元/小时） | +| 21 | bonus_deduction_ratio | NUMERIC(5,4) | NO | 打赏课抽成比例 | +| 22 | base_income | NUMERIC(12,2) | NO | 基础课收入 | +| 23 | bonus_income | NUMERIC(12,2) | NO | 附加课收入 | +| 24 | total_course_income | NUMERIC(12,2) | NO | 课时收入合计 | +| 25 | sprint_bonus | NUMERIC(12,2) | NO | 冲刺奖金 | +| 26 | top_rank_bonus | NUMERIC(12,2) | NO | Top3排名奖金 | +| 27 | recharge_commission | NUMERIC(12,2) | NO | 充值提成 | +| 28 | other_bonus | NUMERIC(12,2) | NO | 其他奖金 | +| 29 | total_bonus | NUMERIC(12,2) | NO | 奖金合计 | +| 30 | gross_salary | NUMERIC(12,2) | NO | 应发工资 | +| 31 | vacation_days | INTEGER | NO | 次月可休假天数 | +| 32 | vacation_unlimited | BOOLEAN | NO | 休假自由标记 | +| 33 | calc_notes | TEXT | YES | 计算备注 | +| 34 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 35 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 工资计算公式 + +### 课时收入 +``` +基础课收入 = base_hours × (base_course_price - base_deduction) +附加课收入 = bonus_hours × 190 × (1 - bonus_deduction_ratio) +课时收入合计 = 基础课收入 + 附加课收入 +``` + +### 奖金 +``` +冲刺奖金: H>=190得300元, H>=220得800元（不累计取最高） +Top3奖金: 1st=1000元, 2nd=600元, 3rd=400元 +充值提成: 来自dws_assistant_recharge_commission +``` + +### 应发工资 +``` +gross_salary = total_course_income + total_bonus +``` + +## 计算示例 + +| 项目 | 数值 | 计算过程 | +|------|------|----------| +| 基础课小时数 | 170 | 来自monthly_summary | +| 附加课小时数 | 15 | 来自monthly_summary | +| 等级 | 中级(20) | base_course_price=108 | +| 档位 | T3 | base_deduction=13, bonus_ratio=0.35 | +| 基础课收入 | 16,150 | 170 × (108-13) | +| 附加课收入 | 1,852.5 | 15 × 190 × 0.65 | +| 冲刺奖金 | 300 | 185>=190 | +| 应发工资 | 18,302.5 | 16,150 + 1,852.5 + 300 | + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ⚠️ 部分可回溯 | +| 数据范围 | 2025年8月起 | +| 依赖表 | dws_assistant_monthly_summary, cfg_*, dws_assistant_recharge_commission | +| 限制 | 充值提成需手工导入历史数据 | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_daily_summary.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_daily_summary.md new file mode 100644 index 0000000..deab3bf --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_daily_summary.md @@ -0,0 +1,125 @@ +# dws_finance_daily_summary 财务日度汇总表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_finance_daily_summary | +| 主键 | id | +| 唯一键 | (site_id, stat_date) | +| 数据来源 | dwd_settlement_head + 多个DWD事实表 | +| 更新频率 | 每小时更新当日数据 | +| 说明 | 以"日期"为粒度，汇总当日财务数据 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | stat_date | DATE | NO | 统计日期 | +| 5 | gross_amount | NUMERIC(14,2) | NO | 发生额合计 | +| 6 | table_fee_amount | NUMERIC(14,2) | NO | 台费正价 | +| 7 | goods_amount | NUMERIC(14,2) | NO | 商品正价 | +| 8 | assistant_pd_amount | NUMERIC(14,2) | NO | 助教基础课正价（陪打） | +| 9 | assistant_cx_amount | NUMERIC(14,2) | NO | 助教激励课正价（超休） | +| 10 | discount_total | NUMERIC(14,2) | NO | 优惠合计 | +| 11 | discount_groupbuy | NUMERIC(14,2) | NO | 团购优惠 | +| 12 | discount_vip | NUMERIC(14,2) | NO | 会员折扣 | +| 13 | discount_gift_card | NUMERIC(14,2) | NO | 赠送卡抵扣 | +| 14 | discount_manual | NUMERIC(14,2) | NO | 手动调整 | +| 15 | discount_rounding | NUMERIC(14,2) | NO | 抹零 | +| 16 | discount_other | NUMERIC(14,2) | NO | 其他优惠 | +| 17 | confirmed_income | NUMERIC(14,2) | NO | 确认收入 = 发生额 - 优惠 | +| 18 | cash_inflow_total | NUMERIC(14,2) | NO | 现金流入合计 | +| 19 | cash_pay_amount | NUMERIC(14,2) | NO | 收银实付 | +| 20 | groupbuy_pay_amount | NUMERIC(14,2) | NO | 团购支付金额 | +| 21 | platform_settlement_amount | NUMERIC(14,2) | NO | 平台回款金额（导入） | +| 22 | platform_fee_amount | NUMERIC(14,2) | NO | 平台佣金+服务费（导入） | +| 23 | recharge_cash_inflow | NUMERIC(14,2) | NO | 充值现金流入 | +| 24 | card_consume_total | NUMERIC(14,2) | NO | 卡消费合计 | +| 25 | cash_card_consume | NUMERIC(14,2) | NO | 储值卡消费 | +| 26 | gift_card_consume | NUMERIC(14,2) | NO | 赠送卡消费 | +| 27 | cash_outflow_total | NUMERIC(14,2) | NO | 现金流出合计 | +| 28 | cash_balance_change | NUMERIC(14,2) | NO | 现金余额变动 | +| 29 | recharge_count | INTEGER | NO | 充值笔数 | +| 30 | recharge_total | NUMERIC(14,2) | NO | 充值总额（含赠送） | +| 31 | recharge_cash | NUMERIC(14,2) | NO | 充值现金部分 | +| 32 | recharge_gift | NUMERIC(14,2) | NO | 充值赠送部分 | +| 33 | first_recharge_count | INTEGER | NO | 首充笔数 | +| 34 | first_recharge_amount | NUMERIC(14,2) | NO | 首充金额 | +| 35 | renewal_count | INTEGER | NO | 续充笔数 | +| 36 | renewal_amount | NUMERIC(14,2) | NO | 续充金额 | +| 37 | order_count | INTEGER | NO | 结账单数 | +| 38 | member_order_count | INTEGER | NO | 会员订单数 | +| 39 | guest_order_count | INTEGER | NO | 散客订单数 | +| 40 | avg_order_amount | NUMERIC(12,2) | NO | 平均客单价 | +| 41 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 42 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 结账汇总：dwd_settlement_head +```sql +SELECT + DATE(pay_time) AS stat_date, + SUM(table_charge_money) AS table_fee_amount, + SUM(goods_money) AS goods_amount, + SUM(assistant_pd_money) AS assistant_pd_amount, + SUM(assistant_cx_money) AS assistant_cx_amount, + SUM(member_discount_amount) AS discount_vip, + SUM(adjust_amount) AS discount_manual, + SUM(rounding_amount) AS discount_rounding, + SUM(pay_amount) AS cash_pay_amount, + SUM(balance_amount) AS cash_card_consume, + SUM(gift_card_amount) AS gift_card_consume, + COUNT(*) AS order_count +FROM billiards_dwd.dwd_settlement_head +WHERE settle_type = 1 +GROUP BY DATE(pay_time); +``` + +### 团购核销：dwd_groupbuy_redemption +```sql +SELECT + DATE(create_time) AS stat_date, + SUM(coupon_money) AS groupbuy_pay_amount +FROM billiards_dwd.dwd_groupbuy_redemption +WHERE is_delete = 0 +GROUP BY DATE(create_time); +``` + +### 充值订单：dwd_recharge_order +```sql +SELECT + DATE(pay_time) AS stat_date, + COUNT(*) AS recharge_count, + SUM(pay_amount) AS recharge_cash, + SUM(point_amount) AS recharge_gift, + SUM(CASE WHEN is_first = 1 THEN 1 ELSE 0 END) AS first_recharge_count +FROM billiards_dwd.dwd_recharge_order +GROUP BY DATE(pay_time); +``` + +## 使用说明 + +**计算公式** +``` +gross_amount = table_fee_amount + goods_amount + assistant_pd_amount + assistant_cx_amount +discount_total = discount_groupbuy + discount_vip + discount_gift_card + discount_manual + discount_rounding + discount_other +confirmed_income = gross_amount - discount_total +cash_inflow_total = cash_pay_amount + groupbuy_pay_amount + platform_settlement_amount + recharge_cash_inflow +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-16 ~ 至今 | +| 依赖表 | dwd_settlement_head, dwd_groupbuy_redemption, dwd_recharge_order, dws_platform_settlement | +| 注意 | platform_settlement需Excel导入 | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_discount_detail.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_discount_detail.md new file mode 100644 index 0000000..8ac63c2 --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_discount_detail.md @@ -0,0 +1,90 @@ +# dws_finance_discount_detail 优惠明细表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_finance_discount_detail | +| 主键 | id | +| 唯一键 | (site_id, stat_date, discount_type_code) | +| 数据来源 | dwd_settlement_head + dwd_groupbuy_redemption | +| 更新频率 | 每日更新 | +| 说明 | 以"日期+优惠类型"为粒度，分析优惠构成 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | stat_date | DATE | NO | 统计日期 | +| 5 | discount_type_code | VARCHAR(30) | NO | 优惠类型代码 | +| 6 | discount_type_name | VARCHAR(50) | NO | 优惠类型名称 | +| 7 | discount_amount | NUMERIC(14,2) | NO | 优惠金额 | +| 8 | discount_ratio | NUMERIC(5,4) | NO | 优惠占比（占总优惠） | +| 9 | usage_count | INTEGER | NO | 使用次数 | +| 10 | affected_orders | INTEGER | NO | 影响订单数 | +| 11 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 12 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 优惠类型说明 + +| discount_type_code | discount_type_name | 数据来源 | +|--------------------|--------------------|----------| +| GROUPBUY | 团购优惠 | dwd_settlement_head.coupon_amount | +| VIP | 会员折扣 | dwd_settlement_head.member_discount_amount | +| GIFT_CARD | 赠送卡抵扣 | dwd_settlement_head.gift_card_amount | +| MANUAL | 手动调整 | dwd_settlement_head.adjust_amount | +| ROUNDING | 抹零 | dwd_settlement_head.rounding_amount | +| BIG_CUSTOMER | 大客户优惠 | dwd_settlement_head（特定会员优惠） | +| OTHER | 其他优惠 | 其他无法归类的优惠 | + +## 数据来源 + +```sql +-- 从结账头表提取各类优惠 +SELECT + DATE(pay_time) AS stat_date, + 'VIP' AS discount_type_code, + '会员折扣' AS discount_type_name, + SUM(member_discount_amount) AS discount_amount, + COUNT(CASE WHEN member_discount_amount > 0 THEN 1 END) AS usage_count, + COUNT(DISTINCT CASE WHEN member_discount_amount > 0 THEN order_settle_id END) AS affected_orders +FROM billiards_dwd.dwd_settlement_head +WHERE settle_type = 1 +GROUP BY DATE(pay_time) + +UNION ALL + +SELECT + DATE(pay_time) AS stat_date, + 'GROUPBUY' AS discount_type_code, + '团购优惠' AS discount_type_name, + SUM(coupon_amount) AS discount_amount, + COUNT(CASE WHEN coupon_amount > 0 THEN 1 END) AS usage_count, + COUNT(DISTINCT CASE WHEN coupon_amount > 0 THEN order_settle_id END) AS affected_orders +FROM billiards_dwd.dwd_settlement_head +WHERE settle_type = 1 +GROUP BY DATE(pay_time) + +-- ... 其他优惠类型 +``` + +## 使用说明 + +**占比计算** +```sql +discount_ratio = discount_amount / SUM(discount_amount) OVER (PARTITION BY stat_date) +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-16 ~ 至今 | +| 依赖表 | dwd_settlement_head, dwd_groupbuy_redemption | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_expense_summary.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_expense_summary.md new file mode 100644 index 0000000..b4f66ae --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_expense_summary.md @@ -0,0 +1,87 @@ +# dws_finance_expense_summary 支出结构表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_finance_expense_summary | +| 主键 | id | +| 唯一键 | (site_id, expense_month, expense_type_code, import_batch_no) | +| 数据来源 | Excel手动导入 | +| 更新频率 | 按需导入 | +| 说明 | 以"月份+支出类型"为粒度，记录支出数据 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | expense_month | DATE | NO | 支出月份（月第一天） | +| 5 | expense_type_code | VARCHAR(30) | NO | 支出类型代码 | +| 6 | expense_type_name | VARCHAR(50) | NO | 支出类型名称 | +| 7 | expense_category | VARCHAR(20) | YES | 支出大类 | +| 8 | expense_amount | NUMERIC(14,2) | NO | 支出金额 | +| 9 | expense_detail | TEXT | YES | 支出明细说明 | +| 10 | import_batch_no | VARCHAR(50) | YES | 导入批次号 | +| 11 | import_file_name | VARCHAR(200) | YES | 导入文件名 | +| 12 | import_time | TIMESTAMPTZ | YES | 导入时间 | +| 13 | import_user | VARCHAR(50) | YES | 导入操作人 | +| 14 | remark | TEXT | YES | 备注 | +| 15 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 16 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 支出类型说明 + +| expense_type_code | expense_type_name | expense_category | +|-------------------|-------------------|------------------| +| RENT | 房租 | FIXED_COST | +| UTILITY | 水电费 | FIXED_COST | +| PROPERTY | 物业费 | FIXED_COST | +| SALARY | 工资 | VARIABLE_COST | +| REIMBURSE | 报销 | VARIABLE_COST | +| PLATFORM_FEE | 平台费用 | VARIABLE_COST | +| MAINTENANCE | 维修保养 | VARIABLE_COST | +| CONSUMABLES | 耗材 | VARIABLE_COST | +| MARKETING | 营销费用 | VARIABLE_COST | +| OTHER | 其他 | OTHER | + +## Excel导入模板 + +| 月份 | 支出类型 | 支出金额 | 明细说明 | 备注 | +|------|----------|----------|----------|------| +| 2026-01 | 房租 | 50000.00 | 1月房租 | | +| 2026-01 | 水电费 | 8000.00 | 1月水电 | | +| 2026-01 | 工资 | 120000.00 | 员工工资 | | + +### 导入规则 +- **月份**: 必填，格式 2026-01 或 2026/01/01 +- **支出类型**: 必填，需匹配支出类型名称 +- **支出金额**: 必填，单位：元 +- **明细说明**: 选填 +- **备注**: 选填 + +## 使用说明 + +**月度支出汇总** +```sql +SELECT + expense_month, + expense_category, + SUM(expense_amount) AS total_expense +FROM billiards_dws.dws_finance_expense_summary +GROUP BY expense_month, expense_category +ORDER BY expense_month, expense_category; +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ❌ 不可自动回溯 | +| 原因 | 数据来源为Excel手工导入，DWD层无此数据 | +| 处理 | 需要人工补录历史数据 | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_income_structure.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_income_structure.md new file mode 100644 index 0000000..3d52e34 --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_income_structure.md @@ -0,0 +1,88 @@ +# dws_finance_income_structure 收入结构分析表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_finance_income_structure | +| 主键 | id | +| 唯一键 | (site_id, stat_date, structure_type, category_code) | +| 数据来源 | dwd_table_fee_log + dwd_assistant_service_log + cfg_area_category | +| 更新频率 | 每日更新 | +| 说明 | 以"日期+区域/类型"为粒度，分析收入结构 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | stat_date | DATE | NO | 统计日期 | +| 5 | structure_type | VARCHAR(20) | NO | 结构类型。**枚举值**: AREA（区域）, INCOME_TYPE（收入类型） | +| 6 | category_code | VARCHAR(30) | NO | 分类代码 | +| 7 | category_name | VARCHAR(50) | NO | 分类名称 | +| 8 | income_amount | NUMERIC(14,2) | NO | 收入金额 | +| 9 | income_ratio | NUMERIC(5,4) | NO | 收入占比 | +| 10 | order_count | INTEGER | NO | 订单数 | +| 11 | duration_minutes | INTEGER | NO | 时长（分钟） | +| 12 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 13 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 分类代码说明 + +### 按区域分析 (structure_type = 'AREA') +| category_code | category_name | 来源 | +|---------------|---------------|------| +| BILLIARD | 台球散台 | A区/B区/C区/TV台 | +| BILLIARD_VIP | 台球VIP | VIP包厢 | +| SNOOKER | 斯诺克 | 斯诺克区 | +| MAHJONG | 麻将棋牌 | 麻将房/M7/M8/666/发财 | +| KTV | K歌娱乐 | K包/k包活动区/幸会158 | +| SPECIAL | 补时长 | 补时长 | +| OTHER | 其他 | 未映射区域 | + +### 按收入类型分析 (structure_type = 'INCOME_TYPE') +| category_code | category_name | +|---------------|---------------| +| TABLE_FEE | 台费收入 | +| GOODS | 商品收入 | +| ASSISTANT_BASE | 助教基础课收入 | +| ASSISTANT_BONUS | 助教附加课收入 | + +## 数据来源 + +### 按区域汇总台费 +```sql +SELECT + DATE(tfl.ledger_end_time) AS stat_date, + COALESCE(ac.category_code, 'OTHER') AS category_code, + COALESCE(ac.category_name, '其他') AS category_name, + SUM(tfl.ledger_amount) AS income_amount, + SUM(tfl.ledger_count) AS duration_seconds, + COUNT(DISTINCT tfl.order_settle_id) AS order_count +FROM billiards_dwd.dwd_table_fee_log tfl +LEFT JOIN billiards_dwd.dim_table dt ON dt.table_id = tfl.site_table_id +LEFT JOIN billiards_dws.cfg_area_category ac ON dt.site_table_area_name = ac.source_area_name +WHERE tfl.is_delete = 0 +GROUP BY DATE(tfl.ledger_end_time), COALESCE(ac.category_code, 'OTHER'), COALESCE(ac.category_name, '其他'); +``` + +## 使用说明 + +**占比计算** +```sql +-- income_ratio = 当前分类收入 / 当日总收入 +income_ratio = income_amount / SUM(income_amount) OVER (PARTITION BY stat_date, structure_type) +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-21 ~ 至今 | +| 依赖表 | dwd_table_fee_log, dwd_assistant_service_log, dim_table, cfg_area_category | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_recharge_summary.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_recharge_summary.md new file mode 100644 index 0000000..6a7145a --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_finance_recharge_summary.md @@ -0,0 +1,95 @@ +# dws_finance_recharge_summary 充值统计表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_finance_recharge_summary | +| 主键 | id | +| 唯一键 | (site_id, stat_date) | +| 数据来源 | dwd_recharge_order | +| 更新频率 | 每日更新 | +| 说明 | 以"日期"为粒度，统计充值数据，区分首充/续充 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | stat_date | DATE | NO | 统计日期 | +| 5 | recharge_count | INTEGER | NO | 充值笔数 | +| 6 | recharge_total | NUMERIC(14,2) | NO | 充值总额（含赠送） | +| 7 | recharge_cash | NUMERIC(14,2) | NO | 现金充值金额 | +| 8 | recharge_gift | NUMERIC(14,2) | NO | 赠送金额 | +| 9 | first_recharge_count | INTEGER | NO | 首充笔数 | +| 10 | first_recharge_cash | NUMERIC(14,2) | NO | 首充现金 | +| 11 | first_recharge_gift | NUMERIC(14,2) | NO | 首充赠送 | +| 12 | first_recharge_total | NUMERIC(14,2) | NO | 首充总额 | +| 13 | renewal_count | INTEGER | NO | 续充笔数 | +| 14 | renewal_cash | NUMERIC(14,2) | NO | 续充现金 | +| 15 | renewal_gift | NUMERIC(14,2) | NO | 续充赠送 | +| 16 | renewal_total | NUMERIC(14,2) | NO | 续充总额 | +| 17 | recharge_member_count | INTEGER | NO | 充值会员数（去重） | +| 18 | new_member_count | INTEGER | NO | 新增会员数 | +| 19 | total_card_balance | NUMERIC(14,2) | NO | 全部会员卡余额（当日末） | +| 20 | cash_card_balance | NUMERIC(14,2) | NO | 储值卡余额 | +| 21 | gift_card_balance | NUMERIC(14,2) | NO | 赠送卡余额 | +| 22 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 23 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 充值订单：dwd_recharge_order +```sql +SELECT + DATE(pay_time) AS stat_date, + COUNT(*) AS recharge_count, + SUM(pay_amount + point_amount) AS recharge_total, + SUM(pay_amount) AS recharge_cash, + SUM(point_amount) AS recharge_gift, + -- 首充 + SUM(CASE WHEN is_first = 1 THEN 1 ELSE 0 END) AS first_recharge_count, + SUM(CASE WHEN is_first = 1 THEN pay_amount ELSE 0 END) AS first_recharge_cash, + SUM(CASE WHEN is_first = 1 THEN point_amount ELSE 0 END) AS first_recharge_gift, + -- 续充 + SUM(CASE WHEN is_first = 0 THEN 1 ELSE 0 END) AS renewal_count, + SUM(CASE WHEN is_first = 0 THEN pay_amount ELSE 0 END) AS renewal_cash, + -- 会员数 + COUNT(DISTINCT member_id) AS recharge_member_count +FROM billiards_dwd.dwd_recharge_order +GROUP BY DATE(pay_time); +``` + +### 卡余额快照：dim_member_card_account +```sql +-- 截至stat_date当日末的卡余额 +SELECT + SUM(balance) AS total_card_balance, + SUM(CASE WHEN card_type_id = 2793249295533893 THEN balance ELSE 0 END) AS cash_card_balance, + SUM(CASE WHEN card_type_id != 2793249295533893 THEN balance ELSE 0 END) AS gift_card_balance +FROM billiards_dwd.dim_member_card_account +WHERE scd2_start_time <= :stat_date + INTERVAL '1 day' + AND (scd2_end_time IS NULL OR scd2_end_time > :stat_date + INTERVAL '1 day'); +``` + +## 使用说明 + +**首充判断** +- is_first = 1: 首充 +- is_first = 0: 续充 + +**储值卡ID** +- 储值卡 card_type_id = 2793249295533893 + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-21 ~ 至今 | +| 依赖表 | dwd_recharge_order, dim_member_card_account | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_consumption_summary.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_consumption_summary.md new file mode 100644 index 0000000..4629a3f --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_consumption_summary.md @@ -0,0 +1,102 @@ +# dws_member_consumption_summary 会员消费汇总表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_member_consumption_summary | +| 主键 | id | +| 唯一键 | (site_id, member_id, stat_date) | +| 数据来源 | dwd_settlement_head + 关联明细表 | +| 更新频率 | 每日更新 | +| 说明 | 以"会员"为粒度，统计消费行为和滚动窗口指标 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | member_id | BIGINT | NO | 会员ID（member_id=0散客不入此表） | +| 5 | stat_date | DATE | NO | 统计基准日期 | +| 6 | member_nickname | VARCHAR(100) | YES | 会员昵称 | +| 7 | member_mobile | VARCHAR(20) | YES | 手机号（脱敏） | +| 8 | card_grade_name | VARCHAR(50) | YES | 卡等级名称 | +| 9 | register_date | DATE | YES | 注册日期 | +| 10 | first_consume_date | DATE | YES | 首次消费日期 | +| 11 | last_consume_date | DATE | YES | 最近消费日期 | +| 12 | total_visit_count | INTEGER | NO | 累计到店次数 | +| 13 | total_consume_amount | NUMERIC(14,2) | NO | 累计消费金额 | +| 14 | total_recharge_amount | NUMERIC(14,2) | NO | 累计充值金额 | +| 15 | total_table_fee | NUMERIC(14,2) | NO | 累计台费 | +| 16 | total_goods_amount | NUMERIC(14,2) | NO | 累计商品消费 | +| 17 | total_assistant_amount | NUMERIC(14,2) | NO | 累计助教服务消费 | +| 18-23 | visit_count_7d/10d/15d/30d/60d/90d | INTEGER | NO | 近N天到店次数 | +| 24-29 | consume_amount_7d/10d/15d/30d/60d/90d | NUMERIC(14,2) | NO | 近N天消费金额 | +| 30 | cash_card_balance | NUMERIC(14,2) | NO | 储值卡余额 | +| 31 | gift_card_balance | NUMERIC(14,2) | NO | 赠送卡余额 | +| 32 | total_card_balance | NUMERIC(14,2) | NO | 总卡余额 | +| 33 | days_since_last | INTEGER | YES | 距离最近消费的天数 | +| 34 | is_active_7d | BOOLEAN | NO | 近7天是否活跃 | +| 35 | is_active_30d | BOOLEAN | NO | 近30天是否活跃 | +| 36 | is_active_90d | BOOLEAN | NO | 近90天是否活跃 | +| 37 | customer_tier | VARCHAR(20) | YES | 客户分层（高价值/中等/低活跃/流失） | +| 38 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 39 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 消费统计来源：dwd_settlement_head +```sql +SELECT + site_id, + member_id, + DATE(pay_time) AS consume_date, + COUNT(*) AS visit_count, + SUM(consume_money) AS consume_amount, + SUM(table_charge_money) AS table_fee, + SUM(goods_money) AS goods_amount, + SUM(assistant_pd_money + assistant_cx_money) AS assistant_amount +FROM billiards_dwd.dwd_settlement_head +WHERE member_id != 0 -- 排除散客 + AND settle_type = 1 -- 已结账 +GROUP BY site_id, member_id, DATE(pay_time); +``` + +### 卡余额来源：dim_member_card_account +```sql +SELECT + tenant_member_id AS member_id, + SUM(CASE WHEN card_type_id = 2793249295533893 THEN balance ELSE 0 END) AS cash_card_balance, + SUM(CASE WHEN card_type_id != 2793249295533893 THEN balance ELSE 0 END) AS gift_card_balance +FROM billiards_dwd.dim_member_card_account +WHERE scd2_is_current = 1 +GROUP BY tenant_member_id; +``` + +## 使用说明 + +**散客处理** +- member_id=0 的散客不进入此表统计 + +**客户分层规则** +```sql +customer_tier = CASE + WHEN consume_amount_30d >= 1000 THEN '高价值' + WHEN consume_amount_30d >= 300 THEN '中等' + WHEN is_active_30d THEN '低活跃' + ELSE '流失' +END +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-16 ~ 至今 | +| 依赖表 | dwd_settlement_head, dim_member, dim_member_card_account | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_visit_detail.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_visit_detail.md new file mode 100644 index 0000000..6136158 --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_member_visit_detail.md @@ -0,0 +1,119 @@ +# dws_member_visit_detail 会员来店明细表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_member_visit_detail | +| 主键 | id | +| 唯一键 | (site_id, member_id, order_settle_id) | +| 数据来源 | dwd_settlement_head + 关联明细表 | +| 更新频率 | 每日增量更新 | +| 说明 | 以"会员+订单"为粒度，记录每次来店消费明细 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | member_id | BIGINT | NO | 会员ID（散客不入此表） | +| 5 | order_settle_id | BIGINT | NO | 结账单ID | +| 6 | visit_date | DATE | NO | 来店日期 | +| 7 | visit_time | TIMESTAMPTZ | YES | 来店时间 | +| 8 | member_nickname | VARCHAR(100) | YES | 会员昵称 | +| 9 | member_mobile | VARCHAR(20) | YES | 手机号 | +| 10 | member_birthday | DATE | YES | 会员生日 | +| 11 | table_id | BIGINT | YES | 台桌ID | +| 12 | table_name | VARCHAR(50) | YES | 台桌名称 | +| 13 | area_name | VARCHAR(50) | YES | 区域名称（原始） | +| 14 | area_category | VARCHAR(20) | YES | 区域分类 | +| 15 | table_fee | NUMERIC(12,2) | NO | 台费 | +| 16 | goods_amount | NUMERIC(12,2) | NO | 商品金额 | +| 17 | assistant_amount | NUMERIC(12,2) | NO | 助教服务金额 | +| 18 | total_consume | NUMERIC(12,2) | NO | 消费总额（正价） | +| 19 | total_discount | NUMERIC(12,2) | NO | 优惠总额 | +| 20 | actual_pay | NUMERIC(12,2) | NO | 实付金额 | +| 21 | cash_pay | NUMERIC(12,2) | NO | 现金/刷卡支付 | +| 22 | cash_card_pay | NUMERIC(12,2) | NO | 储值卡支付 | +| 23 | gift_card_pay | NUMERIC(12,2) | NO | 赠送卡支付 | +| 24 | groupbuy_pay | NUMERIC(12,2) | NO | 团购券支付 | +| 25 | table_duration_min | INTEGER | NO | 台桌使用时长（分钟） | +| 26 | assistant_duration_min | INTEGER | NO | 助教服务时长（分钟） | +| 27 | assistant_services | JSONB | YES | 助教服务列表 | +| 28 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 29 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 数据来源 + +### 主表来源：dwd_settlement_head +```sql +SELECT + site_id, + tenant_id, + member_id, + order_settle_id, + DATE(pay_time) AS visit_date, + pay_time AS visit_time, + member_name AS member_nickname, + member_phone AS member_mobile, + table_id, + table_charge_money AS table_fee, + goods_money AS goods_amount, + assistant_pd_money + assistant_cx_money AS assistant_amount, + consume_money AS total_consume, + member_discount_amount + coupon_amount + adjust_amount AS total_discount, + pay_amount AS actual_pay, + balance_amount AS cash_card_pay, + gift_card_amount AS gift_card_pay +FROM billiards_dwd.dwd_settlement_head +WHERE member_id != 0 + AND settle_type = 1; +``` + +### 助教服务明细：dwd_assistant_service_log +```sql +-- 聚合为JSONB格式 +SELECT + order_settle_id, + jsonb_agg(jsonb_build_object( + 'assistant_id', site_assistant_id, + 'nickname', nickname, + 'duration_min', income_seconds / 60, + 'amount', ledger_amount + )) AS assistant_services +FROM billiards_dwd.dwd_assistant_service_log +GROUP BY order_settle_id; +``` + +## 使用说明 + +**assistant_services JSON格式** +```json +[ + {"assistant_id": 123, "nickname": "小燕", "duration_min": 60, "amount": 108.00}, + {"assistant_id": 456, "nickname": "小明", "duration_min": 30, "amount": 54.00} +] +``` + +**区域分类映射** +```sql +-- 通过cfg_area_category映射 +area_category = COALESCE( + (SELECT category_name FROM billiards_dws.cfg_area_category + WHERE source_area_name = dim_table.site_table_area_name), + '其他' +) +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ✅ 完全可回溯 | +| 数据范围 | 2025-07-16 ~ 至今 | +| 依赖表 | dwd_settlement_head, dwd_assistant_service_log, dim_table, dim_member | diff --git a/etl_billiards/docs/bd_manual/dws/BD_manual_dws_platform_settlement.md b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_platform_settlement.md new file mode 100644 index 0000000..b2e974a --- /dev/null +++ b/etl_billiards/docs/bd_manual/dws/BD_manual_dws_platform_settlement.md @@ -0,0 +1,100 @@ +# dws_platform_settlement 平台回款/服务费表 + +> 生成时间：2026-02-03 + +## 表信息 + +| 属性 | 值 | +|------|-----| +| Schema | billiards_dws | +| 表名 | dws_platform_settlement | +| 主键 | id | +| 数据来源 | Excel手动导入 | +| 更新频率 | 按需导入 | +| 说明 | 以"回款日期+平台+订单"为粒度，记录平台结算数据 | + +## 字段说明 + +| 序号 | 字段名 | 类型 | 可空 | 说明 | +|------|--------|------|------|------| +| 1 | id | BIGSERIAL | NO | 自增主键 | +| 2 | site_id | BIGINT | NO | 门店ID | +| 3 | tenant_id | BIGINT | NO | 租户ID | +| 4 | settlement_date | DATE | NO | 回款日期 | +| 5 | platform_type | VARCHAR(30) | NO | 平台类型。**枚举值**: MEITUAN, DOUYIN, DIANPING, OTHER | +| 6 | platform_name | VARCHAR(50) | YES | 平台名称 | +| 7 | platform_order_no | VARCHAR(100) | YES | 平台订单号 | +| 8 | order_settle_id | BIGINT | YES | 关联的结账单ID | +| 9 | settlement_amount | NUMERIC(14,2) | NO | 回款金额（实际入账） | +| 10 | commission_amount | NUMERIC(14,2) | NO | 佣金（平台抽成） | +| 11 | service_fee | NUMERIC(14,2) | NO | 服务费 | +| 12 | gross_amount | NUMERIC(14,2) | NO | 订单原始金额 | +| 13 | import_batch_no | VARCHAR(50) | YES | 导入批次号 | +| 14 | import_file_name | VARCHAR(200) | YES | 导入文件名 | +| 15 | import_time | TIMESTAMPTZ | YES | 导入时间 | +| 16 | import_user | VARCHAR(50) | YES | 导入操作人 | +| 17 | remark | TEXT | YES | 备注 | +| 18 | created_at | TIMESTAMPTZ | NO | 创建时间 | +| 19 | updated_at | TIMESTAMPTZ | NO | 更新时间 | + +## 平台类型说明 + +| platform_type | platform_name | 说明 | +|---------------|---------------|------| +| MEITUAN | 美团 | 美团团购/外卖 | +| DOUYIN | 抖音 | 抖音团购 | +| DIANPING | 大众点评 | 大众点评团购 | +| OTHER | 其他 | 其他平台 | + +## Excel导入模板 + +| 回款日期 | 平台 | 平台订单号 | 订单金额 | 回款金额 | 佣金 | 服务费 | 备注 | +|----------|------|------------|----------|----------|------|--------|------| +| 2026-01-15 | 美团 | MT202601150001 | 200.00 | 186.00 | 12.00 | 2.00 | | +| 2026-01-15 | 抖音 | DY202601150001 | 150.00 | 142.50 | 6.00 | 1.50 | | + +### 导入规则 +- **回款日期**: 必填，实际到账日期 +- **平台**: 必填，美团/抖音/大众点评/其他 +- **平台订单号**: 选填，用于追溯 +- **订单金额**: 必填，订单原始金额 +- **回款金额**: 必填，实际到账金额 +- **佣金**: 选填，平台抽成 +- **服务费**: 选填 + +### 金额关系 +``` +settlement_amount = gross_amount - commission_amount - service_fee +``` + +## 使用说明 + +**日度平台回款汇总** +```sql +SELECT + settlement_date, + platform_type, + SUM(settlement_amount) AS total_settlement, + SUM(commission_amount) AS total_commission, + SUM(service_fee) AS total_service_fee +FROM billiards_dws.dws_platform_settlement +GROUP BY settlement_date, platform_type +ORDER BY settlement_date, platform_type; +``` + +**关联到财务日度汇总** +```sql +-- dws_finance_daily_summary.platform_settlement_amount +SELECT stat_date, SUM(settlement_amount) +FROM billiards_dws.dws_platform_settlement +WHERE settlement_date = :stat_date +GROUP BY stat_date; +``` + +## 可回溯性 + +| 项目 | 说明 | +|------|------| +| 可回溯 | ❌ 不可自动回溯 | +| 原因 | 数据来源为Excel手工导入，需从平台后台导出 | +| 处理 | 需要人工补录历史平台结算数据 | diff --git a/etl_billiards/docs/dws_tables_dictionary.md b/etl_billiards/docs/dws_tables_dictionary.md new file mode 100644 index 0000000..9803466 --- /dev/null +++ b/etl_billiards/docs/dws_tables_dictionary.md @@ -0,0 +1,585 @@ +# DWS 数据字典 + +## 概述 + +DWS（Data Warehouse Service）层是数据仓库的汇总层，基于DWD明细层数据构建，为上层应用和报表提供预聚合的数据服务。 + +### 表清单 + +| 分类 | 表名 | 说明 | 更新频率 | +|------|------|------|----------| +| **配置表** | cfg_performance_tier | 绩效档位配置 | 手动维护 | +| | cfg_assistant_level_price | 助教等级定价 | 手动维护 | +| | cfg_bonus_rules | 奖金规则配置 | 手动维护 | +| | cfg_area_category | 台区分类映射 | 手动维护 | +| | cfg_skill_type | 技能课程类型映射 | 手动维护 | +| **助教维度** | dws_assistant_daily_detail | 助教日度业绩明细 | 每小时 | +| | dws_assistant_monthly_summary | 助教月度业绩汇总 | 每日 | +| | dws_assistant_customer_stats | 助教服务客户统计 | 每日 | +| | dws_assistant_salary_calc | 助教工资计算详情 | 月初 | +| | dws_assistant_recharge_commission | 助教充值提成 | Excel导入 | +| **客户维度** | dws_member_consumption_summary | 会员消费汇总 | 每日 | +| | dws_member_visit_detail | 会员来店明细 | 每日 | +| **财务维度** | dws_finance_daily_summary | 财务日度汇总 | 每小时 | +| | dws_finance_income_structure | 收入结构分析 | 每日 | +| | dws_finance_discount_detail | 优惠明细 | 每日 | +| | dws_finance_recharge_summary | 充值统计 | 每日 | +| | dws_finance_expense_summary | 支出结构 | Excel导入 | +| | dws_assistant_finance_analysis | 助教收支分析 | 每日 | +| | dws_platform_settlement | 平台回款/服务费 | Excel导入 | +| **订单汇总** | dws_order_summary | 订单汇总 | 每日 | + +--- + +## 一、配置表 + +### 1.1 cfg_performance_tier - 绩效档位配置 + +| 字段 | 类型 | 说明 | +|------|------|------| +| tier_id | SERIAL | 档位ID（主键） | +| tier_code | VARCHAR(20) | 档位代码（T0-T5, NEW） | +| tier_name | VARCHAR(50) | 档位名称 | +| tier_level | INTEGER | 档位等级（-1=新入职, 0-5=正常档位） | +| min_hours | NUMERIC(10,2) | 最低业绩小时数阈值（>=） | +| max_hours | NUMERIC(10,2) | 最高业绩小时数阈值（<），NULL=无上限 | +| base_deduction | NUMERIC(10,2) | 专业课抽成（元/小时），球房从基础课扣除 | +| bonus_deduction_ratio | NUMERIC(5,4) | 打赏课抽成比例（0-1），球房从附加课扣除 | +| vacation_days | INTEGER | 次月可休假天数 | +| vacation_unlimited | BOOLEAN | 休假自由标记（5档为TRUE） | +| is_new_hire_tier | BOOLEAN | 是否为新入职专用档位 | +| effective_from | DATE | 生效起始日期 | +| effective_to | DATE | 生效截止日期 | + +**档位配置（来自DWS数据库处理需求.md）：** + +| tier_code | tier_name | 业绩阈值 | 专业课抽成 | 打赏课抽成 | 休假 | +|-----------|-----------|----------|-----------|-----------|------| +| T0 | 0档-淘汰压力 | H < 100 | 28元/时 | 50% | 3天 | +| T1 | 1档-及格档 | 100 ≤ H < 130 | 18元/时 | 40% | 4天 | +| T2 | 2档-良好档 | 130 ≤ H < 160 | 15元/时 | 38% | 4天 | +| T3 | 3档-优秀档 | 160 ≤ H < 190 | 13元/时 | 35% | 5天 | +| T4 | 4档-卓越加速档 | 190 ≤ H < 220 | 10元/时 | 33% | 6天 | +| T5 | 5档-冠军加速档 | H ≥ 220 | 8元/时 | 30% | 休假自由 | +| NEW | 新入职档位 | - | 18元/时 | 40% | 4天 | + +**业务规则：** +- 6档绩效（T0-T5），根据有效业绩小时数（基础课+附加课）匹配 +- 新入职档位：月1日0点后入职者首月使用NEW档位，按1档抽成标准 +- 支持按时间生效，历史月份使用历史规则 + +### 1.2 cfg_assistant_level_price - 助教等级定价 + +| 字段 | 类型 | 说明 | +|------|------|------| +| price_id | SERIAL | 定价ID（主键） | +| level_code | INTEGER | 等级代码（8/10/20/30/40） | +| level_name | VARCHAR(20) | 等级名称 | +| base_course_price | NUMERIC(10,2) | 基础课客户支付价格（元/小时） | +| bonus_course_price | NUMERIC(10,2) | 附加课客户支付价格（固定190元） | +| effective_from | DATE | 生效起始日期 | +| effective_to | DATE | 生效截止日期 | + +**等级定价（客户支付价格）：** + +| level_code | level_name | 基础课价格 | 附加课价格 | +|------------|------------|-----------|-----------| +| 8 | 助教管理 | 98元/时 | 190元/时 | +| 10 | 初级 | 98元/时 | 190元/时 | +| 20 | 中级 | 108元/时 | 190元/时 | +| 30 | 高级 | 118元/时 | 190元/时 | +| 40 | 星级 | 138元/时 | 190元/时 | + +**注意：** 此价格为客户支付价格，助教实际收入需减去档位抽成 + +### 1.3 cfg_bonus_rules - 奖金规则配置 + +| 字段 | 类型 | 说明 | +|------|------|------| +| rule_id | SERIAL | 规则ID（主键） | +| rule_type | VARCHAR(20) | 规则类型（SPRINT/TOP_RANK） | +| rule_code | VARCHAR(30) | 规则代码 | +| rule_name | VARCHAR(50) | 规则名称 | +| threshold_hours | NUMERIC(10,2) | 小时数阈值（冲刺奖金） | +| rank_position | INTEGER | 排名位置（Top奖金） | +| bonus_amount | NUMERIC(12,2) | 奖金金额（元） | +| is_cumulative | BOOLEAN | 是否可累计 | +| priority | INTEGER | 优先级 | +| effective_from | DATE | 生效起始日期 | +| effective_to | DATE | 生效截止日期 | + +**业务规则：** +- 冲刺奖金：H>=190得300元，H>=220得800元，不累计取最高档 +- Top3奖金：1st=1000元，2nd=600元，3rd=400元，并列都算 + +### 1.4 cfg_area_category - 台区分类映射 + +| 字段 | 类型 | 说明 | +|------|------|------| +| category_id | SERIAL | 分类ID（主键） | +| source_area_name | VARCHAR(100) | 源区域名称（来自dim_table.site_table_area_name） | +| category_code | VARCHAR(20) | 分类代码 | +| category_name | VARCHAR(50) | 分类名称 | +| match_type | VARCHAR(10) | 匹配类型（exact/like/default） | +| match_priority | INTEGER | 匹配优先级（数字越小优先级越高） | +| is_active | BOOLEAN | 是否启用 | + +**分类代码（基于BD_manual_dim_table.md实际数据）：** + +| category_code | category_name | 匹配规则 | +|---------------|---------------|----------| +| BILLIARD | 普通台球区 | A区, B区, C区 | +| BILLIARD_VIP | VIP台球包厢 | VIP包厢 | +| SNOOKER | 斯诺克区 | 斯诺克区 | +| MAHJONG | 麻将房 | 麻将房 | +| KTV | KTV包间 | K包 | +| SPECIAL | 补时长专用 | 补时长 | +| OTHER | 其他区域 | 默认匹配 | + +### 1.5 cfg_skill_type - 技能课程类型映射 + +| 字段 | 类型 | 说明 | +|------|------|------| +| skill_type_id | SERIAL | 映射ID（主键） | +| skill_id | BIGINT | 技能ID | +| skill_name | VARCHAR(50) | 技能名称 | +| course_type_code | VARCHAR(10) | 课程类型代码 | +| course_type_name | VARCHAR(20) | 课程类型名称 | +| is_active | BOOLEAN | 是否启用 | + +**课程类型：** +- BASE = 基础课/陪打 +- BONUS = 附加课/超休 + +--- + +## 二、助教维度表 + +### 2.1 dws_assistant_daily_detail - 助教日度业绩明细 + +**粒度：** 助教 + 日期 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| site_id | BIGINT | 门店ID | +| tenant_id | BIGINT | 租户ID | +| assistant_id | BIGINT | 助教ID | +| assistant_nickname | VARCHAR(50) | 助教花名 | +| stat_date | DATE | 统计日期 | +| assistant_level_code | INTEGER | 助教等级代码（SCD2 as-of） | +| assistant_level_name | VARCHAR(20) | 助教等级名称 | +| total_service_count | INTEGER | 总服务次数 | +| base_service_count | INTEGER | 基础课服务次数 | +| bonus_service_count | INTEGER | 附加课服务次数 | +| total_seconds | INTEGER | 总计费时长（秒） | +| base_seconds | INTEGER | 基础课计费时长 | +| bonus_seconds | INTEGER | 附加课计费时长 | +| total_hours | NUMERIC(10,2) | 总计费小时数 | +| base_hours | NUMERIC(10,2) | 基础课小时数 | +| bonus_hours | NUMERIC(10,2) | 附加课小时数 | +| total_ledger_amount | NUMERIC(12,2) | 总计费金额 | +| base_ledger_amount | NUMERIC(12,2) | 基础课计费金额 | +| bonus_ledger_amount | NUMERIC(12,2) | 附加课计费金额 | +| unique_customers | INTEGER | 服务客户数（去重） | +| unique_tables | INTEGER | 服务台桌数（去重） | +| trashed_seconds | INTEGER | 被废除的服务时长 | +| trashed_count | INTEGER | 被废除的服务次数 | + +**数据来源：** dwd_assistant_service_log + dwd_assistant_trash_event + +### 2.2 dws_assistant_monthly_summary - 助教月度业绩汇总 + +**粒度：** 助教 + 月份 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| site_id | BIGINT | 门店ID | +| assistant_id | BIGINT | 助教ID | +| stat_month | DATE | 统计月份（月第一天） | +| hire_date | DATE | 入职日期 | +| is_new_hire | BOOLEAN | 是否新入职 | +| work_days | INTEGER | 有服务天数 | +| total_hours | NUMERIC(10,2) | 总计费小时数 | +| base_hours | NUMERIC(10,2) | 基础课小时数 | +| bonus_hours | NUMERIC(10,2) | 附加课小时数 | +| effective_hours | NUMERIC(10,2) | 有效业绩小时数 | +| trashed_hours | NUMERIC(10,2) | 被废除小时数 | +| tier_id | INTEGER | 档位ID | +| tier_code | VARCHAR(20) | 档位代码 | +| tier_name | VARCHAR(50) | 档位名称 | +| rank_by_hours | INTEGER | 月度排名 | +| rank_with_ties | INTEGER | 考虑并列的排名 | + +**业务规则：** +- 有效业绩 = total_hours - trashed_hours +- 新入职判断：入职日期 >= 月1日0点 +- 排名：按effective_hours降序，并列都算 + +### 2.3 dws_assistant_customer_stats - 助教服务客户统计 + +**粒度：** 助教 + 客户 + 统计日期 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| assistant_id | BIGINT | 助教ID | +| member_id | BIGINT | 客户ID | +| stat_date | DATE | 统计基准日期 | +| first_service_date | DATE | 首次服务日期 | +| last_service_date | DATE | 最近服务日期 | +| total_service_count | INTEGER | 累计服务次数 | +| total_service_hours | NUMERIC(10,2) | 累计服务小时数 | +| service_count_7d | INTEGER | 近7天服务次数 | +| service_count_30d | INTEGER | 近30天服务次数 | +| service_count_90d | INTEGER | 近90天服务次数 | +| is_active_7d | BOOLEAN | 近7天是否活跃 | +| is_active_30d | BOOLEAN | 近30天是否活跃 | + +**业务规则：** +- 散客（member_id=0）不进入此表 +- 滚动窗口：7/10/15/30/60/90天 + +### 2.4 dws_assistant_salary_calc - 助教工资计算详情 + +**粒度：** 助教 + 工资月份 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| assistant_id | BIGINT | 助教ID | +| assistant_nickname | VARCHAR(50) | 助教花名 | +| salary_month | DATE | 工资月份（月第一天） | +| assistant_level_code | INTEGER | 助教等级代码（8/10/20/30/40） | +| assistant_level_name | VARCHAR(20) | 助教等级名称 | +| hire_date | DATE | 入职日期 | +| is_new_hire | BOOLEAN | 是否新入职 | +| effective_hours | NUMERIC(10,2) | 有效业绩小时数（基础课+附加课-废除） | +| base_hours | NUMERIC(10,2) | 基础课/专业课小时数 | +| bonus_hours | NUMERIC(10,2) | 附加课/打赏课小时数 | +| tier_id | INTEGER | 档位ID | +| tier_code | VARCHAR(20) | 档位代码（T0-T5/NEW） | +| tier_name | VARCHAR(50) | 档位名称 | +| rank_with_ties | INTEGER | 月度排名（考虑并列，用于Top3奖金） | +| base_course_price | NUMERIC(10,2) | 基础课客户支付价格（98/108/118/138） | +| bonus_course_price | NUMERIC(10,2) | 附加课客户支付价格（固定190） | +| base_deduction | NUMERIC(10,2) | 专业课抽成（元/小时），档位决定 | +| bonus_deduction_ratio | NUMERIC(5,4) | 打赏课抽成比例（0-1），档位决定 | +| base_income | NUMERIC(12,2) | 基础课收入 | +| bonus_income | NUMERIC(12,2) | 附加课收入 | +| total_course_income | NUMERIC(12,2) | 课时收入合计 | +| sprint_bonus | NUMERIC(12,2) | 冲刺奖金（H>=190:300, H>=220:800） | +| top_rank_bonus | NUMERIC(12,2) | Top3排名奖金（1st:1000, 2nd:600, 3rd:400） | +| recharge_commission | NUMERIC(12,2) | 充值提成 | +| other_bonus | NUMERIC(12,2) | 其他奖金（手动调整） | +| total_bonus | NUMERIC(12,2) | 奖金合计 | +| gross_salary | NUMERIC(12,2) | 应发工资 | +| vacation_days | INTEGER | 次月可休假天数 | +| vacation_unlimited | BOOLEAN | 休假自由标记（5档为TRUE） | +| calc_notes | TEXT | 计算备注（异常说明等） | + +**工资计算公式（来自DWS数据库处理需求.md）：** + +``` +基础课收入 = 基础课小时数 × (客户支付价格 - 专业课抽成) +附加课收入 = 附加课小时数 × 190 × (1 - 打赏课抽成比例) +应发工资 = 课时收入 + 奖金 +``` + +**计算示例（中级助教185小时，3档）：** +- 基础课170小时: 170 × (108 - 13) = 16,150元 +- 附加课15小时: 15 × 190 × (1 - 0.35) = 1,852.5元 +- 课时收入: 18,002.5元 +- 冲刺奖金（H≥190未达到）: 0元 +- 应发工资: 18,002.5元 + +--- + +## 三、客户维度表 + +### 3.1 dws_member_consumption_summary - 会员消费汇总 + +**粒度：** 会员 + 统计日期 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| member_id | BIGINT | 会员ID | +| stat_date | DATE | 统计基准日期 | +| first_consume_date | DATE | 首次消费日期 | +| last_consume_date | DATE | 最近消费日期 | +| total_visit_count | INTEGER | 累计到店次数 | +| total_consume_amount | NUMERIC(14,2) | 累计消费金额 | +| visit_count_7d | INTEGER | 近7天到店次数 | +| visit_count_30d | INTEGER | 近30天到店次数 | +| consume_amount_30d | NUMERIC(14,2) | 近30天消费金额 | +| cash_card_balance | NUMERIC(14,2) | 储值卡余额 | +| gift_card_balance | NUMERIC(14,2) | 赠送卡余额 | +| customer_tier | VARCHAR(20) | 客户分层 | + +**客户分层规则：** +- 高价值：90天内消费>=3次 且 消费金额>=1000 +- 中等：30天内有消费 +- 低活跃：90天内有消费但30天内无消费 +- 流失：90天内无消费 + +### 3.2 dws_member_visit_detail - 会员来店明细 + +**粒度：** 会员 + 订单 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| member_id | BIGINT | 会员ID | +| order_settle_id | BIGINT | 结账单ID | +| visit_date | DATE | 来店日期 | +| table_name | VARCHAR(50) | 台桌名称 | +| area_category | VARCHAR(20) | 区域分类 | +| table_fee | NUMERIC(12,2) | 台费 | +| goods_amount | NUMERIC(12,2) | 商品金额 | +| assistant_amount | NUMERIC(12,2) | 助教服务金额 | +| total_consume | NUMERIC(12,2) | 消费总额 | +| actual_pay | NUMERIC(12,2) | 实付金额 | +| assistant_services | JSONB | 助教服务明细（JSON） | + +--- + +## 四、财务维度表 + +### 4.1 dws_finance_daily_summary - 财务日度汇总 + +**粒度：** 日期 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| stat_date | DATE | 统计日期 | +| gross_amount | NUMERIC(14,2) | 发生额合计 | +| table_fee_amount | NUMERIC(14,2) | 台费正价 | +| goods_amount | NUMERIC(14,2) | 商品正价 | +| assistant_pd_amount | NUMERIC(14,2) | 助教基础课正价 | +| assistant_cx_amount | NUMERIC(14,2) | 助教激励课正价 | +| discount_total | NUMERIC(14,2) | 优惠合计 | +| discount_groupbuy | NUMERIC(14,2) | 团购优惠 | +| discount_vip | NUMERIC(14,2) | 会员折扣 | +| discount_gift_card | NUMERIC(14,2) | 赠送卡抵扣 | +| discount_manual | NUMERIC(14,2) | 手动调整 | +| discount_rounding | NUMERIC(14,2) | 抹零 | +| discount_other | NUMERIC(14,2) | 其他优惠（手动调整拆分） | +| confirmed_income | NUMERIC(14,2) | 确认收入 | +| cash_inflow_total | NUMERIC(14,2) | 现金流入合计 | +| cash_pay_amount | NUMERIC(14,2) | 收银实付 | +| groupbuy_pay_amount | NUMERIC(14,2) | 团购支付金额 | +| platform_settlement_amount | NUMERIC(14,2) | 平台回款金额 | +| platform_fee_amount | NUMERIC(14,2) | 平台服务费+佣金 | +| recharge_cash_inflow | NUMERIC(14,2) | 充值现金流入 | +| card_consume_total | NUMERIC(14,2) | 卡消费合计 | +| cash_card_consume | NUMERIC(14,2) | 储值卡消费 | +| gift_card_consume | NUMERIC(14,2) | 赠送卡消费 | +| cash_outflow_total | NUMERIC(14,2) | 现金流出合计 | +| cash_balance_change | NUMERIC(14,2) | 现金结余 | +| recharge_count | INTEGER | 充值笔数 | +| recharge_total | NUMERIC(14,2) | 充值总额 | +| recharge_cash | NUMERIC(14,2) | 充值现金部分 | +| recharge_gift | NUMERIC(14,2) | 充值赠送部分 | +| first_recharge_count | INTEGER | 首充笔数 | +| renewal_count | INTEGER | 续充笔数 | +| order_count | INTEGER | 结账单数 | +| member_order_count | INTEGER | 会员订单数 | +| guest_order_count | INTEGER | 散客订单数 | +| avg_order_amount | NUMERIC(12,2) | 平均客单价 | + +**计算公式：** +- 发生额 = table_charge_money + goods_money + assistant_pd_money + assistant_cx_money +- 团购支付金额 = pl_coupon_sale_amount > 0 ? pl_coupon_sale_amount : groupbuy_redemption.ledger_unit_price +- 团购优惠 = coupon_amount - 团购支付金额 +- 优惠合计 = 团购优惠 + 会员折扣 + 赠送卡抵扣 + 手动调整 + 抹零 +- 其他优惠 = adjust_amount - 大客户优惠（不足0按0处理） +- 确认收入 = 发生额 - 优惠合计 +- 平台回款金额 = dws_platform_settlement.settlement_amount（若无导入，则使用团购支付金额） +- 平台服务费 = commission_amount + service_fee +- 现金流入合计 = 收银实付 + 平台回款金额 + 充值现金流入 +- 现金流出合计 = 支出汇总 + 平台服务费 +- 现金结余 = 现金流入合计 - 现金流出合计 + +**财务指标数据来源矩阵（字段 → 来源 → 口径）** +| 字段 | 来源表 | 口径说明 | +|------|--------|----------| +| gross_amount | dwd_settlement_head | table_charge_money + goods_money + assistant_pd_money + assistant_cx_money | +| discount_groupbuy | dwd_settlement_head + dwd_groupbuy_redemption | coupon_amount - 团购支付金额 | +| discount_vip | dwd_settlement_head | member_discount_amount | +| discount_gift_card | dwd_settlement_head | gift_card_amount | +| discount_manual | dwd_settlement_head | adjust_amount（手动调整总额） | +| discount_rounding | dwd_settlement_head | rounding_amount | +| discount_other | dwd_settlement_head | adjust_amount - 大客户优惠（配置映射） | +| confirmed_income | dwd_settlement_head | gross_amount - discount_total | +| cash_pay_amount | dwd_settlement_head | pay_amount（收银实付） | +| groupbuy_pay_amount | dwd_settlement_head + dwd_groupbuy_redemption | pl_coupon_sale_amount 或 ledger_unit_price | +| platform_settlement_amount | dws_platform_settlement | settlement_amount（Excel导入） | +| platform_fee_amount | dws_platform_settlement | commission_amount + service_fee | +| recharge_cash_inflow | dwd_recharge_order | pay_money（现金充值） | +| cash_inflow_total | dwd_settlement_head + dws_platform_settlement + dwd_recharge_order | 收银实付 + 平台回款 + 充值现金 | +| cash_outflow_total | dws_finance_expense_summary + dws_platform_settlement | 支出汇总 + 平台服务费 | +| cash_balance_change | dws_finance_daily_summary | cash_inflow_total - cash_outflow_total | + +### 4.2 dws_finance_recharge_summary - 充值统计 + +**粒度：** 日期 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| stat_date | DATE | 统计日期 | +| recharge_count | INTEGER | 充值笔数 | +| recharge_total | NUMERIC(14,2) | 充值总额（含赠送） | +| recharge_cash | NUMERIC(14,2) | 现金充值金额 | +| recharge_gift | NUMERIC(14,2) | 赠送金额 | +| first_recharge_count | INTEGER | 首充笔数 | +| first_recharge_cash | NUMERIC(14,2) | 首充现金 | +| renewal_count | INTEGER | 续充笔数 | +| renewal_cash | NUMERIC(14,2) | 续充现金 | +| cash_card_balance | NUMERIC(14,2) | 储值卡余额 | +| gift_card_balance | NUMERIC(14,2) | 赠送卡余额 | + +**数据来源：** dwd_recharge_order（is_first字段区分首充/续充） + +### 4.3 dws_finance_income_structure - 收入结构分析 + +**粒度：** 日期 + 结构类型 + 分类 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| stat_date | DATE | 统计日期 | +| structure_type | VARCHAR(20) | 结构类型（INCOME_TYPE/AREA） | +| category_code | VARCHAR(30) | 分类代码 | +| category_name | VARCHAR(50) | 分类名称 | +| income_amount | NUMERIC(14,2) | 收入金额 | +| income_ratio | NUMERIC(5,4) | 收入占比 | +| order_count | INTEGER | 订单数 | +| duration_minutes | INTEGER | 时长（分钟） | + +**结构类型说明：** + +1. **INCOME_TYPE（按收入类型）：** + - TABLE_FEE = 台费收入 + - GOODS = 商品收入 + - ASSISTANT_BASE = 助教基础课 + - ASSISTANT_BONUS = 助教附加课 + +2. **AREA（按区域）：** + - 使用cfg_area_category映射（BILLIARD/BILLIARD_VIP/SNOOKER/MAHJONG/KTV/OTHER） + +**数据来源：** dwd_settlement_head, dwd_table_fee_log, dwd_assistant_service_log + +### 4.4 dws_finance_discount_detail - 优惠明细 + +**粒度：** 日期 + 优惠类型 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| stat_date | DATE | 统计日期 | +| discount_type_code | VARCHAR(30) | 优惠类型代码 | +| discount_type_name | VARCHAR(50) | 优惠类型名称 | +| discount_amount | NUMERIC(14,2) | 优惠金额 | +| discount_ratio | NUMERIC(5,4) | 优惠占比（占总优惠） | +| usage_count | INTEGER | 使用次数 | +| affected_orders | INTEGER | 影响订单数 | + +**优惠类型：** +- GROUPBUY = 团购优惠（coupon_amount - 团购实付金额） +- VIP = 会员折扣（member_discount_amount） +- GIFT_CARD = 赠送卡抵扣（gift_card_amount） +- ROUNDING = 抹零（rounding_amount） +- BIG_CUSTOMER = 大客户优惠（基于配置映射的手动调整） +- OTHER = 其他优惠（手动调整中除大客户外部分） + +**数据来源：** dwd_settlement_head, dwd_groupbuy_redemption + +### 4.5 dws_finance_expense_summary - 支出结构（Excel导入） + +**粒度：** 月份 + 支出类型 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| expense_month | DATE | 支出月份 | +| expense_type_code | VARCHAR(30) | 支出类型代码 | +| expense_type_name | VARCHAR(50) | 支出类型名称 | +| expense_category | VARCHAR(20) | 支出大类 | +| expense_amount | NUMERIC(14,2) | 支出金额 | +| import_batch_no | VARCHAR(50) | 导入批次号 | + +**支出类型：** +- RENT = 房租 +- UTILITY = 水电费 +- PROPERTY = 物业费 +- SALARY = 工资 +- REIMBURSE = 报销 +- PLATFORM_FEE = 平台服务费 +- OTHER = 其他 + +### 4.6 dws_platform_settlement - 平台回款（Excel导入） + +**粒度：** 回款日期 + 平台 + 订单 + +| 字段 | 类型 | 说明 | +|------|------|------| +| id | BIGSERIAL | 主键 | +| settlement_date | DATE | 回款日期 | +| platform_type | VARCHAR(30) | 平台类型 | +| platform_name | VARCHAR(50) | 平台名称 | +| platform_order_no | VARCHAR(100) | 平台订单号 | +| order_settle_id | BIGINT | 关联的结账单ID | +| settlement_amount | NUMERIC(14,2) | 回款金额 | +| commission_amount | NUMERIC(14,2) | 佣金 | +| service_fee | NUMERIC(14,2) | 服务费 | +| gross_amount | NUMERIC(14,2) | 订单原始金额 | +| import_batch_no | VARCHAR(50) | 导入批次号 | + +--- + +## 五、时间分层机制 + +### 5.1 时间口径定义 + +| 时间窗口 | 说明 | 边界规则 | +|----------|------|----------| +| 本周 | 从本周一到今天 | 周起始日为周一 | +| 上周 | 上周一到上周日 | 完整7天 | +| 本月 | 从月1日到今天 | 月第一天0点起 | +| 上月 | 上月完整月份 | 完整自然月 | +| 前3个月不含本月 | 三个月前月初到上月末 | 不含当前月 | +| 前3个月含本月 | 两个月前月初到今天 | 含当前月 | +| 本季度 | 季度第一月1日到今天 | 季度起始 | +| 上季度 | 上季度完整三个月 | 完整自然季 | +| 最近半年 | 往前6个月（不含本月） | 不含当前月 | + +### 5.2 滚动窗口 + +支持以下滚动窗口统计： +- 近7天 +- 近10天 +- 近15天 +- 近30天 +- 近60天 +- 近90天 + +### 5.3 环比计算 + +环比规则：对比上一个等长区间 +- 如查询1月1日-1月15日，环比为12月17日-12月31日 + +--- + +## 六、数据更新策略 + +| 表类型 | 更新频率 | 幂等方式 | +|--------|----------|----------| +| 日度明细表 | 每小时 | delete-before-insert（按日期窗口） | +| 日度汇总表 | 每小时 | delete-before-insert（按日期） | +| 月度汇总表 | 每日 | delete-before-insert（按月份） | +| 客户统计表 | 每日 | delete-before-insert（按统计日期） | +| Excel导入表 | 手动 | 按import_batch_no去重 | diff --git a/etl_billiards/docs/index_tables.md b/etl_billiards/docs/index_tables.md new file mode 100644 index 0000000..ca102d8 --- /dev/null +++ b/etl_billiards/docs/index_tables.md @@ -0,0 +1,494 @@ +## 1. 客户召回表 + +| 客户姓名 | 召回指数 | +|----------|----------| +| 陈腾鑫 | 10.00 | +| 章先生 | 10.00 | +| 孙总 | 10.00 | +| 梅 | 10.00 | +| 胡先生 | 10.00 | +| 黄先生 | 9.63 | +| 小熊 | 9.52 | +| 周先生 | 9.41 | +| 李先生 | 9.39 | +| 王 | 9.27 | +| 张无忌 | 9.20 | +| 黄先生 | 8.96 | +| 陈德韩 | 8.94 | +| 胡总 | 8.93 | +| T | 8.89 | +| 候 | 8.88 | +| 孙先生 | 8.87 | +| 王先生 | 8.86 | +| 清 | 8.86 | +| amy | 8.84 | +| 林先生 | 8.84 | +| 张先生 | 8.79 | +| 刘先生 | 8.79 | +| 黄国磊 | 8.79 | +| 游 | 8.79 | +| 陈先生 | 8.79 | +| 陈 | 8.79 | +| 大G | 8.79 | +| 李先生 | 8.79 | +| 孙启明 | 8.79 | +| 陈先生 | 8.79 | +| 罗先生 | 8.79 | +| 刘哥 | 8.79 | +| 杨 | 8.79 | +| 枫先生 | 8.79 | +| 老宋 | 8.79 | +| 黄先生 | 8.79 | +| 刘女士 | 8.79 | +| 彭先生 | 8.79 | +| 李 | 8.79 | +| 桂先生 | 8.79 | +| 王先生 | 8.79 | +| 潘先生 | 8.79 | +| 方先生 | 8.79 | +| 郑先生 | 8.79 | +| 阿亮 | 8.79 | +| 孟紫龙 | 8.79 | +| 林总 | 8.78 | +| 林志铭 | 8.64 | +| 罗超 | 8.63 | +| 张丹逸 | 8.52 | +| 谢俊 | 8.07 | +| 王龙 | 7.80 | +| 唐先生 | 7.79 | +| 周周 | 7.47 | +| 曾巧明 | 6.83 | +| 昌哥 | 6.17 | +| 江先生 | 5.84 | +| 袁 | 5.24 | +| 蔡总 | 4.73 | +| 胡先生 | 4.51 | +| 陈先生 | 4.45 | +| 明哥 | 3.92 | +| 公孙先生 | 3.57 | +| 曾先生 | 3.47 | +| 黄生 | 3.46 | +| 葛先生 | 3.35 | +| 轩哥 | 3.32 | +| 张先生 | 2.73 | +| 叶先生 | 2.61 | +| 小燕 | 2.39 | +| 罗先生 | 2.38 | +| 李先生 | 2.23 | +| 陈淑涛 | 2.23 | +| 肖先生 | 2.23 | +| 范先生 | 2.14 | +| 常总 | 1.47 | +| 董贝 | 1.04 | +| 陈小姐 | 1.04 | +| 林先生 | 0.90 | +| 柳先生 | 0.61 | +| 林先生 | 0.20 | +| 潘先生 | 0.20 | +| 曾丹烨 | 0.07 | +| 魏先生 | 0.00 | +| 艾宇民 | 0.00 | +| 吴生 | 0.00 | +| 卢广贤 | 0.00 | +| 陈泽斌 | 0.00 | +| 李先生 | 0.00 | + +共 90 条记录 + +## 2. 助教客户关系表 + +| 助教花名 | 客户姓名 | 关系指数 | +|----------|----------|----------| +| 卡顿 | 葛先生 | 10.00 | +| 小燕 | 葛先生 | 10.00 | +| 七七 | 轩哥 | 10.00 | +| 佳怡 | 罗先生 | 10.00 | +| 璇子 | 轩哥 | 10.00 | +| 阿清 | 张先生 | 10.00 | +| 璇子 | 江先生 | 10.00 | +| CC | 周周 | 10.00 | +| 周周 | 周周 | 10.00 | +| 小燕 | 小燕 | 10.00 | +| 卡顿 | 小燕 | 10.00 | +| 姜姜 | 张先生 | 10.00 | +| 小侯 | 张先生 | 10.00 | +| 渔渔 | 张先生 | 10.00 | +| 欣欣 | 张先生 | 10.00 | +| 千千 | 张先生 | 10.00 | +| 小A | 张先生 | 10.00 | +| 甜甜 | 张先生 | 10.00 | +| 小A | 周先生 | 10.00 | +| 欣欣 | 周先生 | 10.00 | +| 千千 | 周先生 | 10.00 | +| 甜甜 | 周先生 | 10.00 | +| 涛涛 | 蔡总 | 10.00 | +| 婉婉 | 吴先生 | 10.00 | +| 千千 | 梅 | 10.00 | +| 甜甜 | 梅 | 10.00 | +| 小A | 梅 | 10.00 | +| 欣欣 | 梅 | 10.00 | +| 球球 | 周周 | 10.00 | +| 涛涛 | 轩哥 | 10.00 | +| 小不点 | 周周 | 10.00 | +| 小柔 | 蔡总 | 10.00 | +| 年糕 | 葛先生 | 10.00 | +| 佳怡 | 陈腾鑫 | 10.00 | +| 小不点 | 罗先生 | 10.00 | +| 球球 | 罗先生 | 10.00 | +| 小柔 | 轩哥 | 10.00 | +| 阿清 | 梅 | 10.00 | +| 阿清 | 胡先生 | 10.00 | +| 佳怡 | 陈先生 | 10.00 | +| 小不点 | 轩哥 | 10.00 | +| 佳怡 | 小熊 | 10.00 | +| 球球 | 轩哥 | 10.00 | +| 阿清 | 孙总 | 10.00 | +| CC | 罗先生 | 10.00 | +| 周周 | 罗先生 | 10.00 | +| 小柔 | 明哥 | 9.96 | +| 渔渔 | 李先生 | 9.88 | +| 姜姜 | 李先生 | 9.88 | +| 小侯 | 李先生 | 9.88 | +| 年糕 | 常总 | 9.85 | +| 婉婉 | 明哥 | 9.61 | +| 乔西 | 陈先生 | 9.59 | +| 璇子 | 蔡总 | 9.46 | +| CC | 常总 | 9.40 | +| 周周 | 常总 | 9.40 | +| 七七 | 蔡总 | 9.24 | +| 甜甜 | 孙总 | 9.24 | +| 小A | 孙总 | 9.24 | +| 欣欣 | 孙总 | 9.24 | +| 千千 | 孙总 | 9.24 | +| 七七 | 胡先生 | 9.20 | +| 千千 | 小熊 | 9.02 | +| 甜甜 | 小熊 | 9.02 | +| 欣欣 | 小熊 | 9.02 | +| 小A | 小熊 | 9.02 | +| 佳怡 | 胡先生 | 9.02 | +| 涛涛 | 小燕 | 8.66 | +| 阿清 | 轩哥 | 8.53 | +| 年糕 | 叶先生 | 8.51 | +| 小不点 | 张先生 | 8.39 | +| 球球 | 张先生 | 8.39 | +| 阿清 | 葛先生 | 8.36 | +| 周周 | 张先生 | 8.30 | +| CC | 张先生 | 8.30 | +| 甜甜 | 胡先生 | 8.04 | +| 千千 | 胡先生 | 8.04 | +| 欣欣 | 胡先生 | 8.04 | +| 小A | 胡先生 | 8.04 | +| 小不点 | 小熊 | 7.88 | +| 球球 | 小熊 | 7.88 | +| 小侯 | 胡先生 | 7.86 | +| 姜姜 | 胡先生 | 7.86 | +| 渔渔 | 胡先生 | 7.86 | +| 乔西 | 罗先生 | 7.86 | +| 小不点 | 胡先生 | 7.65 | +| 球球 | 胡先生 | 7.65 | +| 球球 | 孙总 | 7.56 | +| 小不点 | 孙总 | 7.56 | +| 璇子 | 孙总 | 7.46 | +| 阿清 | 清 | 7.35 | +| 小A | 小燕 | 7.09 | +| 甜甜 | 小燕 | 7.09 | +| 欣欣 | 小燕 | 7.09 | +| 千千 | 小燕 | 7.09 | +| 甜甜 | 公孙先生 | 7.07 | +| 千千 | 公孙先生 | 7.07 | +| 欣欣 | 公孙先生 | 7.07 | +| 小A | 公孙先生 | 7.07 | +| 婉婉 | 孙总 | 7.03 | +| 菲菲 | 陈腾鑫 | 6.92 | +| 橙子 | 陈腾鑫 | 6.92 | +| 希希 | 陈腾鑫 | 6.92 | +| 婉婉 | 章先生 | 6.91 | +| 婉婉 | 公孙先生 | 6.86 | +| CC | 林先生 | 6.77 | +| 周周 | 林先生 | 6.77 | +| 阿清 | 小燕 | 6.76 | +| 苏苏 | 蔡总 | 6.64 | +| 七七 | 小燕 | 6.60 | +| 小不点 | 江先生 | 6.59 | +| 球球 | 江先生 | 6.59 | +| 涛涛 | 罗先生 | 6.52 | +| 凤梨 | 葛先生 | 6.51 | +| 佳怡 | 轩哥 | 6.49 | +| 年糕 | 轩哥 | 6.44 | +| 年糕 | 小燕 | 6.43 | +| CC | 轩哥 | 6.26 | +| 周周 | 轩哥 | 6.26 | +| yy | 公孙先生 | 6.13 | +| 阿清 | 陈腾鑫 | 6.04 | +| 佳怡 | 周周 | 6.03 | +| 七七 | 江先生 | 5.93 | +| CC | 林先生 | 5.87 | +| 周周 | 林先生 | 5.87 | +| 年糕 | 王 | 5.76 | +| 年糕 | 李先生 | 5.72 | +| 七七 | 孙总 | 5.69 | +| 苏苏 | 黄先生 | 5.66 | +| 婉婉 | 叶先生 | 5.55 | +| 涛涛 | 叶先生 | 5.55 | +| 凤梨 | 叶先生 | 5.54 | +| 小A | 黄先生 | 5.53 | +| 甜甜 | 黄先生 | 5.53 | +| 千千 | 黄先生 | 5.53 | +| 欣欣 | 黄先生 | 5.53 | +| yy | 叶先生 | 5.53 | +| 苏苏 | 罗先生 | 5.48 | +| 小侯 | 葛先生 | 5.47 | +| 渔渔 | 葛先生 | 5.47 | +| 姜姜 | 葛先生 | 5.47 | +| 佳怡 | 林志铭 | 5.45 | +| 婉婉 | 葛先生 | 5.37 | +| CC | 小熊 | 5.29 | +| 周周 | 小熊 | 5.29 | +| 涛涛 | 孙总 | 5.20 | +| 小敌 | 李先生 | 5.09 | +| 吱吱 | 李先生 | 5.09 | +| 周周 | 葛先生 | 5.08 | +| CC | 葛先生 | 5.08 | +| 甜甜 | 蔡总 | 5.04 | +| 千千 | 蔡总 | 5.04 | +| 欣欣 | 蔡总 | 5.04 | +| 小A | 蔡总 | 5.04 | +| 婉婉 | 轩哥 | 5.03 | +| 年糕 | 胡先生 | 5.02 | +| 吱吱 | 葛先生 | 4.88 | +| 小敌 | 葛先生 | 4.88 | +| 婉婉 | 王 | 4.87 | +| yy | 张先生 | 4.66 | +| 璇子 | 罗先生 | 4.65 | +| yy | 葛先生 | 4.59 | +| 苏苏 | 柳先生 | 4.58 | +| 乔西 | 蔡总 | 4.50 | +| 七七 | 张先生 | 4.36 | +| 乔西 | 葛先生 | 4.33 | +| 乔西 | 小熊 | 4.33 | +| 周周 | 江先生 | 4.32 | +| CC | 江先生 | 4.32 | +| Amy | 轩哥 | 4.31 | +| 年糕 | 罗超 | 4.20 | +| yy | 林志铭 | 4.19 | +| 年糕 | 艾宇民 | 4.16 | +| 阿清 | 黄先生 | 4.14 | +| 七七 | 罗超 | 4.12 | +| 年糕 | 范先生 | 4.08 | +| 凤梨 | 林先生 | 4.07 | +| 璇子 | 张先生 | 4.06 | +| 球球 | 常总 | 4.05 | +| 小不点 | 常总 | 4.05 | +| yy | 孙总 | 3.99 | +| 七七 | 葛先生 | 3.93 | +| 乔西 | 轩哥 | 3.90 | +| 年糕 | 小熊 | 3.85 | +| 千千 | 李先生 | 3.73 | +| 欣欣 | 李先生 | 3.73 | +| 小A | 李先生 | 3.73 | +| 甜甜 | 李先生 | 3.73 | +| 姜姜 | 轩哥 | 3.62 | +| 渔渔 | 轩哥 | 3.62 | +| 小侯 | 轩哥 | 3.62 | +| 迟迟 | 轩哥 | 3.60 | +| 泡芙 | 轩哥 | 3.60 | +| 小琳 | 轩哥 | 3.60 | +| 七七 | 罗先生 | 3.57 | +| 年糕 | 胡总 | 3.47 | +| 欣欣 | 葛先生 | 3.43 | +| 甜甜 | 葛先生 | 3.43 | +| 千千 | 葛先生 | 3.43 | +| 小A | 葛先生 | 3.43 | +| 七七 | 林总 | 3.43 | +| 乔西 | 陈德韩 | 3.38 | +| 泡芙 | 林总 | 3.31 | +| 迟迟 | 林总 | 3.31 | +| 小琳 | 林总 | 3.31 | +| 涛涛 | 葛先生 | 3.27 | +| 阿清 | 罗先生 | 3.16 | +| 璇子 | 周周 | 3.16 | +| 阿清 | 王先生 | 3.14 | +| 小柳 | 轩哥 | 3.06 | +| 迟迟 | 陈腾鑫 | 3.04 | +| 小琳 | 陈腾鑫 | 3.04 | +| 泡芙 | 陈腾鑫 | 3.04 | +| 瑶瑶 | 蔡总 | 2.92 | +| 图图 | 蔡总 | 2.92 | +| 小A | 轩哥 | 2.91 | +| 千千 | 轩哥 | 2.91 | +| 欣欣 | 轩哥 | 2.91 | +| 甜甜 | 轩哥 | 2.91 | +| 年糕 | 罗先生 | 2.84 | +| 小不点 | 黄先生 | 2.73 | +| 球球 | 黄先生 | 2.73 | +| 渔渔 | 梅 | 2.72 | +| 姜姜 | 梅 | 2.72 | +| 小侯 | 梅 | 2.72 | +| 欣欣 | 陈先生 | 2.68 | +| 千千 | 陈先生 | 2.68 | +| 小A | 陈先生 | 2.68 | +| 甜甜 | 陈先生 | 2.68 | +| 婉婉 | 江先生 | 2.67 | +| 千千 | 枫先生 | 2.67 | +| 欣欣 | 枫先生 | 2.67 | +| 小A | 枫先生 | 2.67 | +| 甜甜 | 枫先生 | 2.67 | +| 阿清 | 枫先生 | 2.67 | +| 乔西 | 张无忌 | 2.55 | +| 甜甜 | 范先生 | 2.51 | +| 千千 | 范先生 | 2.51 | +| 小A | 范先生 | 2.51 | +| 欣欣 | 范先生 | 2.51 | +| 七七 | 林先生 | 2.45 | +| CC | T | 2.36 | +| 周周 | T | 2.36 | +| 苏苏 | 周周 | 2.36 | +| 小侯 | 周先生 | 2.28 | +| 渔渔 | 周先生 | 2.28 | +| 姜姜 | 周先生 | 2.28 | +| 涛涛 | 胡总 | 2.28 | +| 苏苏 | 林先生 | 2.14 | +| 渔渔 | 彭先生 | 2.07 | +| 小侯 | 彭先生 | 2.07 | +| 姜姜 | 彭先生 | 2.07 | +| 小侯 | 清 | 2.03 | +| 甜甜 | 清 | 2.03 | +| 小A | 清 | 2.03 | +| 欣欣 | 清 | 2.03 | +| 千千 | 清 | 2.03 | +| 渔渔 | 清 | 2.03 | +| 姜姜 | 清 | 2.03 | +| 苏苏 | 张先生 | 1.94 | +| 千千 | 林总 | 1.88 | +| 甜甜 | 林总 | 1.88 | +| 欣欣 | 林总 | 1.88 | +| 小A | 林总 | 1.88 | +| 甜甜 | 陈腾鑫 | 1.82 | +| 欣欣 | 陈腾鑫 | 1.82 | +| 千千 | 陈腾鑫 | 1.82 | +| 小A | 陈腾鑫 | 1.82 | +| 佳怡 | 彭先生 | 1.80 | +| 婉婉 | 周先生 | 1.77 | +| 苏苏 | 周先生 | 1.68 | +| CC | 昌哥 | 1.64 | +| 周周 | 昌哥 | 1.64 | +| 球球 | 蔡总 | 1.57 | +| 小不点 | 蔡总 | 1.57 | +| 苏苏 | 李先生 | 1.53 | +| 吱吱 | 李先生 | 1.50 | +| 小敌 | 李先生 | 1.50 | +| 婉婉 | 刘哥 | 1.46 | +| CC | 林总 | 1.39 | +| 周周 | 林总 | 1.39 | +| 小不点 | T | 1.38 | +| 球球 | T | 1.38 | +| 悠悠 | 张先生 | 1.38 | +| 布丁 | 张先生 | 1.38 | +| 小怡 | 周先生 | 1.37 | +| 雯雯 | 周先生 | 1.37 | +| 素素 | 周先生 | 1.37 | +| 嘉嘉 | 轩哥 | 1.31 | +| 小柔 | 葛先生 | 1.30 | +| 乔西 | 张先生 | 1.29 | +| 小不点 | 候 | 1.23 | +| 球球 | 候 | 1.23 | +| 嘉嘉 | 罗先生 | 1.22 | +| 小侯 | T | 1.19 | +| 渔渔 | T | 1.19 | +| 姜姜 | T | 1.19 | +| 小侯 | 黄先生 | 1.19 | +| 小敌 | 林先生 | 1.19 | +| 姜姜 | 黄先生 | 1.19 | +| 吱吱 | 林先生 | 1.19 | +| 渔渔 | 黄先生 | 1.19 | +| 球球 | 葛先生 | 1.16 | +| 小不点 | 葛先生 | 1.16 | +| Amy | amy | 1.15 | +| 乔西 | T | 1.12 | +| 球球 | 老宋 | 1.10 | +| 小不点 | 老宋 | 1.10 | +| 乔西 | 林先生 | 1.01 | +| 素素 | 张先生 | 0.98 | +| 小怡 | 张先生 | 0.98 | +| 雯雯 | 张先生 | 0.98 | +| 佳怡 | T | 0.96 | +| 年糕 | 张先生 | 0.94 | +| 小侯 | 陈腾鑫 | 0.88 | +| 渔渔 | 陈腾鑫 | 0.88 | +| 姜姜 | 陈腾鑫 | 0.88 | +| 阿清 | 李先生 | 0.85 | +| 球球 | 林总 | 0.83 | +| 小不点 | 林总 | 0.83 | +| 婉婉 | 常总 | 0.77 | +| 小侯 | 艾宇民 | 0.76 | +| 姜姜 | 艾宇民 | 0.76 | +| 渔渔 | 艾宇民 | 0.76 | +| 小敌 | 郑先生 | 0.74 | +| 吱吱 | 郑先生 | 0.74 | +| 千千 | 罗先生 | 0.72 | +| 甜甜 | 罗先生 | 0.72 | +| 小A | 罗先生 | 0.72 | +| 欣欣 | 罗先生 | 0.72 | +| 球球 | 小燕 | 0.67 | +| 小不点 | 小燕 | 0.67 | +| 年糕 | 周先生 | 0.65 | +| 卡顿 | 罗先生 | 0.62 | +| 小燕 | 罗先生 | 0.62 | +| 小敌 | 刘哥 | 0.60 | +| 吱吱 | 刘哥 | 0.60 | +| 小柔 | 孟紫龙 | 0.56 | +| 阿清 | 候 | 0.54 | +| 乔西 | 候 | 0.49 | +| 小敌 | 张先生 | 0.46 | +| 甜甜 | T | 0.46 | +| 小A | T | 0.46 | +| 欣欣 | T | 0.46 | +| 千千 | T | 0.46 | +| 吱吱 | 张先生 | 0.46 | +| 小A | 游 | 0.38 | +| 千千 | 游 | 0.38 | +| 甜甜 | 游 | 0.38 | +| 欣欣 | 游 | 0.38 | +| 苏苏 | 葛先生 | 0.34 | +| 渔渔 | 候 | 0.32 | +| 小侯 | 候 | 0.32 | +| 姜姜 | 候 | 0.32 | +| 苏苏 | T | 0.31 | +| 婉婉 | 罗先生 | 0.26 | +| 涛涛 | 候 | 0.24 | +| 苏苏 | 候 | 0.23 | +| 阿清 | 常总 | 0.23 | +| 小不点 | 李先生 | 0.22 | +| 球球 | 李先生 | 0.22 | +| 小柔 | T | 0.19 | +| 年糕 | 潘先生 | 0.19 | +| 婉婉 | 候 | 0.18 | +| 小柔 | 罗先生 | 0.17 | +| 梦梦 | 葛先生 | 0.14 | +| 欣怡 | 葛先生 | 0.14 | +| 大姚 | 葛先生 | 0.14 | +| 椰子 | 葛先生 | 0.14 | +| 璇子 | 林先生 | 0.11 | +| 年糕 | 明哥 | 0.09 | +| 涛涛 | 张先生 | 0.08 | +| 周周 | 大G | 0.02 | +| 佳怡 | 大G | 0.02 | +| CC | 大G | 0.02 | +| 周周 | 明哥 | 0.00 | +| Amy | 明哥 | 0.00 | +| 小怡 | 叶先生 | 0.00 | +| 乔西 | 林先生 | 0.00 | +| 素素 | 叶先生 | 0.00 | +| 雯雯 | 叶先生 | 0.00 | +| 梦梦 | 蔡总 | 0.00 | +| 欣怡 | 蔡总 | 0.00 | +| 椰子 | 蔡总 | 0.00 | +| 大姚 | 蔡总 | 0.00 | +| 周周 | 游 | 0.00 | +| 小柔 | 昌哥 | 0.00 | +| CC | 游 | 0.00 | +| 佳怡 | 游 | 0.00 | +| CC | 明哥 | 0.00 | +| 小柔 | 江先生 | 0.00 | + +共 391 条记录 \ No newline at end of file diff --git a/etl_billiards/docs/index_tables_output.txt b/etl_billiards/docs/index_tables_output.txt new file mode 100644 index 0000000000000000000000000000000000000000..66da8ac5031a1636c6bd506692d477e11d241952 GIT binary patch literal 46056 zcmd5_eQ2F$6~9n%wph0^9Aew;Zc+r%_TGG_Y}7iLQkYrV+BQj-v|T@zg)C{-)OFoj zt7T)1>EONbV}*$_*s!DMic_alS>tL~9pWnL{HGNqidYaQPKDL?4{jt`#Nvmn*ZU zTV}C&Kj+4dZk^gxPWKo8d9d!^nTGEre_Hpo?L@^#uFwx*PfvYd!~53$aB4-}{_-ff zKtr`uNo7(=e`t-<`<|aTwC1LoWsQa7*Ni+@F0w+ol(&yp`ilMk^H;d9)>^hwa%lK3 ztD9dV)$f#}9i`S;j`ocIYH0bV^W>2I?-#hiYqXbIX4e=Pbj{MSaQx~Kjb`u~ zEZ0S~4-M$oV3}Ooucw4p*jySd?J303i@C`%J(YZPSWgG9ac)54Ls>66gH%^1%kRS%Pw9Rn?J*Lg#I^=6I%aCRwBC~ zOM7+GbRmn)+lL1~=j_*OxzIuyx^XMwCjWaLEhh_cg4W1e;Z*XCGgj>yuif7>GHIj4 z5|ipVdbINWF^j!c9vwVu<%eA=U+#PI4(l!~5k)w9bHCk-zLbpbveI&7YVj898po%) z`hGcJ)6#|GFYYY&93Hl4*r{aYur}-IOtTisz9`DB-FK{zS8ufZjuJ=NIr~38{Kg&~ zKO{FlH}LI=JqHe}_940XW4lIYZri8ZM;{{B$&Yjn=$ft(c&CLi9c(J@*> z?1k!#mpAM7(H3Yqk<4r=eX02v*D*^AXT{UWLmM@*h1RHsef+EMyE`@fXrJ8&>)89# z-bC}qq?>6SWWCt>vlg2DcYV2gb8z^zA&rjl8k~RZsm*NKI{BMIecOxo!E5v6K{v}9NyIfwZS!)I?o}A=}&x^%D2af&-&!d`303r z$6KOOm!D)$gkmw>3nLkQruUx@qg z*ztQe=XA(vUmjZ3Gydw(_Q!~AlOYx{q^?Yh0$wdF4_jCc0mcK+BISvC9H3uArq zZ1217%&sked13U7pBR)iv%kF%ZmRv{-cRM!`Cb@tDs(63S^Gos_P{{$_P{{W@W5Em z|JBj=Mxv9hAC{HJnX!3&GZA0apY-l%zM9EmAbHs5Q;;<52yx4>uaE4)j&ObLo4pTj z%K9exi~I*^Z1er?+%Zb%*;wE)sx4-6!858XxehTX~$g6{kyYvow0G*c>B45U>a(m-@j zEztb5VU`Z0xgK;NnS0ET#w$z@`-{aN@`oN6$i}CYr*br-D$|gtYSQ*H38O2V`J)Qc z0|Ql<9vCPFYR_7jSA5cnGF*$y<@};HdScs;BE3WHi80a9o@y5Lr%^vlmy_^pw4A}l zt=uAwws1;=tlR?wbwDkgl34^)ap-(5OG7Z?HHQZRsyaL{Q1$15fh4ZWdPEw0o`&eq zWebc^39|=Ct2{7}%snvTn(KjpayN^_MH;A@B|0<{8Z^b+5vE>H?xxERdH%gUOkbcV ztDROMI%Fmag6}&_)M)t9T^$KtR4qYk{OyGUI}L zt_KG4haMPlf9Qh{?}}-CE>FesBp9+k4-8awXnDnSu>GOx$pZsbfgTvB>d;lp<#dXX zDoy4U^6M5lI>OW@(yRXmhBgZ&UbPqGl?Mi@0xcL)1)>~HG1>duGCsXx4MOX1gjSC7QzxI8eHO+K{os>q~@2L_Ud?zIQu6~8NhF!VP*_(l-N z^muKcoy8`E&Tyv$RX83PC`xHPBh$g0+FN^l=y-qDF61x$IWWsy?%tu^m4%Lu&~-&w zoNfk@>5%?3%HljQP)A0a@siBt4q|WeaOE>Mij=dz_2=@!E4Fjgz0!7lh>o_90imvs z4TJCWAq1nns>WAKYiNZ#5^ zmoPelH$;(K`{o2=aJGQ5H*KGtFxo?}1Vum(3={!v7#um`6_iC9a_WJqS`Q3V(|TZ_ ziq-=o9-VCR=2r-i{b@5dvW>Pd6@z-m+Or$RV6Tq)x*iy)Kkb2m;y6Vo&8s5Jt74Nk z7CKr(r5Vo-DgSJKg0GzVTDlX=b2liCdte~1Zi#?$x0z<&Mz%NRVEi}bBbG=mUDy6( zaLq3^U0w_%4~te+f+vY+Y8rh##bLr(8nLR~*!~+A#Ls!=@6&%Oo8IN{8f)He~2c6VuGakm^2pgxu zA_?j0p}bA|`Ug?b9=t6&$JSanWAHo-sxd7RZw<9FqQfGO*3h>=7}{)vcqJGp!fN|3 zjKQ8N9$`HYR_4B6OWBUrqcaWM#YG-n`>GycNY)m&aBY@O2-SjpV(nWrjKQ4|)S1-Q zu#CYS1!UnCX;i~}E|R$ReJP?ta!=HK(e@1)gQFjcO%`uh3EmK$2Wcn$nGTMB=v>PK zBW{7(%8cLf;|ddLt~PEk24|lrZfN}>V{nW`)}f6}gwYlHpQxtMR z(|91D3ey7vb*d~{)e(Fy>bQAepw6KO#_*0G?A)01wyh;2AF@D-2~NqD}^vxL%ocw+(JidczqUmD-R6hNjxx6XWgP#@*N&j-B`R;HDuE$ zV~^GBjq!k@jr@>a2?nA=`xODkkaOSDL%Sm{t5~9kR31oor&ZtfDyQ~$p|_57*8>CD zxd+A#wUw)1&v~Vu^hDcxqZdu(o$_tzEA|^kMdPx+y)fd@L#tIx1$TQ;4D`T2`GN-q z$`?E^P;RC5h$N4$@cuQb{k1(M!jMW3^)NgzP<^9$6R4clPDBOwT2Yl|M?m$H9RXQ} z_Js}NQ%A_FkcE3-#Ql^90I)jhxAe(JxXYT8%KoOiUntunD^N^MhUs0yxfq`-z4-C{(&~^?<9>Oam zbL}Z0VaS(kkUX^CKO_wK8XM9p4-6!84-7&iaUoWGT9O0F~hz-iWqhTloQ$! zkWJVTkhaaKV(;F2U?7QWXGWL~&U4U=x(!2S6fVzQ7p9|_1!}+PP8jkM zQ`|Z%kwe~8MAqSffizcFArqhEL>$?l9pOlFbYkTljny>&OU*_7AGS*z~ z6`~jEwety#!JZ2DRiB^G^mYhCZb>2=UsdS)pTujDp8PNZ#|>oTb_67E?WrYG!C5rw zBj`Pq%=4oP){cNGSUUpZlJ?w{Bq4l4_t>-@wt5W2DGxeOKIK6N(kTx*kbPS`p-ea- zFXe#|zjw4b_g0elJo(+|2diNM9(i+%^|6mak{GYu-!n2P`;7m-m( zGAdzC66GPwBO4Vm!5IJYs0r-}zSVA#hukiV@q%(!AB=c6TH7gRUU5E+Jc$Pe z>QyZqe{pBI=kV}~|MhS3YlHf_fas8KD4>i%*X3aZ_PMAk&?CspyC`y4vQRl0Kpt10 zOAA8W-`dwl-qwzQBA*=rS)GTZ<8shVi;z9YsRQDY9Rca89RcZ*9RZz;YCA_P3HD4V ztJQm^)^HO9<^OgBl>OTgkRQ`Nd&=k1aE!4})AhWhB0 zzFXd!ybND>AYU%C)fACp=j@hvAs8==-TIE9nM>!`lwW&bJk_^&(VlRJ!!1K...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 17:20:21 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 17:20:21 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7867-20260202-172021\n2026-02-02 17:20:21 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 17:20:21 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7867-20260202-172021\n2026-02-02 17:20:21 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 13:16:58+08:00 ~ 2026-02-02 19:16:58+08:00]\n2026-02-02 17:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 17:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.40s\n2026-02-02 17:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 17:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.49s\n2026-02-02 17:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 17:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.22s\n2026-02-02 17:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.22s\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.18s\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.20s\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.24s\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.23s\n2026-02-02 17:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 17:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.37s\n2026-02-02 17:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 17:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.66s\n2026-02-02 17:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.22s\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.20s\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.26s\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.26s\n2026-02-02 17:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.17s\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.20s\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.18s\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.24s\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.27s\n2026-02-02 17:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.22s\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.22s\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.20s\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.20s\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.23s\n2026-02-02 17:20:27 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.22s\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.27s\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.21s\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.20s\n2026-02-02 17:20:28 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.20s\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.22s\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.25s\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.22s\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.21s\n2026-02-02 17:20:29 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.20s\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.21s\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.20s\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.20s\n2026-02-02 17:20:30 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 17:20:31 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.24s\n2026-02-02 17:20:31 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 17:20:31 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.22s\n2026-02-02 17:20:31 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 17:20:31 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.20s\n2026-02-02 17:20:31 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 14, 'skipped': 60}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 13, 'skipped': 61}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 5, 'skipped': 167}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 5, 'skipped': 167}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 6, 'updated': 0, 'processed': 6}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 6, 'updated': 0, 'processed': 6}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 9, 'updated': 0, 'processed': 9}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 9, 'updated': 0, 'processed': 9}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 8, 'updated': 0, 'processed': 8}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 8, 'updated': 0, 'processed': 8}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 17:20:31 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 17:20:31 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "926c4d29", + "executed_at": "2026-02-02T16:16:57.920781", + "status": "success", + "exit_code": 0, + "duration_seconds": 209.076515, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 14条, 事实表新增: 21条, 事实表更新: 0条\n 维表: dim_table: +0, ~11, dim_store_goods: +0, ~3\n 事实表: dwd_settlement_head: +4, ~0, dwd_table_fee_log: +4, ~0, dwd_groupbuy_redemption: +4, ~0, dwd_platform_coupon_redemption: +5, ~0, dwd_payment: +4, ~0\n【错误】2026-02-02 16:16:59 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 16:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 16:17:01 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 14:16:57 --window-end 2026-02-02 16:16:57\n2026-02-02 16:16:58 [INFO] etl_billiards: 配置加载完成\n2026-02-02 16:16:58 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 16:16:58 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 16:16:58 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=0babb46114af4e6290e77e89dc3227b6\n2026-02-02 16:16:58 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7820-20260202-161658\n2026-02-02 16:16:58 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 16:16:58 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:16:59 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 16:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7821-20260202-161659\n2026-02-02 16:16:59 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 16:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 16:17:00 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7822-20260202-161700\n2026-02-02 16:17:00 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 16:17:00 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:01 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 16:17:01 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7823-20260202-161701\n2026-02-02 16:17:01 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:01 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00] 未发现需要抓取的小票\n2026-02-02 16:17:01 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 16:17:02 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7824-20260202-161702\n2026-02-02 16:17:02 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 16:17:02 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:02 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 16:17:02 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7825-20260202-161702\n2026-02-02 16:17:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 16:17:02 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:02 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 16:17:03 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7826-20260202-161703\n2026-02-02 16:17:03 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 16:17:03 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:05 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 16:17:05 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7827-20260202-161705\n2026-02-02 16:17:05 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 16:17:05 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:05 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 16:17:05 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7828-20260202-161705\n2026-02-02 16:17:05 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 16:17:05 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:06 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 3, 'updated': 0, 'skipped': 169, 'errors': 0}\n2026-02-02 16:17:06 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7829-20260202-161706\n2026-02-02 16:17:06 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 16:17:06 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:33 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10036, 'inserted': 4, 'updated': 0, 'skipped': 10032, 'errors': 0}\n2026-02-02 16:17:34 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7830-20260202-161734\n2026-02-02 16:17:34 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 16:17:34 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:17:41 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 16:17:42 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7831-20260202-161742\n2026-02-02 16:17:42 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 16:17:42 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:18:10 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2451, 'inserted': 0, 'updated': 0, 'skipped': 2451, 'errors': 0}\n2026-02-02 16:18:10 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7832-20260202-161810\n2026-02-02 16:18:10 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 16:18:10 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:18:10 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 12, 'updated': 0, 'skipped': 62, 'errors': 0}\n2026-02-02 16:18:11 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7833-20260202-161811\n2026-02-02 16:18:11 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 16:18:11 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:18:11 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 16:18:11 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7834-20260202-161811\n2026-02-02 16:18:11 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 16:18:11 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:18:14 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 3, 'updated': 0, 'skipped': 169, 'errors': 0}\n2026-02-02 16:18:14 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7835-20260202-161814\n2026-02-02 16:18:14 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 16:18:14 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:18:15 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 16:18:15 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7836-20260202-161815\n2026-02-02 16:18:15 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 16:18:15 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:18:16 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 16:18:16 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7837-20260202-161816\n2026-02-02 16:18:16 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 16:18:16 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:19:00 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17498, 'inserted': 5, 'updated': 0, 'skipped': 17493, 'errors': 0}\n2026-02-02 16:19:01 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7838-20260202-161901\n2026-02-02 16:19:01 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 16:19:01 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:19:01 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 13, 'inserted': 3, 'updated': 0, 'skipped': 10, 'errors': 0}\n2026-02-02 16:19:01 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7839-20260202-161901\n2026-02-02 16:19:01 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 16:19:01 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:19:45 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8133, 'inserted': 4, 'updated': 0, 'skipped': 8129, 'errors': 0}\n2026-02-02 16:19:45 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7840-20260202-161945\n2026-02-02 16:19:45 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 16:19:45 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:20:15 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11813, 'inserted': 4, 'updated': 0, 'skipped': 11809, 'errors': 0}\n2026-02-02 16:20:15 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7841-20260202-162015\n2026-02-02 16:20:15 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 16:20:15 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:20:15 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 16, 'inserted': 4, 'updated': 0, 'skipped': 12, 'errors': 0}\n2026-02-02 16:20:16 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7842-20260202-162016\n2026-02-02 16:20:16 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 16:20:16 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:20:16 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 16:20:17 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7843-20260202-162017\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7843-20260202-162017\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 12:16:57+08:00 ~ 2026-02-02 18:16:57+08:00]\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.42s\n2026-02-02 16:20:17 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.50s\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.23s\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.23s\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.19s\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.21s\n2026-02-02 16:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 16:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.29s\n2026-02-02 16:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 16:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.25s\n2026-02-02 16:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 16:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.37s\n2026-02-02 16:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 16:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.57s\n2026-02-02 16:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 16:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.20s\n2026-02-02 16:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 16:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.20s\n2026-02-02 16:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.26s\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.26s\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.18s\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.21s\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.18s\n2026-02-02 16:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.23s\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.25s\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.24s\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.23s\n2026-02-02 16:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.20s\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.19s\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.22s\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.23s\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.27s\n2026-02-02 16:20:23 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.22s\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.20s\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.20s\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.21s\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.21s\n2026-02-02 16:20:24 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.22s\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.21s\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.21s\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.22s\n2026-02-02 16:20:25 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.20s\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.20s\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.23s\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.20s\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.19s\n2026-02-02 16:20:26 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 11, 'skipped': 63}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 10, 'skipped': 64}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 3, 'skipped': 169}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 3, 'skipped': 169}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 5, 'updated': 0, 'processed': 5}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 5, 'updated': 0, 'processed': 5}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 16:20:26 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 16:20:26 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "47fa666c", + "executed_at": "2026-02-02T15:16:57.634176", + "status": "success", + "exit_code": 0, + "duration_seconds": 204.7959, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 12条, 事实表新增: 20条, 事实表更新: 0条\n 维表: dim_table: +0, ~9, dim_store_goods: +0, ~3\n 事实表: dwd_settlement_head: +4, ~0, dwd_table_fee_log: +3, ~0, dwd_groupbuy_redemption: +3, ~0, dwd_platform_coupon_redemption: +6, ~0, dwd_payment: +4, ~0\n【错误】2026-02-02 15:16:59 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 15:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 15:17:00 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 13:16:57 --window-end 2026-02-02 15:16:57\n2026-02-02 15:16:57 [INFO] etl_billiards: 配置加载完成\n2026-02-02 15:16:57 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 15:16:57 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 15:16:58 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=4b076e3a454a4b3195e8e37d2a4b2006\n2026-02-02 15:16:58 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7796-20260202-151658\n2026-02-02 15:16:58 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 15:16:58 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:16:59 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 15:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7797-20260202-151659\n2026-02-02 15:16:59 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 15:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:16:59 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 15:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7798-20260202-151659\n2026-02-02 15:16:59 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 15:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:00 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 15:17:00 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7799-20260202-151700\n2026-02-02 15:17:00 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:01 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00] 未发现需要抓取的小票\n2026-02-02 15:17:01 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 15:17:01 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7800-20260202-151701\n2026-02-02 15:17:01 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 15:17:01 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:01 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 15:17:02 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7801-20260202-151702\n2026-02-02 15:17:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 15:17:02 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:02 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 15:17:02 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7802-20260202-151702\n2026-02-02 15:17:02 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 15:17:02 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:04 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 15:17:04 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7803-20260202-151704\n2026-02-02 15:17:04 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 15:17:04 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:05 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 15:17:05 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7804-20260202-151705\n2026-02-02 15:17:05 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 15:17:05 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:05 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 3, 'updated': 0, 'skipped': 169, 'errors': 0}\n2026-02-02 15:17:06 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7805-20260202-151706\n2026-02-02 15:17:06 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 15:17:06 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:30 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10032, 'inserted': 3, 'updated': 0, 'skipped': 10029, 'errors': 0}\n2026-02-02 15:17:30 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7806-20260202-151730\n2026-02-02 15:17:30 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 15:17:30 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:17:38 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 15:17:38 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7807-20260202-151738\n2026-02-02 15:17:38 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 15:17:38 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:06 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2451, 'inserted': 0, 'updated': 0, 'skipped': 2451, 'errors': 0}\n2026-02-02 15:18:06 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7808-20260202-151806\n2026-02-02 15:18:06 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 15:18:06 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:06 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 9, 'updated': 0, 'skipped': 65, 'errors': 0}\n2026-02-02 15:18:07 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7809-20260202-151807\n2026-02-02 15:18:07 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 15:18:07 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:07 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 15:18:07 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7810-20260202-151807\n2026-02-02 15:18:07 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 15:18:07 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:09 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 3, 'updated': 0, 'skipped': 169, 'errors': 0}\n2026-02-02 15:18:09 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7811-20260202-151809\n2026-02-02 15:18:09 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 15:18:09 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:09 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 15:18:10 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7812-20260202-151810\n2026-02-02 15:18:10 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 15:18:10 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:11 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 15:18:11 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7813-20260202-151811\n2026-02-02 15:18:11 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 15:18:11 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:56 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17493, 'inserted': 6, 'updated': 0, 'skipped': 17487, 'errors': 0}\n2026-02-02 15:18:57 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7814-20260202-151857\n2026-02-02 15:18:57 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 15:18:57 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:18:57 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 12, 'inserted': 3, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 15:18:57 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7815-20260202-151857\n2026-02-02 15:18:57 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 15:18:57 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:19:41 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8129, 'inserted': 3, 'updated': 0, 'skipped': 8126, 'errors': 0}\n2026-02-02 15:19:41 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7816-20260202-151941\n2026-02-02 15:19:41 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 15:19:41 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:20:10 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11809, 'inserted': 4, 'updated': 0, 'skipped': 11805, 'errors': 0}\n2026-02-02 15:20:10 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7817-20260202-152010\n2026-02-02 15:20:10 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 15:20:10 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:20:10 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 14, 'inserted': 4, 'updated': 0, 'skipped': 10, 'errors': 0}\n2026-02-02 15:20:11 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7818-20260202-152011\n2026-02-02 15:20:11 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 15:20:11 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:20:11 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 15:20:11 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7819-20260202-152012\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7819-20260202-152012\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 11:16:57+08:00 ~ 2026-02-02 17:16:57+08:00]\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.42s\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.47s\n2026-02-02 15:20:12 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.28s\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.24s\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.20s\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.21s\n2026-02-02 15:20:13 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 15:20:14 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.26s\n2026-02-02 15:20:14 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 15:20:14 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.23s\n2026-02-02 15:20:14 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 15:20:14 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.39s\n2026-02-02 15:20:14 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 15:20:15 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.69s\n2026-02-02 15:20:15 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 15:20:15 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.21s\n2026-02-02 15:20:15 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 15:20:15 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.22s\n2026-02-02 15:20:15 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.27s\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.27s\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.20s\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.19s\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.19s\n2026-02-02 15:20:16 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.26s\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.25s\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.23s\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.23s\n2026-02-02 15:20:17 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.22s\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.21s\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.23s\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.23s\n2026-02-02 15:20:18 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.28s\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.22s\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.21s\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.21s\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.23s\n2026-02-02 15:20:19 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.23s\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.23s\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.23s\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.23s\n2026-02-02 15:20:20 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.22s\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.22s\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.21s\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.27s\n2026-02-02 15:20:21 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 15:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.23s\n2026-02-02 15:20:22 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 15:20:22 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.21s\n2026-02-02 15:20:22 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 9, 'skipped': 65}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 8, 'skipped': 66}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 3, 'skipped': 169}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 3, 'skipped': 169}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 6, 'updated': 0, 'processed': 6}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 6, 'updated': 0, 'processed': 6}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 15:20:22 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 15:20:22 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "2d422111", + "executed_at": "2026-02-02T14:16:57.126884", + "status": "success", + "exit_code": 0, + "duration_seconds": 189.098646, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 9条, 事实表新增: 21条, 事实表更新: 0条\n 维表: dim_table: +0, ~7, dim_store_goods: +0, ~2\n 事实表: dwd_settlement_head: +4, ~0, dwd_table_fee_log: +3, ~0, dwd_groupbuy_redemption: +3, ~0, dwd_platform_coupon_redemption: +7, ~0, dwd_payment: +4, ~0\n【错误】2026-02-02 14:16:58 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 14:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 14:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 12:16:57 --window-end 2026-02-02 14:16:57\n2026-02-02 14:16:57 [INFO] etl_billiards: 配置加载完成\n2026-02-02 14:16:57 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 14:16:57 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 14:16:57 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=acb9074c3c104351b82d42cbae988241\n2026-02-02 14:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7772-20260202-141657\n2026-02-02 14:16:57 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 14:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:16:58 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 14:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7773-20260202-141658\n2026-02-02 14:16:58 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 14:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 14:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7774-20260202-141659\n2026-02-02 14:16:59 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 14:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 14:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7775-20260202-141659\n2026-02-02 14:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:00 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00] 未发现需要抓取的小票\n2026-02-02 14:17:00 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 14:17:00 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7776-20260202-141700\n2026-02-02 14:17:00 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 14:17:00 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:01 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 14:17:01 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7777-20260202-141701\n2026-02-02 14:17:01 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 14:17:01 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:01 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 14:17:01 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7778-20260202-141701\n2026-02-02 14:17:01 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 14:17:01 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:03 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 14:17:04 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7779-20260202-141704\n2026-02-02 14:17:04 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 14:17:04 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:04 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 14:17:04 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7780-20260202-141704\n2026-02-02 14:17:04 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 14:17:04 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:04 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 2, 'updated': 0, 'skipped': 170, 'errors': 0}\n2026-02-02 14:17:05 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7781-20260202-141705\n2026-02-02 14:17:05 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 14:17:05 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:27 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10029, 'inserted': 3, 'updated': 0, 'skipped': 10026, 'errors': 0}\n2026-02-02 14:17:27 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7782-20260202-141727\n2026-02-02 14:17:27 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 14:17:27 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:17:34 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 14:17:34 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7783-20260202-141734\n2026-02-02 14:17:34 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 14:17:34 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:01 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2451, 'inserted': 0, 'updated': 0, 'skipped': 2451, 'errors': 0}\n2026-02-02 14:18:01 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7784-20260202-141801\n2026-02-02 14:18:01 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 14:18:01 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:02 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 8, 'updated': 0, 'skipped': 66, 'errors': 0}\n2026-02-02 14:18:02 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7785-20260202-141802\n2026-02-02 14:18:02 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 14:18:02 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:02 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 14:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7786-20260202-141802\n2026-02-02 14:18:02 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 14:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:04 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 2, 'updated': 0, 'skipped': 170, 'errors': 0}\n2026-02-02 14:18:04 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7787-20260202-141804\n2026-02-02 14:18:04 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 14:18:04 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:04 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 14:18:05 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7788-20260202-141805\n2026-02-02 14:18:05 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 14:18:05 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:06 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 14:18:06 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7789-20260202-141806\n2026-02-02 14:18:06 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 14:18:06 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:47 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17487, 'inserted': 7, 'updated': 0, 'skipped': 17480, 'errors': 0}\n2026-02-02 14:18:47 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7790-20260202-141847\n2026-02-02 14:18:47 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 14:18:47 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:18:47 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 9, 'inserted': 2, 'updated': 0, 'skipped': 7, 'errors': 0}\n2026-02-02 14:18:47 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7791-20260202-141847\n2026-02-02 14:18:47 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 14:18:47 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:19:29 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8126, 'inserted': 3, 'updated': 0, 'skipped': 8123, 'errors': 0}\n2026-02-02 14:19:29 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7792-20260202-141929\n2026-02-02 14:19:29 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 14:19:29 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:19:54 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11805, 'inserted': 4, 'updated': 0, 'skipped': 11801, 'errors': 0}\n2026-02-02 14:19:54 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7793-20260202-141954\n2026-02-02 14:19:54 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 14:19:54 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:19:55 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 10, 'inserted': 4, 'updated': 0, 'skipped': 6, 'errors': 0}\n2026-02-02 14:19:55 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7794-20260202-141955\n2026-02-02 14:19:55 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 14:19:55 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:19:56 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 14:19:56 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7795-20260202-141956\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7795-20260202-141956\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 10:16:57+08:00 ~ 2026-02-02 16:16:57+08:00]\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.49s\n2026-02-02 14:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.47s\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.26s\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.22s\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.18s\n2026-02-02 14:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.20s\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.25s\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.22s\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.36s\n2026-02-02 14:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 14:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.59s\n2026-02-02 14:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 14:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.21s\n2026-02-02 14:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 14:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.20s\n2026-02-02 14:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.26s\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.27s\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.18s\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.18s\n2026-02-02 14:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.19s\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.23s\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.24s\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.22s\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.22s\n2026-02-02 14:20:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.20s\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.20s\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.22s\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.24s\n2026-02-02 14:20:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.26s\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.23s\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.20s\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.20s\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.21s\n2026-02-02 14:20:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.21s\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.23s\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.22s\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.21s\n2026-02-02 14:20:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.22s\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.20s\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.20s\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.24s\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.20s\n2026-02-02 14:20:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 14:20:06 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.20s\n2026-02-02 14:20:06 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 7, 'skipped': 67}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 6, 'skipped': 68}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 2, 'skipped': 170}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 2, 'skipped': 170}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 14:20:06 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 14:20:06 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "101350da", + "executed_at": "2026-02-02T13:16:56.744049", + "status": "success", + "exit_code": 0, + "duration_seconds": 184.542973, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 7条, 事实表新增: 16条, 事实表更新: 0条\n 维表: dim_table: +0, ~4, dim_store_goods: +0, ~3\n 事实表: dwd_settlement_head: +4, ~0, dwd_table_fee_log: +3, ~0, dwd_groupbuy_redemption: +3, ~0, dwd_platform_coupon_redemption: +2, ~0, dwd_payment: +4, ~0\n【错误】2026-02-02 13:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 13:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 13:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 11:16:56 --window-end 2026-02-02 13:16:56\n2026-02-02 13:16:57 [INFO] etl_billiards: 配置加载完成\n2026-02-02 13:16:57 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 13:16:57 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 13:16:57 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=1307568bb1df4a41b94fc5fe912b388e\n2026-02-02 13:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7748-20260202-131657\n2026-02-02 13:16:57 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 13:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 13:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7749-20260202-131658\n2026-02-02 13:16:58 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 13:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:16:58 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 13:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7750-20260202-131658\n2026-02-02 13:16:58 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 13:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7751-20260202-131659\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00] 未发现需要抓取的小票\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7752-20260202-131659\n2026-02-02 13:16:59 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 13:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:00 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 13:17:00 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7753-20260202-131700\n2026-02-02 13:17:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 13:17:00 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:00 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 13:17:00 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7754-20260202-131700\n2026-02-02 13:17:00 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 13:17:00 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:02 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 13:17:03 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7755-20260202-131703\n2026-02-02 13:17:03 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 13:17:03 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:03 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 13:17:03 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7756-20260202-131703\n2026-02-02 13:17:03 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 13:17:03 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:03 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 3, 'updated': 0, 'skipped': 169, 'errors': 0}\n2026-02-02 13:17:04 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7757-20260202-131704\n2026-02-02 13:17:04 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 13:17:04 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:25 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10026, 'inserted': 3, 'updated': 0, 'skipped': 10023, 'errors': 0}\n2026-02-02 13:17:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7758-20260202-131725\n2026-02-02 13:17:25 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 13:17:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:32 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 13:17:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7759-20260202-131732\n2026-02-02 13:17:32 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 13:17:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:17:59 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2451, 'inserted': 0, 'updated': 0, 'skipped': 2451, 'errors': 0}\n2026-02-02 13:18:00 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7760-20260202-131800\n2026-02-02 13:18:00 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 13:18:00 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:00 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 5, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 13:18:00 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7761-20260202-131800\n2026-02-02 13:18:00 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 13:18:00 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 13:18:01 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7762-20260202-131801\n2026-02-02 13:18:01 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 13:18:01 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:03 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 3, 'updated': 0, 'skipped': 169, 'errors': 0}\n2026-02-02 13:18:03 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7763-20260202-131803\n2026-02-02 13:18:03 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 13:18:03 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:04 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 13:18:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7764-20260202-131804\n2026-02-02 13:18:04 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 13:18:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:05 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 13:18:05 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7765-20260202-131805\n2026-02-02 13:18:05 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 13:18:05 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:46 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17480, 'inserted': 2, 'updated': 0, 'skipped': 17478, 'errors': 0}\n2026-02-02 13:18:46 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7766-20260202-131846\n2026-02-02 13:18:46 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 13:18:46 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:18:46 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 7, 'inserted': 5, 'updated': 0, 'skipped': 2, 'errors': 0}\n2026-02-02 13:18:47 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7767-20260202-131847\n2026-02-02 13:18:47 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 13:18:47 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:19:25 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8123, 'inserted': 3, 'updated': 0, 'skipped': 8120, 'errors': 0}\n2026-02-02 13:19:26 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7768-20260202-131926\n2026-02-02 13:19:26 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 13:19:26 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:19:50 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11801, 'inserted': 4, 'updated': 0, 'skipped': 11797, 'errors': 0}\n2026-02-02 13:19:50 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7769-20260202-131950\n2026-02-02 13:19:50 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 13:19:50 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:19:50 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 6, 'inserted': 4, 'updated': 0, 'skipped': 2, 'errors': 0}\n2026-02-02 13:19:51 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7770-20260202-131951\n2026-02-02 13:19:51 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 13:19:51 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:19:51 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 13:19:51 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7771-20260202-131952\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7771-20260202-131952\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 09:16:56+08:00 ~ 2026-02-02 15:16:56+08:00]\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.36s\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.49s\n2026-02-02 13:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.23s\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.22s\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.18s\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.19s\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.26s\n2026-02-02 13:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 13:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.21s\n2026-02-02 13:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 13:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.36s\n2026-02-02 13:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.57s\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.19s\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.19s\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.25s\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.24s\n2026-02-02 13:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.16s\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.17s\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.17s\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.22s\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.23s\n2026-02-02 13:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.21s\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.20s\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.19s\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.19s\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.20s\n2026-02-02 13:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.20s\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.25s\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.20s\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.19s\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.19s\n2026-02-02 13:19:58 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.19s\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.19s\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.21s\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.21s\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.20s\n2026-02-02 13:19:59 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.20s\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.19s\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.19s\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.22s\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.19s\n2026-02-02 13:20:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 13:20:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.19s\n2026-02-02 13:20:01 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 4, 'skipped': 70}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 3, 'skipped': 71}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 3, 'skipped': 169}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 3, 'skipped': 169}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 3, 'updated': 0, 'processed': 3}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 2, 'updated': 0, 'processed': 2}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 2, 'updated': 0, 'processed': 2}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 13:20:01 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 13:20:01 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "9fec4a10", + "executed_at": "2026-02-02T12:16:56.398831", + "status": "success", + "exit_code": 0, + "duration_seconds": 180.610921, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 7条, 事实表新增: 11条, 事实表更新: 0条\n 维表: dim_table: +0, ~4, dim_member: +0, ~1, dim_store_goods: +0, ~2\n 事实表: dwd_settlement_head: +2, ~0, dwd_table_fee_log: +1, ~0, dwd_member_balance_change: +1, ~0, dwd_groupbuy_redemption: +1, ~0, dwd_platform_coupon_redemption: +4, ~0, dwd_payment: +2, ~0\n【错误】2026-02-02 12:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 12:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 12:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 10:16:56 --window-end 2026-02-02 12:16:56\n2026-02-02 12:16:56 [INFO] etl_billiards: 配置加载完成\n2026-02-02 12:16:56 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 12:16:56 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 12:16:57 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=d33c7a9f3e324b3e814d23c73038b86b\n2026-02-02 12:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7724-20260202-121657\n2026-02-02 12:16:57 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 12:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 12:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7725-20260202-121657\n2026-02-02 12:16:57 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 12:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 12:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7726-20260202-121658\n2026-02-02 12:16:58 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 12:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 12:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7727-20260202-121658\n2026-02-02 12:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00] 未发现需要抓取的小票\n2026-02-02 12:16:59 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 12:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7728-20260202-121659\n2026-02-02 12:16:59 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 12:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 12:17:00 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7729-20260202-121700\n2026-02-02 12:17:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 12:17:00 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:00 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 12:17:00 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7730-20260202-121700\n2026-02-02 12:17:00 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 12:17:00 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:02 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 1, 'updated': 0, 'skipped': 555, 'errors': 0}\n2026-02-02 12:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7731-20260202-121702\n2026-02-02 12:17:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 12:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 12:17:03 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7732-20260202-121703\n2026-02-02 12:17:03 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 12:17:03 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:03 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 2, 'updated': 0, 'skipped': 170, 'errors': 0}\n2026-02-02 12:17:03 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7733-20260202-121703\n2026-02-02 12:17:03 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 12:17:03 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:24 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10023, 'inserted': 1, 'updated': 0, 'skipped': 10022, 'errors': 0}\n2026-02-02 12:17:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7734-20260202-121725\n2026-02-02 12:17:25 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 12:17:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:31 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 12:17:31 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7735-20260202-121731\n2026-02-02 12:17:31 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 12:17:31 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:17:59 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2451, 'inserted': 1, 'updated': 0, 'skipped': 2450, 'errors': 0}\n2026-02-02 12:17:59 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7736-20260202-121759\n2026-02-02 12:17:59 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 12:17:59 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:00 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 4, 'updated': 0, 'skipped': 70, 'errors': 0}\n2026-02-02 12:18:00 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7737-20260202-121800\n2026-02-02 12:18:00 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 12:18:00 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:00 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 12:18:00 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7738-20260202-121800\n2026-02-02 12:18:00 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 12:18:00 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:01 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 2, 'updated': 0, 'skipped': 170, 'errors': 0}\n2026-02-02 12:18:01 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7739-20260202-121801\n2026-02-02 12:18:01 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 12:18:01 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:01 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 12:18:02 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7740-20260202-121802\n2026-02-02 12:18:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 12:18:02 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:03 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 12:18:03 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7741-20260202-121803\n2026-02-02 12:18:03 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 12:18:03 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:43 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17478, 'inserted': 4, 'updated': 0, 'skipped': 17474, 'errors': 0}\n2026-02-02 12:18:43 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7742-20260202-121843\n2026-02-02 12:18:43 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 12:18:43 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:18:43 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 2, 'inserted': 2, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 12:18:44 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7743-20260202-121844\n2026-02-02 12:18:44 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 12:18:44 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:19:21 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8120, 'inserted': 1, 'updated': 0, 'skipped': 8119, 'errors': 0}\n2026-02-02 12:19:21 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7744-20260202-121921\n2026-02-02 12:19:21 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 12:19:21 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:19:45 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11797, 'inserted': 2, 'updated': 0, 'skipped': 11795, 'errors': 0}\n2026-02-02 12:19:45 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7745-20260202-121945\n2026-02-02 12:19:45 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 12:19:45 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:19:46 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 2, 'inserted': 2, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 12:19:46 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7746-20260202-121946\n2026-02-02 12:19:46 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 12:19:46 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:19:47 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 12:19:47 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7747-20260202-121947\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7747-20260202-121947\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 08:16:56+08:00 ~ 2026-02-02 14:16:56+08:00]\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.35s\n2026-02-02 12:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.44s\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.24s\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.22s\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.18s\n2026-02-02 12:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.19s\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.27s\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.21s\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.34s\n2026-02-02 12:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 12:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.54s\n2026-02-02 12:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 12:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.20s\n2026-02-02 12:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 12:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.19s\n2026-02-02 12:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.25s\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.26s\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.17s\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.17s\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.17s\n2026-02-02 12:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.22s\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.22s\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.22s\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.44s\n2026-02-02 12:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.19s\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.19s\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.24s\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.21s\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.24s\n2026-02-02 12:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.20s\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.19s\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.19s\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.21s\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.19s\n2026-02-02 12:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.20s\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.20s\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.20s\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.20s\n2026-02-02 12:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.21s\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.25s\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.23s\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.18s\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.18s\n2026-02-02 12:19:56 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 4, 'skipped': 70}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 3, 'skipped': 71}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 1, 'skipped': 555}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 2, 'skipped': 170}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 2, 'skipped': 170}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 2, 'updated': 0, 'processed': 2}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 2, 'updated': 0, 'processed': 2}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 2, 'updated': 0, 'processed': 2}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 12:19:56 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 12:19:56 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "25f2f24e", + "executed_at": "2026-02-02T11:16:55.990890", + "status": "success", + "exit_code": 0, + "duration_seconds": 176.757487, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 1条, 事实表新增: 1条, 事实表更新: 0条\n 维表: dim_table: +0, ~1\n 事实表: dwd_platform_coupon_redemption: +1, ~0\n【错误】2026-02-02 11:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 11:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 11:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 09:16:55 --window-end 2026-02-02 11:16:55\n2026-02-02 11:16:56 [INFO] etl_billiards: 配置加载完成\n2026-02-02 11:16:56 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 11:16:56 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 11:16:56 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=4700843076ae45c1bba1a2ad781cb7f0\n2026-02-02 11:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7700-20260202-111656\n2026-02-02 11:16:56 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 11:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 11:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7701-20260202-111657\n2026-02-02 11:16:57 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 11:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 11:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7702-20260202-111657\n2026-02-02 11:16:57 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 11:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 11:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7703-20260202-111658\n2026-02-02 11:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00] 未发现需要抓取的小票\n2026-02-02 11:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 11:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7704-20260202-111659\n2026-02-02 11:16:59 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 11:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 11:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7705-20260202-111659\n2026-02-02 11:16:59 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 11:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 11:17:00 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7706-20260202-111700\n2026-02-02 11:17:00 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 11:17:00 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:02 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 11:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7707-20260202-111702\n2026-02-02 11:17:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 11:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 11:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7708-20260202-111702\n2026-02-02 11:17:02 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 11:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:03 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 11:17:03 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7709-20260202-111703\n2026-02-02 11:17:03 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 11:17:03 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:24 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10022, 'inserted': 0, 'updated': 0, 'skipped': 10022, 'errors': 0}\n2026-02-02 11:17:24 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7710-20260202-111724\n2026-02-02 11:17:24 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 11:17:24 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:31 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 11:17:31 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7711-20260202-111731\n2026-02-02 11:17:31 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 11:17:31 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:58 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2450, 'inserted': 0, 'updated': 0, 'skipped': 2450, 'errors': 0}\n2026-02-02 11:17:58 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7712-20260202-111758\n2026-02-02 11:17:58 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 11:17:58 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:58 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 1, 'updated': 0, 'skipped': 73, 'errors': 0}\n2026-02-02 11:17:59 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7713-20260202-111759\n2026-02-02 11:17:59 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 11:17:59 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:17:59 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 11:17:59 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7714-20260202-111759\n2026-02-02 11:17:59 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 11:17:59 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:18:00 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 11:18:01 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7715-20260202-111801\n2026-02-02 11:18:01 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 11:18:01 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:18:01 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 11:18:01 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7716-20260202-111801\n2026-02-02 11:18:01 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 11:18:01 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:18:02 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 11:18:03 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7717-20260202-111803\n2026-02-02 11:18:03 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 11:18:03 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:18:41 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17474, 'inserted': 1, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 11:18:41 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7718-20260202-111841\n2026-02-02 11:18:41 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 11:18:41 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:18:42 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 11:18:42 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7719-20260202-111842\n2026-02-02 11:18:42 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 11:18:42 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:19:18 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8119, 'inserted': 0, 'updated': 0, 'skipped': 8119, 'errors': 0}\n2026-02-02 11:19:19 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7720-20260202-111919\n2026-02-02 11:19:19 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 11:19:19 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:19:42 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11795, 'inserted': 0, 'updated': 0, 'skipped': 11795, 'errors': 0}\n2026-02-02 11:19:42 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7721-20260202-111942\n2026-02-02 11:19:42 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 11:19:42 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:19:43 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 11:19:43 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7722-20260202-111943\n2026-02-02 11:19:43 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 11:19:43 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:19:43 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 11:19:44 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7723-20260202-111944\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7723-20260202-111944\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 07:16:55+08:00 ~ 2026-02-02 13:16:55+08:00]\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.35s\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.44s\n2026-02-02 11:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.20s\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.24s\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.17s\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.18s\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.22s\n2026-02-02 11:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 11:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.20s\n2026-02-02 11:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 11:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.33s\n2026-02-02 11:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 11:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.54s\n2026-02-02 11:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.19s\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.18s\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.19s\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.19s\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.15s\n2026-02-02 11:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.16s\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.16s\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.21s\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.21s\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.19s\n2026-02-02 11:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.19s\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.18s\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.18s\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.19s\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.19s\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.23s\n2026-02-02 11:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.18s\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.18s\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.20s\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.18s\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.18s\n2026-02-02 11:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.19s\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.19s\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.19s\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.19s\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.18s\n2026-02-02 11:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.18s\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.21s\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.18s\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.18s\n2026-02-02 11:19:52 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 1, 'skipped': 73}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 1, 'skipped': 73}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 11:19:52 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 11:19:52 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "0458ffd6", + "executed_at": "2026-02-02T10:16:55.681120", + "status": "success", + "exit_code": 0, + "duration_seconds": 180.120778, + "summary": "【错误】2026-02-02 10:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 10:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 10:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 08:16:55 --window-end 2026-02-02 10:16:55\n2026-02-02 10:16:56 [INFO] etl_billiards: 配置加载完成\n2026-02-02 10:16:56 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 10:16:56 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 10:16:56 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=7e2a96ee2ca84282929200ccf2fddb66\n2026-02-02 10:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7676-20260202-101656\n2026-02-02 10:16:56 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 10:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:16:57 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 10:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7677-20260202-101657\n2026-02-02 10:16:57 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 10:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:16:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 10:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7678-20260202-101657\n2026-02-02 10:16:57 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 10:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:16:58 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 10:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7679-20260202-101658\n2026-02-02 10:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00] 未发现需要抓取的小票\n2026-02-02 10:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 10:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7680-20260202-101659\n2026-02-02 10:16:59 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 10:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:16:59 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 10:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7681-20260202-101659\n2026-02-02 10:16:59 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 10:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 10:17:00 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7682-20260202-101700\n2026-02-02 10:17:00 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 10:17:00 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:17:02 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 10:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7683-20260202-101702\n2026-02-02 10:17:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 10:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 10:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7684-20260202-101702\n2026-02-02 10:17:02 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 10:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:17:03 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 10:17:03 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7685-20260202-101703\n2026-02-02 10:17:03 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 10:17:03 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:17:26 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10022, 'inserted': 0, 'updated': 0, 'skipped': 10022, 'errors': 0}\n2026-02-02 10:17:26 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7686-20260202-101726\n2026-02-02 10:17:26 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 10:17:26 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:17:32 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 10:17:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7687-20260202-101732\n2026-02-02 10:17:32 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 10:17:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:00 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2450, 'inserted': 0, 'updated': 0, 'skipped': 2450, 'errors': 0}\n2026-02-02 10:18:01 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7688-20260202-101801\n2026-02-02 10:18:01 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 10:18:01 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:01 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 10:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7689-20260202-101801\n2026-02-02 10:18:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 10:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:02 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 10:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7690-20260202-101802\n2026-02-02 10:18:02 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 10:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:03 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 10:18:03 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7691-20260202-101803\n2026-02-02 10:18:03 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 10:18:03 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:04 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 10:18:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7692-20260202-101804\n2026-02-02 10:18:04 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 10:18:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:05 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 10:18:05 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7693-20260202-101805\n2026-02-02 10:18:05 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 10:18:05 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:44 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 10:18:45 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7694-20260202-101845\n2026-02-02 10:18:45 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 10:18:45 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:18:45 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 10:18:45 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7695-20260202-101845\n2026-02-02 10:18:45 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 10:18:45 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:19:21 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8119, 'inserted': 0, 'updated': 0, 'skipped': 8119, 'errors': 0}\n2026-02-02 10:19:21 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7696-20260202-101921\n2026-02-02 10:19:21 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 10:19:21 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:19:44 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11795, 'inserted': 0, 'updated': 0, 'skipped': 11795, 'errors': 0}\n2026-02-02 10:19:45 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7697-20260202-101945\n2026-02-02 10:19:45 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 10:19:45 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:19:45 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 10:19:45 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7698-20260202-101945\n2026-02-02 10:19:45 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 10:19:45 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:19:46 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 10:19:46 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7699-20260202-101946\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7699-20260202-101946\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 06:16:55+08:00 ~ 2026-02-02 12:16:55+08:00]\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.43s\n2026-02-02 10:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.46s\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.17s\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.17s\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.17s\n2026-02-02 10:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.19s\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.25s\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.21s\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.36s\n2026-02-02 10:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 10:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.57s\n2026-02-02 10:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 10:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.19s\n2026-02-02 10:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 10:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.19s\n2026-02-02 10:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.20s\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.21s\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.16s\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.17s\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.17s\n2026-02-02 10:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.21s\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.21s\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.20s\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.20s\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.19s\n2026-02-02 10:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.18s\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.20s\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.22s\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.32s\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.22s\n2026-02-02 10:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.20s\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.23s\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.20s\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.20s\n2026-02-02 10:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.24s\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.20s\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.20s\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.21s\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.19s\n2026-02-02 10:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.19s\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.22s\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.18s\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.20s\n2026-02-02 10:19:55 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 10:19:55 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 10:19:55 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "62567464", + "executed_at": "2026-02-02T09:16:55.297784", + "status": "success", + "exit_code": 0, + "duration_seconds": 178.074014, + "summary": "【错误】2026-02-02 09:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 09:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 09:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 07:16:55 --window-end 2026-02-02 09:16:55\n2026-02-02 09:16:55 [INFO] etl_billiards: 配置加载完成\n2026-02-02 09:16:55 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 09:16:55 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 09:16:55 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=68003b7d5ec141f88ab5497510fc6f47\n2026-02-02 09:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7652-20260202-091655\n2026-02-02 09:16:55 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 09:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 09:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7653-20260202-091656\n2026-02-02 09:16:56 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 09:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 09:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7654-20260202-091657\n2026-02-02 09:16:57 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 09:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 09:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7655-20260202-091657\n2026-02-02 09:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00] 未发现需要抓取的小票\n2026-02-02 09:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 09:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7656-20260202-091658\n2026-02-02 09:16:58 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 09:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 09:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7657-20260202-091659\n2026-02-02 09:16:59 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 09:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 09:16:59 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7658-20260202-091659\n2026-02-02 09:16:59 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 09:16:59 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:17:01 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 09:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7659-20260202-091702\n2026-02-02 09:17:02 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 09:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:17:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 09:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7660-20260202-091702\n2026-02-02 09:17:02 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 09:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:17:02 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 09:17:03 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7661-20260202-091703\n2026-02-02 09:17:03 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 09:17:03 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:17:25 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10022, 'inserted': 0, 'updated': 0, 'skipped': 10022, 'errors': 0}\n2026-02-02 09:17:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7662-20260202-091725\n2026-02-02 09:17:25 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 09:17:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:17:32 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 09:17:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7663-20260202-091732\n2026-02-02 09:17:32 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 09:17:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:00 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2450, 'inserted': 0, 'updated': 0, 'skipped': 2450, 'errors': 0}\n2026-02-02 09:18:00 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7664-20260202-091800\n2026-02-02 09:18:00 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 09:18:00 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:01 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 09:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7665-20260202-091801\n2026-02-02 09:18:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 09:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 09:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7666-20260202-091802\n2026-02-02 09:18:02 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 09:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 09:18:03 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7667-20260202-091803\n2026-02-02 09:18:03 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 09:18:03 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:03 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 09:18:03 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7668-20260202-091803\n2026-02-02 09:18:03 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 09:18:03 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 09:18:04 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7669-20260202-091804\n2026-02-02 09:18:04 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 09:18:04 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:43 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 09:18:43 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7670-20260202-091843\n2026-02-02 09:18:43 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 09:18:43 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:18:44 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 09:18:44 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7671-20260202-091844\n2026-02-02 09:18:44 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 09:18:44 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:19:19 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8119, 'inserted': 0, 'updated': 0, 'skipped': 8119, 'errors': 0}\n2026-02-02 09:19:19 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7672-20260202-091919\n2026-02-02 09:19:19 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 09:19:19 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:19:43 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11795, 'inserted': 0, 'updated': 0, 'skipped': 11795, 'errors': 0}\n2026-02-02 09:19:43 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7673-20260202-091943\n2026-02-02 09:19:43 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 09:19:43 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:19:43 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 09:19:43 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7674-20260202-091943\n2026-02-02 09:19:43 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 09:19:43 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:19:44 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 09:19:44 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7675-20260202-091944\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7675-20260202-091944\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 05:16:55+08:00 ~ 2026-02-02 11:16:55+08:00]\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.36s\n2026-02-02 09:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.43s\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.17s\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.17s\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.17s\n2026-02-02 09:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.18s\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.24s\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.21s\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.33s\n2026-02-02 09:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.57s\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.21s\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.19s\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.20s\n2026-02-02 09:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.20s\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.17s\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.17s\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.17s\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.21s\n2026-02-02 09:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.21s\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.20s\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.20s\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.19s\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.18s\n2026-02-02 09:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.20s\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.20s\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.24s\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.19s\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.19s\n2026-02-02 09:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.18s\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.21s\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.19s\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.20s\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.20s\n2026-02-02 09:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.20s\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.20s\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.19s\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.19s\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.22s\n2026-02-02 09:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 09:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.18s\n2026-02-02 09:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 09:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.19s\n2026-02-02 09:19:53 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 09:19:53 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 09:19:53 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "78be21ce", + "executed_at": "2026-02-02T08:16:54.927356", + "status": "success", + "exit_code": 0, + "duration_seconds": 182.469738, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 66条, 事实表新增: 0条, 事实表更新: 0条\n 维表: dim_store_goods: +0, ~66\n【错误】2026-02-02 08:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 08:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0; 2026-02-02 08:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 06:16:54 --window-end 2026-02-02 08:16:54\n2026-02-02 08:16:55 [INFO] etl_billiards: 配置加载完成\n2026-02-02 08:16:55 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 08:16:55 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 08:16:55 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=741a786d9149476eac23c265e8cf5cba\n2026-02-02 08:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7628-20260202-081655\n2026-02-02 08:16:55 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 08:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:16:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 08:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7629-20260202-081656\n2026-02-02 08:16:56 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 08:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 08:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7630-20260202-081656\n2026-02-02 08:16:56 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 08:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:16:57 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 08:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7631-20260202-081657\n2026-02-02 08:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00] 未发现需要抓取的小票\n2026-02-02 08:16:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 08:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7632-20260202-081658\n2026-02-02 08:16:58 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 08:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 08:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7633-20260202-081659\n2026-02-02 08:16:59 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 08:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:16:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 08:16:59 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7634-20260202-081659\n2026-02-02 08:16:59 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 08:16:59 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:17:01 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 08:17:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7635-20260202-081701\n2026-02-02 08:17:01 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 08:17:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:17:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 08:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7636-20260202-081702\n2026-02-02 08:17:02 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 08:17:02 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:17:02 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 70, 'updated': 0, 'skipped': 102, 'errors': 0}\n2026-02-02 08:17:02 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7637-20260202-081702\n2026-02-02 08:17:02 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 08:17:02 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:17:23 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10022, 'inserted': 0, 'updated': 0, 'skipped': 10022, 'errors': 0}\n2026-02-02 08:17:24 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7638-20260202-081724\n2026-02-02 08:17:24 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 08:17:24 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:17:31 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1777, 'inserted': 0, 'updated': 0, 'skipped': 1777, 'errors': 0}\n2026-02-02 08:17:31 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7639-20260202-081731\n2026-02-02 08:17:31 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 08:17:31 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:00 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2450, 'inserted': 0, 'updated': 0, 'skipped': 2450, 'errors': 0}\n2026-02-02 08:18:00 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7640-20260202-081800\n2026-02-02 08:18:00 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 08:18:00 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:01 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 08:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7641-20260202-081801\n2026-02-02 08:18:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 08:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 08:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7642-20260202-081802\n2026-02-02 08:18:02 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 08:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 08:18:02 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7643-20260202-081802\n2026-02-02 08:18:02 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 08:18:02 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:03 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 08:18:03 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7644-20260202-081803\n2026-02-02 08:18:03 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 08:18:03 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 08:18:04 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7645-20260202-081804\n2026-02-02 08:18:04 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 08:18:04 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:42 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 08:18:42 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7646-20260202-081842\n2026-02-02 08:18:42 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 08:18:42 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:18:42 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 08:18:42 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7647-20260202-081842\n2026-02-02 08:18:42 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 08:18:42 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:19:20 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8119, 'inserted': 0, 'updated': 0, 'skipped': 8119, 'errors': 0}\n2026-02-02 08:19:21 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7648-20260202-081921\n2026-02-02 08:19:21 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 08:19:21 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:19:46 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11795, 'inserted': 0, 'updated': 0, 'skipped': 11795, 'errors': 0}\n2026-02-02 08:19:46 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7649-20260202-081946\n2026-02-02 08:19:46 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 08:19:46 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:19:46 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 08:19:46 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7650-20260202-081946\n2026-02-02 08:19:46 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 08:19:46 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:19:47 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 08:19:47 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 08:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7651-20260202-081947\n2026-02-02 08:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 08:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7651-20260202-081947\n2026-02-02 08:19:47 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 04:16:54+08:00 ~ 2026-02-02 10:16:54+08:00]\n2026-02-02 08:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.36s\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.49s\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.20s\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.18s\n2026-02-02 08:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.18s\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.20s\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.24s\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.22s\n2026-02-02 08:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 08:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.37s\n2026-02-02 08:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 08:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.57s\n2026-02-02 08:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 08:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.21s\n2026-02-02 08:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.20s\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.26s\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.26s\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.18s\n2026-02-02 08:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.20s\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.18s\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.23s\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.23s\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.22s\n2026-02-02 08:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.22s\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.20s\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.21s\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.21s\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.21s\n2026-02-02 08:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.26s\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.21s\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.21s\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.20s\n2026-02-02 08:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.20s\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.20s\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.21s\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.22s\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.22s\n2026-02-02 08:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.22s\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.23s\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.22s\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.24s\n2026-02-02 08:19:56 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 08:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.20s\n2026-02-02 08:19:57 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 08:19:57 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.20s\n2026-02-02 08:19:57 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 66, 'skipped': 106}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 58, 'skipped': 114}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 08:19:57 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 08:19:57 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "122f9128", + "executed_at": "2026-02-02T07:16:54.548199", + "status": "success", + "exit_code": 0, + "duration_seconds": 172.430074, + "summary": "【错误】2026-02-02 07:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 07:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1; 2026-02-02 07:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 05:16:54 --window-end 2026-02-02 07:16:54\n2026-02-02 07:16:54 [INFO] etl_billiards: 配置加载完成\n2026-02-02 07:16:54 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 07:16:54 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 07:16:55 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=820216bf246349889a79af066b3d7673\n2026-02-02 07:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7604-20260202-071655\n2026-02-02 07:16:55 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 07:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 07:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7605-20260202-071655\n2026-02-02 07:16:55 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 07:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 07:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7606-20260202-071656\n2026-02-02 07:16:56 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 07:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 07:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7607-20260202-071657\n2026-02-02 07:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00] 未发现需要抓取的小票\n2026-02-02 07:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 07:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7608-20260202-071657\n2026-02-02 07:16:57 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 07:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 07:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7609-20260202-071658\n2026-02-02 07:16:58 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 07:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 07:16:58 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7610-20260202-071658\n2026-02-02 07:16:58 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 07:16:58 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:00 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 07:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7611-20260202-071700\n2026-02-02 07:17:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 07:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 07:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7612-20260202-071701\n2026-02-02 07:17:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 07:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:01 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 07:17:02 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7613-20260202-071701\n2026-02-02 07:17:02 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 07:17:02 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:20 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10159, 'inserted': 0, 'updated': 0, 'skipped': 10159, 'errors': 0}\n2026-02-02 07:17:20 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7614-20260202-071720\n2026-02-02 07:17:20 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 07:17:20 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:27 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1789, 'inserted': 0, 'updated': 0, 'skipped': 1789, 'errors': 0}\n2026-02-02 07:17:27 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7615-20260202-071727\n2026-02-02 07:17:27 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 07:17:27 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:54 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2479, 'inserted': 0, 'updated': 0, 'skipped': 2479, 'errors': 0}\n2026-02-02 07:17:54 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7616-20260202-071754\n2026-02-02 07:17:54 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 07:17:54 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:54 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 07:17:54 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7617-20260202-071754\n2026-02-02 07:17:54 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 07:17:54 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:55 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 07:17:55 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7618-20260202-071755\n2026-02-02 07:17:55 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 07:17:55 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:55 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 07:17:56 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7619-20260202-071756\n2026-02-02 07:17:56 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 07:17:56 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:56 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 07:17:56 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7620-20260202-071756\n2026-02-02 07:17:56 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 07:17:56 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:17:57 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 07:17:57 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7621-20260202-071757\n2026-02-02 07:17:57 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 07:17:57 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:18:36 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 07:18:36 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7622-20260202-071836\n2026-02-02 07:18:36 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 07:18:36 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:18:36 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 7, 'inserted': 0, 'updated': 0, 'skipped': 7, 'errors': 0}\n2026-02-02 07:18:37 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7623-20260202-071837\n2026-02-02 07:18:37 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 07:18:37 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:19:13 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8250, 'inserted': 0, 'updated': 0, 'skipped': 8250, 'errors': 0}\n2026-02-02 07:19:13 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7624-20260202-071913\n2026-02-02 07:19:13 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 07:19:13 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:19:36 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11962, 'inserted': 0, 'updated': 0, 'skipped': 11962, 'errors': 0}\n2026-02-02 07:19:37 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7625-20260202-071937\n2026-02-02 07:19:37 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 07:19:37 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:19:37 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 5, 'inserted': 0, 'updated': 0, 'skipped': 5, 'errors': 0}\n2026-02-02 07:19:37 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7626-20260202-071937\n2026-02-02 07:19:37 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 07:19:37 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:19:38 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 07:19:38 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7627-20260202-071938\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7627-20260202-071938\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 03:16:54+08:00 ~ 2026-02-02 09:16:54+08:00]\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.37s\n2026-02-02 07:19:38 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.42s\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.17s\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.16s\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.17s\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.19s\n2026-02-02 07:19:39 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 07:19:40 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.23s\n2026-02-02 07:19:40 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 07:19:40 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.21s\n2026-02-02 07:19:40 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 07:19:40 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.34s\n2026-02-02 07:19:40 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.54s\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.19s\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.18s\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.19s\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.19s\n2026-02-02 07:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.16s\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.16s\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.16s\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.21s\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.21s\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.20s\n2026-02-02 07:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.20s\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.19s\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.18s\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.19s\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.20s\n2026-02-02 07:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.24s\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.19s\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.18s\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.18s\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.19s\n2026-02-02 07:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.19s\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.19s\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.19s\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.19s\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.19s\n2026-02-02 07:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.18s\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.19s\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.22s\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.18s\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.18s\n2026-02-02 07:19:46 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 07:19:46 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 07:19:46 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "5f89d5b2", + "executed_at": "2026-02-02T06:16:54.223623", + "status": "success", + "exit_code": 0, + "duration_seconds": 177.257074, + "summary": "【错误】2026-02-02 06:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 06:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1; 2026-02-02 06:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 04:16:54 --window-end 2026-02-02 06:16:54\n2026-02-02 06:16:54 [INFO] etl_billiards: 配置加载完成\n2026-02-02 06:16:54 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 06:16:54 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 06:16:54 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=65fa4d91cff94ec6ae937eab98bfc1fa\n2026-02-02 06:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7580-20260202-061654\n2026-02-02 06:16:54 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 06:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 06:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7581-20260202-061655\n2026-02-02 06:16:55 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 06:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:16:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 06:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7582-20260202-061656\n2026-02-02 06:16:56 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 06:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 06:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7583-20260202-061657\n2026-02-02 06:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00] 未发现需要抓取的小票\n2026-02-02 06:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 06:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7584-20260202-061657\n2026-02-02 06:16:57 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 06:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:16:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 06:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7585-20260202-061658\n2026-02-02 06:16:58 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 06:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 06:16:58 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7586-20260202-061658\n2026-02-02 06:16:58 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 06:16:58 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:00 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 06:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7587-20260202-061700\n2026-02-02 06:17:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 06:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 06:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7588-20260202-061701\n2026-02-02 06:17:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 06:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:02 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 06:17:02 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7589-20260202-061702\n2026-02-02 06:17:02 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 06:17:02 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:22 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10159, 'inserted': 0, 'updated': 0, 'skipped': 10159, 'errors': 0}\n2026-02-02 06:17:22 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7590-20260202-061722\n2026-02-02 06:17:22 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 06:17:22 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:29 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1789, 'inserted': 0, 'updated': 0, 'skipped': 1789, 'errors': 0}\n2026-02-02 06:17:29 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7591-20260202-061729\n2026-02-02 06:17:29 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 06:17:29 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:55 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2479, 'inserted': 0, 'updated': 0, 'skipped': 2479, 'errors': 0}\n2026-02-02 06:17:56 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7592-20260202-061756\n2026-02-02 06:17:56 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 06:17:56 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:56 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 06:17:56 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7593-20260202-061756\n2026-02-02 06:17:56 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 06:17:56 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:56 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 06:17:57 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7594-20260202-061757\n2026-02-02 06:17:57 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 06:17:57 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:57 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 06:17:58 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7595-20260202-061758\n2026-02-02 06:17:58 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 06:17:58 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:58 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 06:17:58 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7596-20260202-061758\n2026-02-02 06:17:58 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 06:17:58 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:17:59 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 06:17:59 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7597-20260202-061759\n2026-02-02 06:17:59 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 06:17:59 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:18:38 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 06:18:38 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7598-20260202-061838\n2026-02-02 06:18:38 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 06:18:38 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:18:38 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 7, 'inserted': 0, 'updated': 0, 'skipped': 7, 'errors': 0}\n2026-02-02 06:18:38 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7599-20260202-061838\n2026-02-02 06:18:38 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 06:18:38 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:19:16 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8250, 'inserted': 0, 'updated': 0, 'skipped': 8250, 'errors': 0}\n2026-02-02 06:19:16 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7600-20260202-061916\n2026-02-02 06:19:16 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 06:19:16 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:19:39 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11962, 'inserted': 0, 'updated': 0, 'skipped': 11962, 'errors': 0}\n2026-02-02 06:19:39 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7601-20260202-061939\n2026-02-02 06:19:39 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 06:19:39 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:19:40 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 5, 'inserted': 0, 'updated': 0, 'skipped': 5, 'errors': 0}\n2026-02-02 06:19:40 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7602-20260202-061940\n2026-02-02 06:19:40 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 06:19:40 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:19:41 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 06:19:41 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7603-20260202-061941\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7603-20260202-061941\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 02:16:54+08:00 ~ 2026-02-02 08:16:54+08:00]\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.38s\n2026-02-02 06:19:41 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.53s\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.22s\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.20s\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.20s\n2026-02-02 06:19:42 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.21s\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.27s\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.25s\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.36s\n2026-02-02 06:19:43 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 06:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.58s\n2026-02-02 06:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 06:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.21s\n2026-02-02 06:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 06:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.21s\n2026-02-02 06:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.23s\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.24s\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.19s\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.19s\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 06:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.19s\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.24s\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.25s\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.24s\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.23s\n2026-02-02 06:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.22s\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.22s\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.23s\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.23s\n2026-02-02 06:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.28s\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.23s\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.22s\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.21s\n2026-02-02 06:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.22s\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.26s\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.23s\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.23s\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.23s\n2026-02-02 06:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.22s\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.22s\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.22s\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.25s\n2026-02-02 06:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 06:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.21s\n2026-02-02 06:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 06:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.22s\n2026-02-02 06:19:51 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 06:19:51 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 06:19:51 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "712b956f", + "executed_at": "2026-02-02T05:16:53.863413", + "status": "success", + "exit_code": 0, + "duration_seconds": 181.977561, + "summary": "【错误】2026-02-02 05:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 05:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1; 2026-02-02 05:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 03:16:53 --window-end 2026-02-02 05:16:53\n2026-02-02 05:16:54 [INFO] etl_billiards: 配置加载完成\n2026-02-02 05:16:54 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 05:16:54 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 05:16:54 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=06ffd41655e6498bb8518c7e11618494\n2026-02-02 05:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7556-20260202-051654\n2026-02-02 05:16:54 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 05:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:16:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 05:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7557-20260202-051655\n2026-02-02 05:16:55 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 05:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 05:16:55 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7558-20260202-051655\n2026-02-02 05:16:55 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 05:16:55 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 05:16:56 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7559-20260202-051656\n2026-02-02 05:16:56 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00] 未发现需要抓取的小票\n2026-02-02 05:16:57 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 05:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7560-20260202-051657\n2026-02-02 05:16:57 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 05:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 05:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7561-20260202-051658\n2026-02-02 05:16:58 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 05:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:16:58 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 05:16:58 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7562-20260202-051658\n2026-02-02 05:16:58 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 05:16:58 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:00 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 05:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7563-20260202-051700\n2026-02-02 05:17:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 05:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 10, 'inserted': 0, 'updated': 0, 'skipped': 10, 'errors': 0}\n2026-02-02 05:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7564-20260202-051701\n2026-02-02 05:17:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 05:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:02 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 05:17:02 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7565-20260202-051702\n2026-02-02 05:17:02 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 05:17:02 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:22 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10159, 'inserted': 0, 'updated': 0, 'skipped': 10159, 'errors': 0}\n2026-02-02 05:17:23 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7566-20260202-051723\n2026-02-02 05:17:23 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 05:17:23 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:30 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1789, 'inserted': 0, 'updated': 0, 'skipped': 1789, 'errors': 0}\n2026-02-02 05:17:30 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7567-20260202-051730\n2026-02-02 05:17:30 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 05:17:30 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:56 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2479, 'inserted': 0, 'updated': 0, 'skipped': 2479, 'errors': 0}\n2026-02-02 05:17:56 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7568-20260202-051756\n2026-02-02 05:17:56 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 05:17:56 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:56 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 05:17:57 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7569-20260202-051757\n2026-02-02 05:17:57 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 05:17:57 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:57 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 05:17:57 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7570-20260202-051757\n2026-02-02 05:17:57 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 05:17:57 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:58 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 05:17:58 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7571-20260202-051758\n2026-02-02 05:17:58 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 05:17:58 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:17:58 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 05:17:59 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7572-20260202-051759\n2026-02-02 05:17:59 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 05:17:59 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:18:00 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 0, 'updated': 0, 'skipped': 69, 'errors': 0}\n2026-02-02 05:18:00 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7573-20260202-051800\n2026-02-02 05:18:00 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 05:18:00 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:18:40 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 05:18:41 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7574-20260202-051841\n2026-02-02 05:18:41 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 05:18:41 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:18:41 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 13, 'inserted': 0, 'updated': 0, 'skipped': 13, 'errors': 0}\n2026-02-02 05:18:41 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7575-20260202-051841\n2026-02-02 05:18:41 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 05:18:41 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:19:19 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8250, 'inserted': 0, 'updated': 0, 'skipped': 8250, 'errors': 0}\n2026-02-02 05:19:19 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7576-20260202-051919\n2026-02-02 05:19:19 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 05:19:19 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:19:43 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11962, 'inserted': 0, 'updated': 0, 'skipped': 11962, 'errors': 0}\n2026-02-02 05:19:43 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7577-20260202-051943\n2026-02-02 05:19:43 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 05:19:43 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:19:44 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 10, 'inserted': 0, 'updated': 0, 'skipped': 10, 'errors': 0}\n2026-02-02 05:19:44 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7578-20260202-051944\n2026-02-02 05:19:44 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 05:19:44 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:19:45 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 05:19:45 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7579-20260202-051945\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7579-20260202-051945\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 01:16:53+08:00 ~ 2026-02-02 07:16:53+08:00]\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.42s\n2026-02-02 05:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.51s\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.22s\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.21s\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.21s\n2026-02-02 05:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.22s\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.26s\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.24s\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.37s\n2026-02-02 05:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 05:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.60s\n2026-02-02 05:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 05:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.22s\n2026-02-02 05:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.22s\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.24s\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.23s\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.20s\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.21s\n2026-02-02 05:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.21s\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.26s\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.25s\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.29s\n2026-02-02 05:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.26s\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.23s\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.23s\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.24s\n2026-02-02 05:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.24s\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.29s\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.23s\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.22s\n2026-02-02 05:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.22s\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.23s\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.23s\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.24s\n2026-02-02 05:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.24s\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.24s\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.24s\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.23s\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.23s\n2026-02-02 05:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 05:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.26s\n2026-02-02 05:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 05:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.22s\n2026-02-02 05:19:55 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 05:19:55 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.22s\n2026-02-02 05:19:55 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 05:19:55 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 05:19:55 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "b1d05764", + "executed_at": "2026-02-02T04:16:53.349381", + "status": "success", + "exit_code": 0, + "duration_seconds": 181.269416, + "summary": "【DWD 装载】维表新增: 0条, 维表更新: 18条, 事实表新增: 36条, 事实表更新: 0条\n 维表: dim_table: +0, ~6, dim_assistant: +0, ~5, dim_member: +0, ~2, dim_store_goods: +0, ~5\n 事实表: dwd_settlement_head: +5, ~0, dwd_table_fee_log: +7, ~0, dwd_table_fee_adjust: +4, ~0, dwd_assistant_service_log: +9, ~0, dwd_member_balance_change: +4, ~0, dwd_recharge_order: +1, ~0, dwd_payment: +6, ~0\n【错误】2026-02-02 04:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 04:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1; 2026-02-02 04:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 02:16:53 --window-end 2026-02-02 04:16:53\n2026-02-02 04:16:53 [INFO] etl_billiards: 配置加载完成\n2026-02-02 04:16:53 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 04:16:53 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 04:16:54 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=218ed57fad4947a0a7d7f808018b2b80\n2026-02-02 04:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7532-20260202-041654\n2026-02-02 04:16:54 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 04:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:16:54 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 04:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7533-20260202-041655\n2026-02-02 04:16:55 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 04:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:16:55 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 04:16:55 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7534-20260202-041655\n2026-02-02 04:16:55 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 04:16:55 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:16:56 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 04:16:56 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7535-20260202-041656\n2026-02-02 04:16:56 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:16:56 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00] 未发现需要抓取的小票\n2026-02-02 04:16:56 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 04:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7536-20260202-041657\n2026-02-02 04:16:57 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 04:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:16:57 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 04:16:57 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7537-20260202-041657\n2026-02-02 04:16:57 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 04:16:57 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:16:57 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 04:16:58 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7538-20260202-041658\n2026-02-02 04:16:58 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 04:16:58 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:00 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 2, 'updated': 0, 'skipped': 554, 'errors': 0}\n2026-02-02 04:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7539-20260202-041700\n2026-02-02 04:17:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 04:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:00 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 10, 'inserted': 9, 'updated': 0, 'skipped': 1, 'errors': 0}\n2026-02-02 04:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7540-20260202-041701\n2026-02-02 04:17:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 04:17:01 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:01 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 5, 'updated': 0, 'skipped': 167, 'errors': 0}\n2026-02-02 04:17:02 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7541-20260202-041702\n2026-02-02 04:17:02 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 04:17:02 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:22 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10159, 'inserted': 7, 'updated': 0, 'skipped': 10152, 'errors': 0}\n2026-02-02 04:17:22 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7542-20260202-041722\n2026-02-02 04:17:22 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 04:17:22 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:29 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1789, 'inserted': 4, 'updated': 0, 'skipped': 1785, 'errors': 0}\n2026-02-02 04:17:29 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7543-20260202-041729\n2026-02-02 04:17:29 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 04:17:29 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:57 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2479, 'inserted': 4, 'updated': 0, 'skipped': 2475, 'errors': 0}\n2026-02-02 04:17:57 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7544-20260202-041757\n2026-02-02 04:17:57 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 04:17:57 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:57 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 6, 'updated': 0, 'skipped': 68, 'errors': 0}\n2026-02-02 04:17:58 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7545-20260202-041758\n2026-02-02 04:17:58 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 04:17:58 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:58 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 04:17:58 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7546-20260202-041758\n2026-02-02 04:17:58 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 04:17:58 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:59 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 4, 'updated': 0, 'skipped': 168, 'errors': 0}\n2026-02-02 04:17:59 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7547-20260202-041759\n2026-02-02 04:17:59 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 04:17:59 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:17:59 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 04:18:00 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7548-20260202-041800\n2026-02-02 04:18:00 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 04:18:00 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:18:01 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 5, 'updated': 0, 'skipped': 64, 'errors': 0}\n2026-02-02 04:18:01 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7549-20260202-041801\n2026-02-02 04:18:01 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 04:18:01 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:18:40 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 04:18:40 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7550-20260202-041840\n2026-02-02 04:18:40 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 04:18:40 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:18:40 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 18, 'inserted': 6, 'updated': 0, 'skipped': 12, 'errors': 0}\n2026-02-02 04:18:40 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7551-20260202-041840\n2026-02-02 04:18:40 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 04:18:40 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:19:18 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8250, 'inserted': 0, 'updated': 0, 'skipped': 8250, 'errors': 0}\n2026-02-02 04:19:18 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7552-20260202-041918\n2026-02-02 04:19:18 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 04:19:18 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:19:42 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11962, 'inserted': 6, 'updated': 0, 'skipped': 11956, 'errors': 0}\n2026-02-02 04:19:42 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7553-20260202-041942\n2026-02-02 04:19:42 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 04:19:42 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:19:42 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 12, 'inserted': 5, 'updated': 0, 'skipped': 7, 'errors': 0}\n2026-02-02 04:19:43 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7554-20260202-041943\n2026-02-02 04:19:43 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 04:19:43 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:19:43 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 04:19:43 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 整数超出范围\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.NumericValueOutOfRange: 错误: 整数超出范围\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7555-20260202-041944\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7555-20260202-041944\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-02 00:16:53+08:00 ~ 2026-02-02 06:16:53+08:00]\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.39s\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.46s\n2026-02-02 04:19:44 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.27s\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.26s\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.25s\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.26s\n2026-02-02 04:19:45 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 04:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.31s\n2026-02-02 04:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 04:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.28s\n2026-02-02 04:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 04:19:46 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.41s\n2026-02-02 04:19:46 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 04:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.57s\n2026-02-02 04:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 04:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.21s\n2026-02-02 04:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 04:19:47 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.23s\n2026-02-02 04:19:47 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.29s\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.29s\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.19s\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.19s\n2026-02-02 04:19:48 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.19s\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.25s\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.25s\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.24s\n2026-02-02 04:19:49 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.23s\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.22s\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.22s\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.23s\n2026-02-02 04:19:50 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.23s\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.28s\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.23s\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.23s\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.21s\n2026-02-02 04:19:51 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.22s\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.22s\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.23s\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.23s\n2026-02-02 04:19:52 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.23s\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.23s\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.22s\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.23s\n2026-02-02 04:19:53 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 04:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.25s\n2026-02-02 04:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 04:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.21s\n2026-02-02 04:19:54 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 04:19:54 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.21s\n2026-02-02 04:19:54 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 6, 'skipped': 68}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 5, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 5, 'skipped': 64}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 5, 'skipped': 64}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 2, 'skipped': 554}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 2, 'skipped': 554}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 5, 'skipped': 167}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 5, 'skipped': 167}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 5, 'updated': 0, 'processed': 5}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 5, 'updated': 0, 'processed': 5}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 7, 'updated': 0, 'processed': 7}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 9, 'updated': 0, 'processed': 9}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 9, 'updated': 0, 'processed': 9}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 4, 'updated': 0, 'processed': 4}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 1, 'updated': 0, 'processed': 1}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 6, 'updated': 0, 'processed': 6}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 04:19:54 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 04:19:54 [INFO] etl_billiards: ETL运行完成\n", + "error": "" + }, + { + "task_id": "b3edf6e8", + "executed_at": "2026-02-02T03:15:55.288621", + "status": "success", + "exit_code": 0, + "duration_seconds": 194.682314, + "summary": "【错误】2026-02-02 03:15:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0,; 2026-02-02 03:15:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1; 2026-02-02 03:15:57 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0,\n【结果】✓ etl_billiards: ETL运行完成", + "output": "[工作目录] C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\n[执行命令] python -m cli.main --tasks ODS_GOODS_CATEGORY,ODS_RECHARGE_SETTLE,ODS_TENANT_GOODS,ODS_SETTLEMENT_TICKET,ODS_GROUP_PACKAGE,ODS_ASSISTANT_ABOLISH,ODS_MEMBER,ODS_ASSISTANT_LEDGER,ODS_STORE_GOODS,ODS_TABLE_USE,ODS_TABLE_FEE_DISCOUNT,ODS_MEMBER_BALANCE,ODS_TABLES,ODS_STORE_GOODS_SALES,ODS_INVENTORY_STOCK,ODS_REFUND,ODS_ASSISTANT_ACCOUNT,ODS_PLATFORM_COUPON,ODS_INVENTORY_CHANGE,ODS_GROUP_BUY_REDEMPTION,ODS_PAYMENT,ODS_SETTLEMENT_RECORDS,ODS_MEMBER_CARD,DWD_LOAD_FROM_ODS --pipeline-flow FULL --window-start 2026-02-02 01:15:55 --window-end 2026-02-02 03:15:55\n2026-02-02 03:15:55 [INFO] etl_billiards: 配置加载完成\n2026-02-02 03:15:55 [INFO] etl_billiards: 门店ID: 2790685415443269\n2026-02-02 03:15:55 [INFO] etl_billiards: 任务列表: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS']\n2026-02-02 03:15:55 [INFO] etl_billiards: 开始运行任务: ['ODS_GOODS_CATEGORY', 'ODS_RECHARGE_SETTLE', 'ODS_TENANT_GOODS', 'ODS_SETTLEMENT_TICKET', 'ODS_GROUP_PACKAGE', 'ODS_ASSISTANT_ABOLISH', 'ODS_MEMBER', 'ODS_ASSISTANT_LEDGER', 'ODS_STORE_GOODS', 'ODS_TABLE_USE', 'ODS_TABLE_FEE_DISCOUNT', 'ODS_MEMBER_BALANCE', 'ODS_TABLES', 'ODS_STORE_GOODS_SALES', 'ODS_INVENTORY_STOCK', 'ODS_REFUND', 'ODS_ASSISTANT_ACCOUNT', 'ODS_PLATFORM_COUPON', 'ODS_INVENTORY_CHANGE', 'ODS_GROUP_BUY_REDEMPTION', 'ODS_PAYMENT', 'ODS_SETTLEMENT_RECORDS', 'ODS_MEMBER_CARD', 'DWD_LOAD_FROM_ODS'], run_uuid=25ea68a0cad7475facb2f3f2de262d3f\n2026-02-02 03:15:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: ODS fetch+load start, dir=export\\JSON\\ODS_GOODS_CATEGORY\\ODS_GOODS_CATEGORY-7508-20260202-031555\n2026-02-02 03:15:55 [INFO] etl_billiards: 开始执行ODS_GOODS_CATEGORY (ODS)\n2026-02-02 03:15:55 [INFO] etl_billiards: ODS_GOODS_CATEGORY: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:15:56 [INFO] etl_billiards: ODS_GOODS_CATEGORY ODS 任务完成: {'fetched': 9, 'inserted': 0, 'updated': 0, 'skipped': 9, 'errors': 0}\n2026-02-02 03:15:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: ODS fetch+load start, dir=export\\JSON\\ODS_RECHARGE_SETTLE\\ODS_RECHARGE_SETTLE-7509-20260202-031556\n2026-02-02 03:15:56 [INFO] etl_billiards: 开始执行ODS_RECHARGE_SETTLE (ODS)\n2026-02-02 03:15:56 [INFO] etl_billiards: ODS_RECHARGE_SETTLE: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:15:57 [INFO] etl_billiards: ODS_RECHARGE_SETTLE ODS 任务完成: {'fetched': 1, 'inserted': 1, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 03:15:57 [INFO] etl_billiards: ODS_TENANT_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_TENANT_GOODS\\ODS_TENANT_GOODS-7510-20260202-031557\n2026-02-02 03:15:57 [INFO] etl_billiards: 开始执行ODS_TENANT_GOODS (ODS)\n2026-02-02 03:15:57 [INFO] etl_billiards: ODS_TENANT_GOODS: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:15:57 [INFO] etl_billiards: ODS_TENANT_GOODS ODS 任务完成: {'fetched': 173, 'inserted': 0, 'updated': 0, 'skipped': 173, 'errors': 0}\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_TICKET\\ODS_SETTLEMENT_TICKET-7511-20260202-031558\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00] 未发现需要抓取的小票\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_SETTLEMENT_TICKET: 小票抓取完成，抓取=0 插入=0 更新=0 跳过=0\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_PACKAGE\\ODS_GROUP_PACKAGE-7512-20260202-031558\n2026-02-02 03:15:58 [INFO] etl_billiards: 开始执行ODS_GROUP_PACKAGE (ODS)\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:15:58 [INFO] etl_billiards: ODS_GROUP_PACKAGE ODS 任务完成: {'fetched': 18, 'inserted': 0, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 03:15:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ABOLISH\\ODS_ASSISTANT_ABOLISH-7513-20260202-031559\n2026-02-02 03:15:59 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ABOLISH (ODS)\n2026-02-02 03:15:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:15:59 [INFO] etl_billiards: ODS_ASSISTANT_ABOLISH ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 03:15:59 [INFO] etl_billiards: ODS_MEMBER: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER\\ODS_MEMBER-7514-20260202-031559\n2026-02-02 03:15:59 [INFO] etl_billiards: 开始执行ODS_MEMBER (ODS)\n2026-02-02 03:15:59 [INFO] etl_billiards: ODS_MEMBER: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:16:01 [INFO] etl_billiards: ODS_MEMBER ODS 任务完成: {'fetched': 556, 'inserted': 0, 'updated': 0, 'skipped': 556, 'errors': 0}\n2026-02-02 03:16:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_LEDGER\\ODS_ASSISTANT_LEDGER-7515-20260202-031601\n2026-02-02 03:16:01 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_LEDGER (ODS)\n2026-02-02 03:16:01 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:16:02 [INFO] etl_billiards: ODS_ASSISTANT_LEDGER ODS 任务完成: {'fetched': 4, 'inserted': 0, 'updated': 0, 'skipped': 4, 'errors': 0}\n2026-02-02 03:16:02 [INFO] etl_billiards: ODS_STORE_GOODS: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS\\ODS_STORE_GOODS-7516-20260202-031602\n2026-02-02 03:16:02 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS (ODS)\n2026-02-02 03:16:02 [INFO] etl_billiards: ODS_STORE_GOODS: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:16:03 [INFO] etl_billiards: ODS_STORE_GOODS ODS 任务完成: {'fetched': 172, 'inserted': 0, 'updated': 0, 'skipped': 172, 'errors': 0}\n2026-02-02 03:16:03 [INFO] etl_billiards: ODS_TABLE_USE: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_USE\\ODS_TABLE_USE-7517-20260202-031603\n2026-02-02 03:16:03 [INFO] etl_billiards: 开始执行ODS_TABLE_USE (ODS)\n2026-02-02 03:16:03 [INFO] etl_billiards: ODS_TABLE_USE: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:16:25 [INFO] etl_billiards: ODS_TABLE_USE ODS 任务完成: {'fetched': 10152, 'inserted': 0, 'updated': 0, 'skipped': 10152, 'errors': 0}\n2026-02-02 03:16:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_TABLE_FEE_DISCOUNT\\ODS_TABLE_FEE_DISCOUNT-7518-20260202-031625\n2026-02-02 03:16:25 [INFO] etl_billiards: 开始执行ODS_TABLE_FEE_DISCOUNT (ODS)\n2026-02-02 03:16:25 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:16:32 [INFO] etl_billiards: ODS_TABLE_FEE_DISCOUNT ODS 任务完成: {'fetched': 1785, 'inserted': 0, 'updated': 0, 'skipped': 1785, 'errors': 0}\n2026-02-02 03:16:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_BALANCE\\ODS_MEMBER_BALANCE-7519-20260202-031632\n2026-02-02 03:16:32 [INFO] etl_billiards: 开始执行ODS_MEMBER_BALANCE (ODS)\n2026-02-02 03:16:32 [INFO] etl_billiards: ODS_MEMBER_BALANCE: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:00 [INFO] etl_billiards: ODS_MEMBER_BALANCE ODS 任务完成: {'fetched': 2475, 'inserted': 0, 'updated': 0, 'skipped': 2475, 'errors': 0}\n2026-02-02 03:17:00 [INFO] etl_billiards: ODS_TABLES: ODS fetch+load start, dir=export\\JSON\\ODS_TABLES\\ODS_TABLES-7520-20260202-031700\n2026-02-02 03:17:00 [INFO] etl_billiards: 开始执行ODS_TABLES (ODS)\n2026-02-02 03:17:00 [INFO] etl_billiards: ODS_TABLES: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:01 [INFO] etl_billiards: ODS_TABLES ODS 任务完成: {'fetched': 74, 'inserted': 0, 'updated': 0, 'skipped': 74, 'errors': 0}\n2026-02-02 03:17:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: ODS fetch+load start, dir=export\\JSON\\ODS_STORE_GOODS_SALES\\ODS_STORE_GOODS_SALES-7521-20260202-031701\n2026-02-02 03:17:01 [INFO] etl_billiards: 开始执行ODS_STORE_GOODS_SALES (ODS)\n2026-02-02 03:17:01 [INFO] etl_billiards: ODS_STORE_GOODS_SALES: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:02 [INFO] etl_billiards: ODS_STORE_GOODS_SALES ODS 任务完成: {'fetched': 0, 'inserted': 0, 'updated': 0, 'skipped': 0, 'errors': 0}\n2026-02-02 03:17:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_STOCK\\ODS_INVENTORY_STOCK-7522-20260202-031702\n2026-02-02 03:17:02 [INFO] etl_billiards: 开始执行ODS_INVENTORY_STOCK (ODS)\n2026-02-02 03:17:02 [INFO] etl_billiards: ODS_INVENTORY_STOCK: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:03 [INFO] etl_billiards: ODS_INVENTORY_STOCK ODS 任务完成: {'fetched': 172, 'inserted': 1, 'updated': 0, 'skipped': 171, 'errors': 0}\n2026-02-02 03:17:03 [INFO] etl_billiards: ODS_REFUND: ODS fetch+load start, dir=export\\JSON\\ODS_REFUND\\ODS_REFUND-7523-20260202-031703\n2026-02-02 03:17:03 [INFO] etl_billiards: 开始执行ODS_REFUND (ODS)\n2026-02-02 03:17:03 [INFO] etl_billiards: ODS_REFUND: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:04 [INFO] etl_billiards: ODS_REFUND ODS 任务完成: {'fetched': 40, 'inserted': 0, 'updated': 0, 'skipped': 40, 'errors': 0}\n2026-02-02 03:17:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: ODS fetch+load start, dir=export\\JSON\\ODS_ASSISTANT_ACCOUNT\\ODS_ASSISTANT_ACCOUNT-7524-20260202-031704\n2026-02-02 03:17:04 [INFO] etl_billiards: 开始执行ODS_ASSISTANT_ACCOUNT (ODS)\n2026-02-02 03:17:04 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:06 [INFO] etl_billiards: ODS_ASSISTANT_ACCOUNT ODS 任务完成: {'fetched': 69, 'inserted': 1, 'updated': 0, 'skipped': 68, 'errors': 0}\n2026-02-02 03:17:06 [INFO] etl_billiards: ODS_PLATFORM_COUPON: ODS fetch+load start, dir=export\\JSON\\ODS_PLATFORM_COUPON\\ODS_PLATFORM_COUPON-7525-20260202-031706\n2026-02-02 03:17:06 [INFO] etl_billiards: 开始执行ODS_PLATFORM_COUPON (ODS)\n2026-02-02 03:17:06 [INFO] etl_billiards: ODS_PLATFORM_COUPON: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:49 [INFO] etl_billiards: ODS_PLATFORM_COUPON ODS 任务完成: {'fetched': 17473, 'inserted': 0, 'updated': 0, 'skipped': 17473, 'errors': 0}\n2026-02-02 03:17:49 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: ODS fetch+load start, dir=export\\JSON\\ODS_INVENTORY_CHANGE\\ODS_INVENTORY_CHANGE-7526-20260202-031749\n2026-02-02 03:17:49 [INFO] etl_billiards: 开始执行ODS_INVENTORY_CHANGE (ODS)\n2026-02-02 03:17:49 [INFO] etl_billiards: ODS_INVENTORY_CHANGE: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:17:49 [INFO] etl_billiards: ODS_INVENTORY_CHANGE ODS 任务完成: {'fetched': 19, 'inserted': 1, 'updated': 0, 'skipped': 18, 'errors': 0}\n2026-02-02 03:17:49 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: ODS fetch+load start, dir=export\\JSON\\ODS_GROUP_BUY_REDEMPTION\\ODS_GROUP_BUY_REDEMPTION-7527-20260202-031749\n2026-02-02 03:17:49 [INFO] etl_billiards: 开始执行ODS_GROUP_BUY_REDEMPTION (ODS)\n2026-02-02 03:17:49 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:18:31 [INFO] etl_billiards: ODS_GROUP_BUY_REDEMPTION ODS 任务完成: {'fetched': 8250, 'inserted': 0, 'updated': 0, 'skipped': 8250, 'errors': 0}\n2026-02-02 03:18:31 [INFO] etl_billiards: ODS_PAYMENT: ODS fetch+load start, dir=export\\JSON\\ODS_PAYMENT\\ODS_PAYMENT-7528-20260202-031831\n2026-02-02 03:18:31 [INFO] etl_billiards: 开始执行ODS_PAYMENT (ODS)\n2026-02-02 03:18:31 [INFO] etl_billiards: ODS_PAYMENT: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:18:58 [INFO] etl_billiards: ODS_PAYMENT ODS 任务完成: {'fetched': 11956, 'inserted': 0, 'updated': 0, 'skipped': 11956, 'errors': 0}\n2026-02-02 03:18:58 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: ODS fetch+load start, dir=export\\JSON\\ODS_SETTLEMENT_RECORDS\\ODS_SETTLEMENT_RECORDS-7529-20260202-031858\n2026-02-02 03:18:58 [INFO] etl_billiards: 开始执行ODS_SETTLEMENT_RECORDS (ODS)\n2026-02-02 03:18:58 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:18:58 [INFO] etl_billiards: ODS_SETTLEMENT_RECORDS ODS 任务完成: {'fetched': 15, 'inserted': 0, 'updated': 0, 'skipped': 15, 'errors': 0}\n2026-02-02 03:18:58 [INFO] etl_billiards: ODS_MEMBER_CARD: ODS fetch+load start, dir=export\\JSON\\ODS_MEMBER_CARD\\ODS_MEMBER_CARD-7530-20260202-031858\n2026-02-02 03:18:58 [INFO] etl_billiards: 开始执行ODS_MEMBER_CARD (ODS)\n2026-02-02 03:18:58 [INFO] etl_billiards: ODS_MEMBER_CARD: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:19:00 [ERROR] etl_billiards: ODS_MEMBER_CARD ODS 任务失败\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.DatatypeMismatch: 错误: 字段 \"able_share_member_discount\" 的类型为 boolean, 但表达式的类型为 integer\nLINE 1: ...9df46f9b236457e1d44e86ab2d7e0d93209dc530a51998e5',1,100.0,10...\n ^\nHINT: 你需要重写或转换表达式\n2026-02-02 03:19:00 [ERROR] etl_billiards: 任务 ODS_MEMBER_CARD 失败: 错误: 字段 \"able_share_member_discount\" 的类型为 boolean, 但表达式的类型为 integer\nLINE 1: ...9df46f9b236457e1d44e86ab2d7e0d93209dc530a51998e5',1,100.0,10...\n ^\nHINT: 你需要重写或转换表达式\nTraceback (most recent call last):\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 112, in run_tasks\n self._run_single_task(task_code, run_uuid, store_id)\n ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 168, in _run_single_task\n result, _ = self._execute_ods_record_and_load(task_code, cursor_data, fetch_dir, run_id)\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\orchestration\\scheduler.py\", line 297, in _execute_ods_record_and_load\n result = task.execute(cursor_data)\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 124, in execute\n inserted, skipped = self._insert_records_schema_aware(\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^\n table=spec.table_name,\n ^^^^^^^^^^^^^^^^^^^^^^\n ...<3 lines>...\n source_endpoint=spec.endpoint if spec.include_source_endpoint else None,\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n )\n ^\n File \"C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\tasks\\ods_tasks.py\", line 462, in _insert_records_schema_aware\n execute_values(cur, sql, chunk, page_size=len(chunk))\n ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n File \"C:\\ProgramData\\miniconda3\\Lib\\site-packages\\psycopg2\\extras.py\", line 1299, in execute_values\n cur.execute(b''.join(parts))\n ~~~~~~~~~~~^^^^^^^^^^^^^^^^^\npsycopg2.errors.DatatypeMismatch: 错误: 字段 \"able_share_member_discount\" 的类型为 boolean, 但表达式的类型为 integer\nLINE 1: ...9df46f9b236457e1d44e86ab2d7e0d93209dc530a51998e5',1,100.0,10...\n ^\nHINT: 你需要重写或转换表达式\n2026-02-02 03:19:00 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取阶段开始，目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7531-20260202-031900\n2026-02-02 03:19:00 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 抓取完成，文件=None，记录数=0\n2026-02-02 03:19:00 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 本地清洗入库开始，源目录=export\\JSON\\DWD_LOAD_FROM_ODS\\DWD_LOAD_FROM_ODS-7531-20260202-031900\n2026-02-02 03:19:00 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 开始执行(1/1)，窗口[2026-02-01 23:15:55+08:00 ~ 2026-02-02 05:15:55+08:00]\n2026-02-02 03:19:00 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site <= billiards_ods.table_fee_transactions\n2026-02-02 03:19:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site，用时 0.38s\n2026-02-02 03:19:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_site_ex <= billiards_ods.table_fee_transactions\n2026-02-02 03:19:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_site_ex，用时 0.51s\n2026-02-02 03:19:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table <= billiards_ods.site_tables_master\n2026-02-02 03:19:01 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table，用时 0.17s\n2026-02-02 03:19:01 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_table_ex <= billiards_ods.site_tables_master\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_table_ex，用时 0.17s\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant <= billiards_ods.assistant_accounts_master\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant，用时 0.18s\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_assistant_ex <= billiards_ods.assistant_accounts_master\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_assistant_ex，用时 0.23s\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member <= billiards_ods.member_profiles\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member，用时 0.25s\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_ex <= billiards_ods.member_profiles\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_ex，用时 0.21s\n2026-02-02 03:19:02 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account <= billiards_ods.member_stored_value_cards\n2026-02-02 03:19:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account，用时 0.33s\n2026-02-02 03:19:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_member_card_account_ex <= billiards_ods.member_stored_value_cards\n2026-02-02 03:19:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_member_card_account_ex，用时 0.56s\n2026-02-02 03:19:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods <= billiards_ods.tenant_goods_master\n2026-02-02 03:19:03 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods，用时 0.20s\n2026-02-02 03:19:03 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_tenant_goods_ex <= billiards_ods.tenant_goods_master\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_tenant_goods_ex，用时 0.19s\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods <= billiards_ods.store_goods_master\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods，用时 0.20s\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_store_goods_ex <= billiards_ods.store_goods_master\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_store_goods_ex，用时 0.21s\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_goods_category <= billiards_ods.stock_goods_category_tree\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_goods_category，用时 0.17s\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package <= billiards_ods.group_buy_packages\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package，用时 0.18s\n2026-02-02 03:19:04 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dim_groupbuy_package_ex <= billiards_ods.group_buy_packages\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dim_groupbuy_package_ex，用时 0.17s\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head <= billiards_ods.settlement_records\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head，用时 0.22s\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_settlement_head_ex <= billiards_ods.settlement_records\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_settlement_head_ex，用时 0.23s\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log <= billiards_ods.table_fee_transactions\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log，用时 0.21s\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_log_ex <= billiards_ods.table_fee_transactions\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_log_ex，用时 0.21s\n2026-02-02 03:19:05 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust <= billiards_ods.table_fee_discount_records\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust，用时 0.20s\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_table_fee_adjust_ex <= billiards_ods.table_fee_discount_records\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_table_fee_adjust_ex，用时 0.19s\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale <= billiards_ods.store_goods_sales_records\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale，用时 0.21s\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_store_goods_sale_ex <= billiards_ods.store_goods_sales_records\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_store_goods_sale_ex，用时 0.20s\n2026-02-02 03:19:06 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log <= billiards_ods.assistant_service_records\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log，用时 0.24s\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_service_log_ex <= billiards_ods.assistant_service_records\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_service_log_ex，用时 0.20s\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event <= billiards_ods.assistant_cancellation_records\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event，用时 0.19s\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_assistant_trash_event_ex <= billiards_ods.assistant_cancellation_records\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_assistant_trash_event_ex，用时 0.20s\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change <= billiards_ods.member_balance_changes\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change，用时 0.19s\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_member_balance_change_ex <= billiards_ods.member_balance_changes\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_member_balance_change_ex，用时 0.20s\n2026-02-02 03:19:07 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption <= billiards_ods.group_buy_redemption_records\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption，用时 0.20s\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_groupbuy_redemption_ex <= billiards_ods.group_buy_redemption_records\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_groupbuy_redemption_ex，用时 0.20s\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption，用时 0.20s\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_platform_coupon_redemption_ex <= billiards_ods.platform_coupon_redemption_records\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_platform_coupon_redemption_ex，用时 0.20s\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order <= billiards_ods.recharge_settlements\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order，用时 0.20s\n2026-02-02 03:19:08 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_recharge_order_ex <= billiards_ods.recharge_settlements\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_recharge_order_ex，用时 0.21s\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_payment <= billiards_ods.payment_transactions\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_payment，用时 0.22s\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund <= billiards_ods.refund_transactions\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund，用时 0.19s\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载开始：billiards_dwd.dwd_refund_ex <= billiards_ods.refund_transactions\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD 装载完成：billiards_dwd.dwd_refund_ex，用时 0.19s\n2026-02-02 03:19:09 [INFO] etl_billiards: DWD_LOAD_FROM_ODS: 完成，统计={'tables': [{'table': 'billiards_dwd.dim_site', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_site_ex', 'mode': 'SCD2', 'processed': 1, 'inserted': 0, 'updated': 0, 'skipped': 1}, {'table': 'billiards_dwd.dim_table', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_table_ex', 'mode': 'SCD2', 'processed': 74, 'inserted': 0, 'updated': 0, 'skipped': 74}, {'table': 'billiards_dwd.dim_assistant', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 0, 'skipped': 69}, {'table': 'billiards_dwd.dim_assistant_ex', 'mode': 'SCD2', 'processed': 69, 'inserted': 0, 'updated': 1, 'skipped': 68}, {'table': 'billiards_dwd.dim_member', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_ex', 'mode': 'SCD2', 'processed': 556, 'inserted': 0, 'updated': 0, 'skipped': 556}, {'table': 'billiards_dwd.dim_member_card_account', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_member_card_account_ex', 'mode': 'SCD2', 'processed': 945, 'inserted': 0, 'updated': 0, 'skipped': 945}, {'table': 'billiards_dwd.dim_tenant_goods', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_tenant_goods_ex', 'mode': 'SCD2', 'processed': 173, 'inserted': 0, 'updated': 0, 'skipped': 173}, {'table': 'billiards_dwd.dim_store_goods', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_store_goods_ex', 'mode': 'SCD2', 'processed': 172, 'inserted': 0, 'updated': 0, 'skipped': 172}, {'table': 'billiards_dwd.dim_goods_category', 'mode': 'SCD2', 'processed': 26, 'inserted': 0, 'updated': 0, 'skipped': 26}, {'table': 'billiards_dwd.dim_groupbuy_package', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dim_groupbuy_package_ex', 'mode': 'SCD2', 'processed': 34, 'inserted': 0, 'updated': 0, 'skipped': 34}, {'table': 'billiards_dwd.dwd_settlement_head', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_settlement_head_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_table_fee_adjust_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_store_goods_sale_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_service_log_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_assistant_trash_event_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_member_balance_change_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_groupbuy_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_platform_coupon_redemption_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_recharge_order_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_payment', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}, {'table': 'billiards_dwd.dwd_refund_ex', 'mode': 'INCREMENT', 'inserted': 0, 'updated': 0, 'processed': 0}], 'errors': []}\n2026-02-02 03:19:09 [INFO] etl_billiards: 所有任务执行完成\n2026-02-02 03:19:09 [INFO] etl_billiards: ETL运行完成\n", "error": "" } ], - "created_at": "2026-02-01T23:27:58.777762", - "updated_at": "2026-02-01T23:31:52.514792" + "created_at": "2026-02-02T03:15:21.512369", + "updated_at": "2026-02-03T17:30:44.801339" } } } \ No newline at end of file diff --git a/etl_billiards/scripts/analyze_discount_patterns.py b/etl_billiards/scripts/analyze_discount_patterns.py new file mode 100644 index 0000000..cfb4fc6 --- /dev/null +++ b/etl_billiards/scripts/analyze_discount_patterns.py @@ -0,0 +1,636 @@ +# -*- coding: utf-8 -*- +""" +优惠口径抽样分析脚本 + +功能说明： + 从dwd_settlement_head表抽样100单，分析以下优惠字段的使用情况： + - adjust_amount: 台费打折/调整（可能包含大客户优惠、其他优惠） + - member_discount_amount: 会员折扣 + - rounding_amount: 抹零金额 + - coupon_amount: 团购抵消台费 + - gift_card_amount: 赠送卡支付 + +分析目标： + 1. 大客户优惠：是否存在"大客户"标识？如何与普通调整区分？ + 2. 会员折扣：是否有非零值？使用场景是什么？ + 3. 抹零：抹零规则？与adjust_amount的关系？ + 4. 其他优惠：adjust_amount中还包含哪些优惠类型？ + +输出： + - 控制台打印分析报告 + - 生成 docs/analysis_discount_patterns.md 报告文件 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +import os +import sys +from datetime import datetime +from decimal import Decimal +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent.parent.parent +sys.path.insert(0, str(project_root)) + +from etl_billiards.utils.config import Config +from etl_billiards.utils.db import DatabaseConnection + + +def analyze_discount_patterns(): + """ + 执行优惠口径抽样分析 + """ + print("=" * 80) + print("优惠口径抽样分析") + print("=" * 80) + print() + + # 加载配置和数据库连接 + config = Config() + db = DatabaseConnection(config) + + try: + # 1. 获取总体统计 + print("【1. 总体统计】") + print("-" * 40) + overall_stats = get_overall_stats(db) + print_overall_stats(overall_stats) + print() + + # 2. 抽样分析优惠订单 + print("【2. 有优惠的订单抽样分析（100单）】") + print("-" * 40) + sample_orders = get_sample_orders_with_discount(db, limit=100) + discount_analysis = analyze_sample_orders(sample_orders) + print_discount_analysis(discount_analysis) + print() + + # 3. adjust_amount详细分析 + print("【3. adjust_amount (台费打折/调整) 详细分析】") + print("-" * 40) + adjust_analysis = analyze_adjust_amount(db) + print_adjust_analysis(adjust_analysis) + print() + + # 4. 会员折扣使用分析 + print("【4. member_discount_amount (会员折扣) 使用分析】") + print("-" * 40) + member_discount_analysis = analyze_member_discount(db) + print_member_discount_analysis(member_discount_analysis) + print() + + # 5. 抹零规则分析 + print("【5. rounding_amount (抹零) 规则分析】") + print("-" * 40) + rounding_analysis = analyze_rounding(db) + print_rounding_analysis(rounding_analysis) + print() + + # 6. 团购优惠分析 + print("【6. 团购优惠分析】") + print("-" * 40) + groupbuy_analysis = analyze_groupbuy(db) + print_groupbuy_analysis(groupbuy_analysis) + print() + + # 7. 生成分析报告 + print("【7. 生成分析报告】") + print("-" * 40) + report = generate_report( + overall_stats, + discount_analysis, + adjust_analysis, + member_discount_analysis, + rounding_analysis, + groupbuy_analysis + ) + + # 保存报告 + report_path = project_root / "etl_billiards" / "docs" / "analysis_discount_patterns.md" + with open(report_path, 'w', encoding='utf-8') as f: + f.write(report) + print(f"报告已保存到: {report_path}") + + finally: + db.close() + + +def get_overall_stats(db: DatabaseConnection) -> Dict[str, Any]: + """ + 获取总体统计数据 + """ + sql = """ + SELECT + COUNT(*) AS total_orders, + COUNT(CASE WHEN adjust_amount != 0 THEN 1 END) AS orders_with_adjust, + COUNT(CASE WHEN member_discount_amount != 0 THEN 1 END) AS orders_with_member_discount, + COUNT(CASE WHEN rounding_amount != 0 THEN 1 END) AS orders_with_rounding, + COUNT(CASE WHEN coupon_amount != 0 THEN 1 END) AS orders_with_coupon, + COUNT(CASE WHEN gift_card_amount != 0 THEN 1 END) AS orders_with_gift_card, + SUM(adjust_amount) AS total_adjust, + SUM(member_discount_amount) AS total_member_discount, + SUM(rounding_amount) AS total_rounding, + SUM(coupon_amount) AS total_coupon, + SUM(gift_card_amount) AS total_gift_card, + SUM(consume_money) AS total_consume, + SUM(pay_amount) AS total_pay + FROM billiards_dwd.dwd_settlement_head + """ + rows = db.query(sql) + return dict(rows[0]) if rows else {} + + +def get_sample_orders_with_discount( + db: DatabaseConnection, + limit: int = 100 +) -> List[Dict[str, Any]]: + """ + 抽样获取有优惠的订单 + """ + sql = """ + SELECT + order_settle_id, + order_trade_no, + create_time, + consume_money, + pay_amount, + adjust_amount, + member_discount_amount, + rounding_amount, + coupon_amount, + gift_card_amount, + balance_amount, + recharge_card_amount, + pl_coupon_sale_amount, + table_charge_money, + goods_money, + assistant_pd_money, + assistant_cx_money, + consume_money - pay_amount - COALESCE(recharge_card_amount, 0) + - COALESCE(gift_card_amount, 0) - COALESCE(balance_amount, 0) AS calculated_discount + FROM billiards_dwd.dwd_settlement_head + WHERE adjust_amount != 0 + OR member_discount_amount != 0 + OR rounding_amount != 0 + OR coupon_amount != 0 + OR gift_card_amount != 0 + ORDER BY RANDOM() + LIMIT %s + """ + rows = db.query(sql, (limit,)) + return [dict(row) for row in rows] if rows else [] + + +def analyze_sample_orders(orders: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + 分析抽样订单 + """ + analysis = { + 'total_sampled': len(orders), + 'with_adjust': 0, + 'with_member_discount': 0, + 'with_rounding': 0, + 'with_coupon': 0, + 'with_gift_card': 0, + 'adjust_values': [], + 'member_discount_values': [], + 'rounding_values': [], + 'coupon_values': [], + 'gift_card_values': [], + } + + for order in orders: + adjust = Decimal(str(order.get('adjust_amount', 0))) + member_discount = Decimal(str(order.get('member_discount_amount', 0))) + rounding = Decimal(str(order.get('rounding_amount', 0))) + coupon = Decimal(str(order.get('coupon_amount', 0))) + gift_card = Decimal(str(order.get('gift_card_amount', 0))) + + if adjust != 0: + analysis['with_adjust'] += 1 + analysis['adjust_values'].append(float(adjust)) + if member_discount != 0: + analysis['with_member_discount'] += 1 + analysis['member_discount_values'].append(float(member_discount)) + if rounding != 0: + analysis['with_rounding'] += 1 + analysis['rounding_values'].append(float(rounding)) + if coupon != 0: + analysis['with_coupon'] += 1 + analysis['coupon_values'].append(float(coupon)) + if gift_card != 0: + analysis['with_gift_card'] += 1 + analysis['gift_card_values'].append(float(gift_card)) + + return analysis + + +def analyze_adjust_amount(db: DatabaseConnection) -> Dict[str, Any]: + """ + 分析adjust_amount字段的分布和模式 + """ + # 1. 值分布 + sql_distribution = """ + SELECT + CASE + WHEN adjust_amount = 0 THEN '0' + WHEN adjust_amount > 0 AND adjust_amount <= 10 THEN '0-10' + WHEN adjust_amount > 10 AND adjust_amount <= 50 THEN '10-50' + WHEN adjust_amount > 50 AND adjust_amount <= 100 THEN '50-100' + WHEN adjust_amount > 100 AND adjust_amount <= 500 THEN '100-500' + WHEN adjust_amount > 500 THEN '>500' + WHEN adjust_amount < 0 AND adjust_amount >= -10 THEN '-10-0' + WHEN adjust_amount < -10 AND adjust_amount >= -50 THEN '-50--10' + WHEN adjust_amount < -50 AND adjust_amount >= -100 THEN '-100--50' + WHEN adjust_amount < -100 THEN '<-100' + END AS range, + COUNT(*) AS count, + SUM(adjust_amount) AS total_amount + FROM billiards_dwd.dwd_settlement_head + WHERE adjust_amount != 0 + GROUP BY range + ORDER BY range + """ + distribution = db.query(sql_distribution) + + # 2. 与消费金额的关系 + sql_ratio = """ + SELECT + ROUND(adjust_amount / NULLIF(consume_money, 0) * 100, 2) AS discount_ratio, + COUNT(*) AS count + FROM billiards_dwd.dwd_settlement_head + WHERE adjust_amount != 0 AND consume_money > 0 + GROUP BY discount_ratio + ORDER BY count DESC + LIMIT 20 + """ + ratio_distribution = db.query(sql_ratio) + + # 3. 典型样本 + sql_samples = """ + SELECT + order_settle_id, + consume_money, + adjust_amount, + ROUND(adjust_amount / NULLIF(consume_money, 0) * 100, 2) AS ratio + FROM billiards_dwd.dwd_settlement_head + WHERE adjust_amount != 0 + ORDER BY ABS(adjust_amount) DESC + LIMIT 10 + """ + samples = db.query(sql_samples) + + return { + 'distribution': [dict(r) for r in distribution] if distribution else [], + 'ratio_distribution': [dict(r) for r in ratio_distribution] if ratio_distribution else [], + 'top_samples': [dict(r) for r in samples] if samples else [] + } + + +def analyze_member_discount(db: DatabaseConnection) -> Dict[str, Any]: + """ + 分析member_discount_amount字段的使用情况 + """ + sql = """ + SELECT + COUNT(*) AS total_orders, + COUNT(CASE WHEN member_discount_amount != 0 THEN 1 END) AS with_discount, + SUM(member_discount_amount) AS total_discount, + AVG(CASE WHEN member_discount_amount != 0 THEN member_discount_amount END) AS avg_discount, + MAX(member_discount_amount) AS max_discount, + MIN(CASE WHEN member_discount_amount != 0 THEN member_discount_amount END) AS min_discount + FROM billiards_dwd.dwd_settlement_head + """ + rows = db.query(sql) + stats = dict(rows[0]) if rows else {} + + # 抽样有会员折扣的订单 + sql_samples = """ + SELECT + order_settle_id, + member_id, + consume_money, + member_discount_amount, + ROUND(member_discount_amount / NULLIF(consume_money, 0) * 100, 2) AS ratio + FROM billiards_dwd.dwd_settlement_head + WHERE member_discount_amount != 0 + LIMIT 20 + """ + samples = db.query(sql_samples) + + return { + 'stats': stats, + 'samples': [dict(r) for r in samples] if samples else [] + } + + +def analyze_rounding(db: DatabaseConnection) -> Dict[str, Any]: + """ + 分析rounding_amount字段的规则 + """ + # 1. 抹零金额分布 + sql_distribution = """ + SELECT + rounding_amount, + COUNT(*) AS count + FROM billiards_dwd.dwd_settlement_head + WHERE rounding_amount != 0 + GROUP BY rounding_amount + ORDER BY count DESC + LIMIT 20 + """ + distribution = db.query(sql_distribution) + + # 2. 抹零与实付金额的关系 + sql_pattern = """ + SELECT + pay_amount, + rounding_amount, + pay_amount + rounding_amount AS before_rounding, + MOD(CAST((pay_amount + rounding_amount) * 100 AS INTEGER), 100) AS cents + FROM billiards_dwd.dwd_settlement_head + WHERE rounding_amount != 0 + LIMIT 20 + """ + patterns = db.query(sql_pattern) + + return { + 'distribution': [dict(r) for r in distribution] if distribution else [], + 'patterns': [dict(r) for r in patterns] if patterns else [] + } + + +def analyze_groupbuy(db: DatabaseConnection) -> Dict[str, Any]: + """ + 分析团购优惠 + """ + # 1. 团购使用统计 + sql_stats = """ + SELECT + COUNT(*) AS total_orders, + COUNT(CASE WHEN coupon_amount != 0 THEN 1 END) AS with_coupon, + COUNT(CASE WHEN pl_coupon_sale_amount != 0 THEN 1 END) AS with_pl_coupon, + SUM(coupon_amount) AS total_coupon_amount, + SUM(pl_coupon_sale_amount) AS total_pl_coupon_sale + FROM billiards_dwd.dwd_settlement_head + """ + stats = db.query(sql_stats) + + # 2. 团购订单样本 + sql_samples = """ + SELECT + sh.order_settle_id, + sh.coupon_amount, + sh.pl_coupon_sale_amount, + gr.ledger_amount AS groupbuy_ledger_amount, + gr.ledger_unit_price AS groupbuy_unit_price + FROM billiards_dwd.dwd_settlement_head sh + LEFT JOIN billiards_dwd.dwd_groupbuy_redemption gr + ON sh.order_settle_id = gr.order_settle_id + WHERE sh.coupon_amount != 0 + LIMIT 20 + """ + samples = db.query(sql_samples) + + return { + 'stats': dict(stats[0]) if stats else {}, + 'samples': [dict(r) for r in samples] if samples else [] + } + + +def print_overall_stats(stats: Dict[str, Any]): + """打印总体统计""" + total = stats.get('total_orders', 0) + print(f"总订单数: {total:,}") + print(f"有adjust_amount的订单: {stats.get('orders_with_adjust', 0):,} ({stats.get('orders_with_adjust', 0)/total*100:.2f}%)") + print(f"有member_discount的订单: {stats.get('orders_with_member_discount', 0):,} ({stats.get('orders_with_member_discount', 0)/total*100:.2f}%)") + print(f"有rounding的订单: {stats.get('orders_with_rounding', 0):,} ({stats.get('orders_with_rounding', 0)/total*100:.2f}%)") + print(f"有coupon的订单: {stats.get('orders_with_coupon', 0):,} ({stats.get('orders_with_coupon', 0)/total*100:.2f}%)") + print(f"有gift_card的订单: {stats.get('orders_with_gift_card', 0):,} ({stats.get('orders_with_gift_card', 0)/total*100:.2f}%)") + print() + print(f"adjust_amount总额: {stats.get('total_adjust', 0):,.2f}") + print(f"member_discount总额: {stats.get('total_member_discount', 0):,.2f}") + print(f"rounding总额: {stats.get('total_rounding', 0):,.2f}") + print(f"coupon总额: {stats.get('total_coupon', 0):,.2f}") + print(f"gift_card总额: {stats.get('total_gift_card', 0):,.2f}") + + +def print_discount_analysis(analysis: Dict[str, Any]): + """打印抽样分析结果""" + print(f"抽样订单数: {analysis['total_sampled']}") + print(f" - 有adjust_amount: {analysis['with_adjust']}") + print(f" - 有member_discount: {analysis['with_member_discount']}") + print(f" - 有rounding: {analysis['with_rounding']}") + print(f" - 有coupon: {analysis['with_coupon']}") + print(f" - 有gift_card: {analysis['with_gift_card']}") + + +def print_adjust_analysis(analysis: Dict[str, Any]): + """打印adjust_amount分析结果""" + print("值分布:") + for item in analysis.get('distribution', []): + print(f" {item.get('range', 'N/A')}: {item.get('count', 0):,} 单, 总额 {item.get('total_amount', 0):,.2f}") + + print("\n折扣比例分布 (Top 10):") + for item in analysis.get('ratio_distribution', [])[:10]: + print(f" {item.get('discount_ratio', 0)}%: {item.get('count', 0):,} 单") + + print("\n大额调整样本 (Top 10):") + for item in analysis.get('top_samples', []): + print(f" 订单{item.get('order_settle_id')}: 消费{item.get('consume_money', 0):,.2f}, 调整{item.get('adjust_amount', 0):,.2f} ({item.get('ratio', 0)}%)") + + +def print_member_discount_analysis(analysis: Dict[str, Any]): + """打印会员折扣分析结果""" + stats = analysis.get('stats', {}) + print(f"总订单数: {stats.get('total_orders', 0):,}") + print(f"有会员折扣的订单: {stats.get('with_discount', 0):,}") + print(f"会员折扣总额: {stats.get('total_discount', 0):,.2f}") + print(f"平均折扣: {stats.get('avg_discount', 0):,.2f}") + print(f"最大折扣: {stats.get('max_discount', 0):,.2f}") + + samples = analysis.get('samples', []) + if samples: + print("\n样本订单:") + for item in samples[:5]: + print(f" 订单{item.get('order_settle_id')}: 会员{item.get('member_id')}, 消费{item.get('consume_money', 0):,.2f}, 折扣{item.get('member_discount_amount', 0):,.2f} ({item.get('ratio', 0)}%)") + else: + print("\n[!] 未发现使用会员折扣的订单，该字段可能未启用") + + +def print_rounding_analysis(analysis: Dict[str, Any]): + """打印抹零分析结果""" + print("抹零金额分布:") + for item in analysis.get('distribution', []): + print(f" {item.get('rounding_amount', 0):,.2f}: {item.get('count', 0):,} 单") + + print("\n抹零模式样本:") + for item in analysis.get('patterns', [])[:5]: + print(f" 实付{item.get('pay_amount', 0):,.2f} + 抹零{item.get('rounding_amount', 0):,.2f} = {item.get('before_rounding', 0):,.2f}") + + +def print_groupbuy_analysis(analysis: Dict[str, Any]): + """打印团购分析结果""" + stats = analysis.get('stats', {}) + print(f"总订单数: {stats.get('total_orders', 0):,}") + print(f"有coupon_amount的订单: {stats.get('with_coupon', 0):,}") + print(f"有pl_coupon_sale_amount的订单: {stats.get('with_pl_coupon', 0):,}") + print(f"coupon_amount总额: {stats.get('total_coupon_amount', 0):,.2f}") + print(f"pl_coupon_sale_amount总额: {stats.get('total_pl_coupon_sale', 0):,.2f}") + + print("\n团购订单样本:") + for item in analysis.get('samples', [])[:5]: + print(f" 订单{item.get('order_settle_id')}: coupon={item.get('coupon_amount', 0):,.2f}, pl_coupon={item.get('pl_coupon_sale_amount', 0):,.2f}, groupbuy_price={item.get('groupbuy_unit_price', 'N/A')}") + + +def generate_report( + overall_stats: Dict[str, Any], + discount_analysis: Dict[str, Any], + adjust_analysis: Dict[str, Any], + member_discount_analysis: Dict[str, Any], + rounding_analysis: Dict[str, Any], + groupbuy_analysis: Dict[str, Any] +) -> str: + """ + 生成Markdown格式的分析报告 + """ + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + total = overall_stats.get('total_orders', 1) + + report = f"""# 优惠口径抽样分析报告 + +**生成时间**: {now} + +## 一、总体统计 + +| 指标 | 数值 | 占比 | +|------|------|------| +| 总订单数 | {overall_stats.get('total_orders', 0):,} | 100% | +| 有adjust_amount的订单 | {overall_stats.get('orders_with_adjust', 0):,} | {overall_stats.get('orders_with_adjust', 0)/total*100:.2f}% | +| 有member_discount的订单 | {overall_stats.get('orders_with_member_discount', 0):,} | {overall_stats.get('orders_with_member_discount', 0)/total*100:.2f}% | +| 有rounding的订单 | {overall_stats.get('orders_with_rounding', 0):,} | {overall_stats.get('orders_with_rounding', 0)/total*100:.2f}% | +| 有coupon的订单 | {overall_stats.get('orders_with_coupon', 0):,} | {overall_stats.get('orders_with_coupon', 0)/total*100:.2f}% | +| 有gift_card的订单 | {overall_stats.get('orders_with_gift_card', 0):,} | {overall_stats.get('orders_with_gift_card', 0)/total*100:.2f}% | + +### 金额统计 + +| 优惠类型 | 总额 | +|----------|------| +| adjust_amount (台费打折/调整) | {overall_stats.get('total_adjust', 0):,.2f} | +| member_discount_amount (会员折扣) | {overall_stats.get('total_member_discount', 0):,.2f} | +| rounding_amount (抹零) | {overall_stats.get('total_rounding', 0):,.2f} | +| coupon_amount (团购抵消台费) | {overall_stats.get('total_coupon', 0):,.2f} | +| gift_card_amount (赠送卡支付) | {overall_stats.get('total_gift_card', 0):,.2f} | + +## 二、adjust_amount (台费打折/调整) 分析 + +### 值分布 + +| 区间 | 订单数 | 总额 | +|------|--------|------| +""" + + for item in adjust_analysis.get('distribution', []): + report += f"| {item.get('range', 'N/A')} | {item.get('count', 0):,} | {item.get('total_amount', 0):,.2f} |\n" + + report += """ +### 分析结论 + +- **是否包含大客户优惠**: 需要进一步分析adjust_amount的业务来源 +- **与普通调整的区分**: 建议查看是否有备注字段或关联的优惠活动表 + +## 三、member_discount_amount (会员折扣) 分析 + +""" + + member_stats = member_discount_analysis.get('stats', {}) + with_discount = member_stats.get('with_discount', 0) + + if with_discount == 0: + report += """### 结论 + +**[!] 该字段未发现任何非零值，会员折扣功能可能未启用。** + +建议：在DWS财务统计中，可以暂时忽略此字段，或将其标记为"待启用"。 +""" + else: + report += f"""### 使用统计 + +| 指标 | 数值 | +|------|------| +| 有会员折扣的订单 | {with_discount:,} | +| 会员折扣总额 | {member_stats.get('total_discount', 0):,.2f} | +| 平均折扣 | {member_stats.get('avg_discount', 0):,.2f} | +| 最大折扣 | {member_stats.get('max_discount', 0):,.2f} | +""" + + report += """ +## 四、rounding_amount (抹零) 分析 + +### 抹零金额分布 + +| 抹零金额 | 订单数 | +|----------|--------| +""" + + for item in rounding_analysis.get('distribution', [])[:10]: + report += f"| {item.get('rounding_amount', 0):,.2f} | {item.get('count', 0):,} |\n" + + report += """ +### 抹零规则推断 + +根据抹零金额分布，推断抹零规则为： +- 抹零到整元（去除角分） +- 或抹零到特定尾数 + +## 五、团购优惠分析 + +""" + + groupbuy_stats = groupbuy_analysis.get('stats', {}) + report += f"""### 使用统计 + +| 指标 | 数值 | +|------|------| +| 有coupon_amount的订单 | {groupbuy_stats.get('with_coupon', 0):,} | +| 有pl_coupon_sale_amount的订单 | {groupbuy_stats.get('with_pl_coupon', 0):,} | +| coupon_amount总额 | {groupbuy_stats.get('total_coupon_amount', 0):,.2f} | +| pl_coupon_sale_amount总额 | {groupbuy_stats.get('total_pl_coupon_sale', 0):,.2f} | + +### 团购支付金额计算路径 + +根据分析，团购支付金额应按以下路径计算： +1. 若 `pl_coupon_sale_amount ≠ 0` → 使用 `pl_coupon_sale_amount` +2. 若 `pl_coupon_sale_amount = 0` 且 `coupon_amount ≠ 0` → 通过 `order_settle_id` 关联 `dwd_groupbuy_redemption` 获取 `ledger_unit_price` + +团购优惠金额 = coupon_amount - 团购支付金额 + +## 六、建议与结论 + +### 优惠口径定义建议 + +| 优惠类型 | 字段来源 | 计算公式 | 状态 | +|----------|----------|----------|------| +| 团购优惠 | settlement + groupbuy | coupon_amount - 团购支付金额 | 可用 | +| 会员折扣 | settlement.member_discount_amount | 直接取值 | 待确认 | +| 赠送卡抵扣 | settlement.gift_card_amount | 直接取值 | 可用 | +| 手动调整 | settlement.adjust_amount | 直接取值 | 可用 | +| 抹零 | settlement.rounding_amount | 直接取值 | 可用 | +| 大客户优惠 | 待分析 | 需要业务确认 | 待定义 | +| 其他优惠 | 待分析 | 需要业务确认 | 待定义 | + +### 下一步行动 + +1. **确认会员折扣是否启用**: 与业务确认member_discount_amount的使用场景 +2. **大客户优惠识别规则**: 与业务确认如何从adjust_amount中识别大客户优惠 +3. **其他优惠分类**: 与业务确认adjust_amount中还包含哪些优惠类型 +""" + + return report + + +if __name__ == "__main__": + analyze_discount_patterns() diff --git a/etl_billiards/scripts/analyze_member_discount_usage.py b/etl_billiards/scripts/analyze_member_discount_usage.py new file mode 100644 index 0000000..43a0704 --- /dev/null +++ b/etl_billiards/scripts/analyze_member_discount_usage.py @@ -0,0 +1,287 @@ +# -*- coding: utf-8 -*- +""" +会员折扣启用分析脚本 + +功能说明： + 确认 dwd_settlement_head.member_discount_amount 字段是否已启用 + +分析内容： + 1. 统计非零记录数 + 2. 按时间分布分析 + 3. 按会员类型分析 + 4. 与其他字段的关联分析 + +输出： + - 控制台打印分析结果 + - 结论：字段是否已启用，使用场景 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +import os +import sys +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent.parent.parent +sys.path.insert(0, str(project_root)) + +from etl_billiards.utils.config import Config +from etl_billiards.utils.db import DatabaseConnection + + +def analyze_member_discount_usage(): + """ + 执行会员折扣启用分析 + """ + print("=" * 80) + print("会员折扣启用分析 (member_discount_amount)") + print("=" * 80) + print() + + # 加载配置和数据库连接 + config = Config() + db = DatabaseConnection(config) + + try: + # 1. 基础统计 + print("【1. 基础统计】") + print("-" * 40) + basic_stats = get_basic_stats(db) + print_basic_stats(basic_stats) + print() + + # 2. 时间分布分析 + print("【2. 时间分布分析】") + print("-" * 40) + time_distribution = get_time_distribution(db) + print_time_distribution(time_distribution) + print() + + # 3. 会员类型分析 + print("【3. 与会员的关联分析】") + print("-" * 40) + member_analysis = get_member_analysis(db) + print_member_analysis(member_analysis) + print() + + # 4. 样本数据 + print("【4. 样本数据】") + print("-" * 40) + samples = get_sample_data(db) + print_samples(samples) + print() + + # 5. 结论 + print("【5. 分析结论】") + print("-" * 40) + print_conclusion(basic_stats) + + finally: + db.close() + + +def get_basic_stats(db: DatabaseConnection) -> Dict[str, Any]: + """ + 获取基础统计数据 + """ + sql = """ + SELECT + COUNT(*) AS total_orders, + COUNT(CASE WHEN member_discount_amount != 0 THEN 1 END) AS with_member_discount, + COUNT(CASE WHEN member_discount_amount > 0 THEN 1 END) AS positive_discount, + COUNT(CASE WHEN member_discount_amount < 0 THEN 1 END) AS negative_discount, + SUM(member_discount_amount) AS total_member_discount, + AVG(CASE WHEN member_discount_amount != 0 THEN member_discount_amount END) AS avg_discount, + MAX(member_discount_amount) AS max_discount, + MIN(member_discount_amount) AS min_discount, + STDDEV(CASE WHEN member_discount_amount != 0 THEN member_discount_amount END) AS stddev_discount + FROM billiards_dwd.dwd_settlement_head + """ + rows = db.query(sql) + return dict(rows[0]) if rows else {} + + +def get_time_distribution(db: DatabaseConnection) -> List[Dict[str, Any]]: + """ + 获取按月份的时间分布 + """ + sql = """ + SELECT + DATE_TRUNC('month', create_time)::DATE AS month, + COUNT(*) AS total_orders, + COUNT(CASE WHEN member_discount_amount != 0 THEN 1 END) AS with_discount, + SUM(member_discount_amount) AS total_discount + FROM billiards_dwd.dwd_settlement_head + GROUP BY DATE_TRUNC('month', create_time) + ORDER BY month DESC + LIMIT 12 + """ + rows = db.query(sql) + return [dict(row) for row in rows] if rows else [] + + +def get_member_analysis(db: DatabaseConnection) -> Dict[str, Any]: + """ + 分析与会员的关联 + """ + # 会员vs非会员 + sql_member_vs_guest = """ + SELECT + CASE WHEN member_id = 0 THEN '散客' ELSE '会员' END AS customer_type, + COUNT(*) AS total_orders, + COUNT(CASE WHEN member_discount_amount != 0 THEN 1 END) AS with_discount, + SUM(member_discount_amount) AS total_discount + FROM billiards_dwd.dwd_settlement_head + GROUP BY CASE WHEN member_id = 0 THEN '散客' ELSE '会员' END + """ + member_vs_guest = db.query(sql_member_vs_guest) + + # 按会员卡等级 + sql_by_grade = """ + SELECT + COALESCE(m.member_card_grade_name, '未知') AS grade_name, + COUNT(*) AS total_orders, + COUNT(CASE WHEN sh.member_discount_amount != 0 THEN 1 END) AS with_discount, + SUM(sh.member_discount_amount) AS total_discount + FROM billiards_dwd.dwd_settlement_head sh + LEFT JOIN billiards_dwd.dim_member m ON sh.member_id = m.member_id + WHERE sh.member_id != 0 + GROUP BY COALESCE(m.member_card_grade_name, '未知') + ORDER BY total_orders DESC + """ + by_grade = db.query(sql_by_grade) + + return { + 'member_vs_guest': [dict(row) for row in member_vs_guest] if member_vs_guest else [], + 'by_grade': [dict(row) for row in by_grade] if by_grade else [] + } + + +def get_sample_data(db: DatabaseConnection) -> List[Dict[str, Any]]: + """ + 获取有会员折扣的样本数据 + """ + sql = """ + SELECT + sh.order_settle_id, + sh.order_trade_no, + sh.create_time, + sh.member_id, + m.nickname AS member_name, + m.member_card_grade_name, + sh.consume_money, + sh.pay_amount, + sh.member_discount_amount, + ROUND(sh.member_discount_amount / NULLIF(sh.consume_money, 0) * 100, 2) AS discount_ratio + FROM billiards_dwd.dwd_settlement_head sh + LEFT JOIN billiards_dwd.dim_member m ON sh.member_id = m.member_id + WHERE sh.member_discount_amount != 0 + ORDER BY sh.create_time DESC + LIMIT 20 + """ + rows = db.query(sql) + return [dict(row) for row in rows] if rows else [] + + +def print_basic_stats(stats: Dict[str, Any]): + """打印基础统计""" + total = stats.get('total_orders', 1) + with_discount = stats.get('with_member_discount', 0) + + print(f"总订单数: {total:,}") + print(f"有会员折扣的订单: {with_discount:,} ({with_discount/total*100:.4f}%)") + print(f" - 正值(折扣): {stats.get('positive_discount', 0):,}") + print(f" - 负值(加价?): {stats.get('negative_discount', 0):,}") + print() + print(f"会员折扣总额: {stats.get('total_member_discount', 0):,.2f}") + print(f"平均折扣: {stats.get('avg_discount', 0) or 0:,.2f}") + print(f"最大折扣: {stats.get('max_discount', 0):,.2f}") + print(f"最小折扣: {stats.get('min_discount', 0):,.2f}") + + +def print_time_distribution(distribution: List[Dict[str, Any]]): + """打印时间分布""" + if not distribution: + print("无数据") + return + + print(f"{'月份':<12} {'总订单':>10} {'有折扣':>10} {'折扣总额':>15}") + print("-" * 50) + for item in distribution: + month = str(item.get('month', 'N/A'))[:7] + total = item.get('total_orders', 0) + with_discount = item.get('with_discount', 0) + total_discount = item.get('total_discount', 0) + print(f"{month:<12} {total:>10,} {with_discount:>10,} {total_discount:>15,.2f}") + + +def print_member_analysis(analysis: Dict[str, Any]): + """打印会员分析""" + print("会员 vs 散客:") + for item in analysis.get('member_vs_guest', []): + print(f" {item.get('customer_type', 'N/A')}: {item.get('total_orders', 0):,} 单, {item.get('with_discount', 0)} 单有折扣, 折扣总额 {item.get('total_discount', 0):,.2f}") + + print("\n按会员卡等级:") + for item in analysis.get('by_grade', []): + print(f" {item.get('grade_name', 'N/A')}: {item.get('total_orders', 0):,} 单, {item.get('with_discount', 0)} 单有折扣") + + +def print_samples(samples: List[Dict[str, Any]]): + """打印样本数据""" + if not samples: + print("[!] 未发现使用会员折扣的订单") + return + + print(f"{'订单ID':<20} {'会员':<15} {'等级':<10} {'消费':>12} {'折扣':>12} {'比例':>8}") + print("-" * 80) + for item in samples[:10]: + order_id = str(item.get('order_settle_id', 'N/A'))[:18] + member = str(item.get('member_name', 'N/A'))[:13] + grade = str(item.get('member_card_grade_name', 'N/A'))[:8] + consume = item.get('consume_money', 0) + discount = item.get('member_discount_amount', 0) + ratio = item.get('discount_ratio', 0) + print(f"{order_id:<20} {member:<15} {grade:<10} {consume:>12,.2f} {discount:>12,.2f} {ratio:>7}%") + + +def print_conclusion(stats: Dict[str, Any]): + """打印分析结论""" + with_discount = stats.get('with_member_discount', 0) + total = stats.get('total_orders', 1) + ratio = with_discount / total * 100 + + if with_discount == 0: + print("【结论】: member_discount_amount 字段 **未启用**") + print() + print("该字段在所有订单中均为0，表明：") + print(" 1. 会员折扣功能在业务系统中未开启") + print(" 2. 或会员折扣通过其他方式（如adjust_amount）记录") + print() + print("【建议】:") + print(" 1. 在DWS财务统计中，暂时不处理此字段") + print(" 2. 将此字段标记为'预留/待启用'") + print(" 3. 后续如果业务启用，再更新统计逻辑") + elif ratio < 1: + print(f"【结论】: member_discount_amount 字段 **极少使用** (仅{ratio:.4f}%订单)") + print() + print("该字段使用率极低，可能是：") + print(" 1. 会员折扣功能刚启用不久") + print(" 2. 仅特定场景使用") + print() + print("【建议】:") + print(" 1. 在DWS财务统计中保留此字段的处理逻辑") + print(" 2. 定期监控使用率变化") + else: + print(f"【结论】: member_discount_amount 字段 **已启用** ({ratio:.2f}%订单使用)") + print() + print("【建议】:") + print(" 1. 在DWS财务优惠明细中正常统计此字段") + print(" 2. 关注会员折扣与其他优惠的叠加规则") + + +if __name__ == "__main__": + analyze_member_discount_usage() diff --git a/etl_billiards/scripts/check_assistant_dim.py b/etl_billiards/scripts/check_assistant_dim.py new file mode 100644 index 0000000..0804657 --- /dev/null +++ b/etl_billiards/scripts/check_assistant_dim.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- +import sys +sys.path.insert(0, '.') +from config.settings import AppConfig +from database.connection import DatabaseConnection +from database.operations import DatabaseOperations + +config = AppConfig.load() +db_conn = DatabaseConnection(config.config['db']['dsn']) +db = DatabaseOperations(db_conn) + +# 检查dim_assistant表结构 +print('=== dim_assistant columns ===') +sql0 = """ + SELECT column_name FROM information_schema.columns + WHERE table_schema = 'billiards_dwd' AND table_name = 'dim_assistant' +""" +for row in db.query(sql0): + print(f' {dict(row)["column_name"]}') + +# 检查dim_assistant数量 +print() +print('=== dim_assistant ===') +sql1 = 'SELECT COUNT(*) as cnt FROM billiards_dwd.dim_assistant WHERE scd2_is_current = 1' +rows = db.query(sql1) +print(f'dim_assistant current count: {dict(rows[0])["cnt"]}') + +# 检查服务记录中的nickname分布 +print() +print('=== Service by nickname ===') +sql2 = """ + SELECT nickname, COUNT(*) as service_count, COUNT(DISTINCT tenant_member_id) as member_count + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 + GROUP BY nickname + ORDER BY service_count DESC + LIMIT 10 +""" +for row in db.query(sql2): + r = dict(row) + print(f' {r["nickname"]}: {r["service_count"]} services, {r["member_count"]} members') + +# 检查assistant_no分布 +print() +print('=== Service by assistant_no ===') +sql3 = """ + SELECT assistant_no, nickname, COUNT(*) as service_count, COUNT(DISTINCT tenant_member_id) as member_count + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 + GROUP BY assistant_no, nickname + ORDER BY service_count DESC + LIMIT 10 +""" +for row in db.query(sql3): + r = dict(row) + print(f' {r["assistant_no"]} ({r["nickname"]}): {r["service_count"]} services, {r["member_count"]} members') + +# 近60天 +print() +print('=== Last 60 days by nickname ===') +sql4 = """ + SELECT nickname, COUNT(*) as service_count, COUNT(DISTINCT tenant_member_id) as member_count + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 + AND last_use_time >= NOW() - INTERVAL '60 days' + GROUP BY nickname + ORDER BY service_count DESC + LIMIT 15 +""" +for row in db.query(sql4): + r = dict(row) + print(f' {r["nickname"]}: {r["service_count"]} services, {r["member_count"]} members') + +db_conn.close() diff --git a/etl_billiards/scripts/check_dwd_service.py b/etl_billiards/scripts/check_dwd_service.py new file mode 100644 index 0000000..78a280b --- /dev/null +++ b/etl_billiards/scripts/check_dwd_service.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +import sys +sys.path.insert(0, '.') +from config.settings import AppConfig +from database.connection import DatabaseConnection +from database.operations import DatabaseOperations + +config = AppConfig.load() +db_conn = DatabaseConnection(config.config['db']['dsn']) +db = DatabaseOperations(db_conn) + +# 检查DWD层服务记录分布 +print("=== DWD层服务记录分析 ===") +print() + +# 1. 总体统计 +sql1 = """ + SELECT + COUNT(*) as total_records, + COUNT(DISTINCT tenant_member_id) as unique_members, + COUNT(DISTINCT site_assistant_id) as unique_assistants, + COUNT(DISTINCT (tenant_member_id, site_assistant_id)) as unique_pairs + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 +""" +r = dict(db.query(sql1)[0]) +print("总体统计:") +print(f" 总服务记录数: {r['total_records']}") +print(f" 唯一会员数: {r['unique_members']}") +print(f" 唯一助教数: {r['unique_assistants']}") +print(f" 唯一客户-助教对: {r['unique_pairs']}") + +# 2. 助教服务会员数分布 +print() +print("助教服务会员数分布 (Top 10):") +sql2 = """ + SELECT site_assistant_id, COUNT(DISTINCT tenant_member_id) as member_count + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 + GROUP BY site_assistant_id + ORDER BY member_count DESC + LIMIT 10 +""" +for row in db.query(sql2): + r = dict(row) + print(f" 助教 {r['site_assistant_id']}: 服务 {r['member_count']} 个会员") + +# 3. 每个客户-助教对的服务次数分布 +print() +print("客户-助教对 服务次数分布 (Top 10):") +sql3 = """ + SELECT tenant_member_id, site_assistant_id, COUNT(*) as service_count + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 + GROUP BY tenant_member_id, site_assistant_id + ORDER BY service_count DESC + LIMIT 10 +""" +for row in db.query(sql3): + r = dict(row) + print(f" 会员 {r['tenant_member_id']} - 助教 {r['site_assistant_id']}: {r['service_count']} 次服务") + +# 4. 近60天的数据 +print() +print("=== 近60天数据 ===") +sql4 = """ + SELECT + COUNT(*) as total_records, + COUNT(DISTINCT tenant_member_id) as unique_members, + COUNT(DISTINCT site_assistant_id) as unique_assistants, + COUNT(DISTINCT (tenant_member_id, site_assistant_id)) as unique_pairs + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 + AND last_use_time >= NOW() - INTERVAL '60 days' +""" +r4 = dict(db.query(sql4)[0]) +print(f" 总服务记录数: {r4['total_records']}") +print(f" 唯一会员数: {r4['unique_members']}") +print(f" 唯一助教数: {r4['unique_assistants']}") +print(f" 唯一客户-助教对: {r4['unique_pairs']}") + +db_conn.close() diff --git a/etl_billiards/scripts/check_intimacy_stats.py b/etl_billiards/scripts/check_intimacy_stats.py new file mode 100644 index 0000000..45e7006 --- /dev/null +++ b/etl_billiards/scripts/check_intimacy_stats.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +import sys +sys.path.insert(0, '.') +from config.settings import AppConfig +from database.connection import DatabaseConnection +from database.operations import DatabaseOperations + +config = AppConfig.load() +db_conn = DatabaseConnection(config.config['db']['dsn']) +db = DatabaseOperations(db_conn) + +# 检查实际统计 +sql = """ + SELECT + COUNT(*) as total_pairs, + COUNT(DISTINCT member_id) as unique_members, + COUNT(DISTINCT assistant_id) as unique_assistants + FROM billiards_dws.dws_member_assistant_intimacy +""" +rows = db.query(sql) +r = dict(rows[0]) +print("DWS亲密指数统计:") +print(f" 总记录数(对): {r['total_pairs']}") +print(f" 唯一会员数: {r['unique_members']}") +print(f" 唯一助教数: {r['unique_assistants']}") + +# 查看助教分布 +sql2 = """ + SELECT assistant_id, COUNT(*) as member_count + FROM billiards_dws.dws_member_assistant_intimacy + GROUP BY assistant_id + ORDER BY member_count DESC + LIMIT 10 +""" +rows2 = db.query(sql2) +print() +print("Top 10 助教 (按服务会员数):") +for row in rows2: + r = dict(row) + print(f" 助教 {r['assistant_id']}: 服务 {r['member_count']} 个会员") + +# 检查DWD层原始数据 +sql3 = """ + SELECT + COUNT(DISTINCT site_assistant_id) as unique_assistants, + COUNT(DISTINCT tenant_member_id) as unique_members + FROM billiards_dwd.dwd_assistant_service_log + WHERE tenant_member_id > 0 AND is_delete = 0 +""" +rows3 = db.query(sql3) +r3 = dict(rows3[0]) +print() +print("DWD层原始数据:") +print(f" 唯一助教数: {r3['unique_assistants']}") +print(f" 唯一会员数: {r3['unique_members']}") + +db_conn.close() diff --git a/etl_billiards/scripts/check_ods_gaps.py b/etl_billiards/scripts/check_ods_gaps.py index 76fc015..279f6a2 100644 --- a/etl_billiards/scripts/check_ods_gaps.py +++ b/etl_billiards/scripts/check_ods_gaps.py @@ -702,6 +702,7 @@ def run_gap_check( content_sample_limit: int | None = None, window_split_unit: str | None = None, window_compensation_hours: int | None = None, + tag: str = "", ) -> dict: cfg = cfg or AppConfig.load({}) tz = ZoneInfo(cfg.get("app.timezone", "Asia/Taipei")) @@ -800,7 +801,7 @@ def run_gap_check( if cutoff: logger.info("CUTOFF=%s overlap_hours=%s", cutoff.isoformat(), cutoff_overlap_hours) - tag_suffix = f"_{args.tag}" if args.tag else "" + tag_suffix = f"_{tag}" if tag else "" client = build_recording_client(cfg, task_code=f"ODS_GAP_CHECK{tag_suffix}") db_state = _init_db_state(cfg) diff --git a/etl_billiards/scripts/create_index_tables.py b/etl_billiards/scripts/create_index_tables.py new file mode 100644 index 0000000..571f4ed --- /dev/null +++ b/etl_billiards/scripts/create_index_tables.py @@ -0,0 +1,185 @@ +# -*- coding: utf-8 -*- +""" +创建指数算法相关表 +""" +import sys +import os +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from config.settings import AppConfig +from database.connection import DatabaseConnection +from database.operations import DatabaseOperations + +# 表DDL +DDL_STATEMENTS = [ + # 参数配置表 + """ + DROP TABLE IF EXISTS billiards_dws.cfg_index_parameters CASCADE; + CREATE TABLE billiards_dws.cfg_index_parameters ( + param_id SERIAL PRIMARY KEY, + index_type VARCHAR(50) NOT NULL, + param_name VARCHAR(100) NOT NULL, + param_value NUMERIC(14,6) NOT NULL, + description TEXT, + effective_from DATE NOT NULL DEFAULT CURRENT_DATE, + effective_to DATE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_cfg_index_parameters UNIQUE (index_type, param_name, effective_from) + ); + CREATE INDEX idx_cfg_index_params_type ON billiards_dws.cfg_index_parameters (index_type); + """, + + # 召回指数表 + """ + DROP TABLE IF EXISTS billiards_dws.dws_member_recall_index CASCADE; + CREATE TABLE billiards_dws.dws_member_recall_index ( + recall_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + tenant_id BIGINT NOT NULL, + member_id BIGINT NOT NULL, + days_since_last_visit INTEGER, + visit_interval_median NUMERIC(10,2), + visit_interval_mad NUMERIC(10,2), + days_since_first_visit INTEGER, + days_since_last_recharge INTEGER, + visits_last_14_days INTEGER NOT NULL DEFAULT 0, + visits_last_60_days INTEGER NOT NULL DEFAULT 0, + score_overdue NUMERIC(10,4), + score_new_bonus NUMERIC(10,4), + score_recharge_bonus NUMERIC(10,4), + score_hot_drop NUMERIC(10,4), + raw_score NUMERIC(14,6), + display_score NUMERIC(4,2), + calc_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + calc_version INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_member_recall UNIQUE (site_id, member_id) + ); + CREATE INDEX idx_dws_recall_display ON billiards_dws.dws_member_recall_index (site_id, display_score DESC); + """, + + # 亲密指数表 + """ + DROP TABLE IF EXISTS billiards_dws.dws_member_assistant_intimacy CASCADE; + CREATE TABLE billiards_dws.dws_member_assistant_intimacy ( + intimacy_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + tenant_id BIGINT NOT NULL, + member_id BIGINT NOT NULL, + assistant_id BIGINT NOT NULL, + session_count INTEGER NOT NULL DEFAULT 0, + total_duration_minutes INTEGER NOT NULL DEFAULT 0, + basic_session_count INTEGER NOT NULL DEFAULT 0, + incentive_session_count INTEGER NOT NULL DEFAULT 0, + days_since_last_session INTEGER, + attributed_recharge_count INTEGER NOT NULL DEFAULT 0, + attributed_recharge_amount NUMERIC(14,2) NOT NULL DEFAULT 0, + score_frequency NUMERIC(10,4), + score_recency NUMERIC(10,4), + score_recharge NUMERIC(10,4), + score_duration NUMERIC(10,4), + burst_multiplier NUMERIC(6,4), + raw_score NUMERIC(14,6), + display_score NUMERIC(4,2), + calc_time TIMESTAMPTZ NOT NULL DEFAULT NOW(), + calc_version INTEGER NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_member_assistant_intimacy UNIQUE (site_id, member_id, assistant_id) + ); + CREATE INDEX idx_dws_intimacy_member ON billiards_dws.dws_member_assistant_intimacy (site_id, member_id, display_score DESC); + CREATE INDEX idx_dws_intimacy_assistant ON billiards_dws.dws_member_assistant_intimacy (site_id, assistant_id, display_score DESC); + """, + + # 分位点历史表 + """ + DROP TABLE IF EXISTS billiards_dws.dws_index_percentile_history CASCADE; + CREATE TABLE billiards_dws.dws_index_percentile_history ( + history_id BIGSERIAL PRIMARY KEY, + site_id BIGINT NOT NULL, + index_type VARCHAR(50) NOT NULL, + calc_time TIMESTAMPTZ NOT NULL, + percentile_5 NUMERIC(14,6), + percentile_95 NUMERIC(14,6), + percentile_5_smoothed NUMERIC(14,6), + percentile_95_smoothed NUMERIC(14,6), + record_count INTEGER, + min_raw_score NUMERIC(14,6), + max_raw_score NUMERIC(14,6), + avg_raw_score NUMERIC(14,6), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CONSTRAINT uk_dws_index_percentile_history UNIQUE (site_id, index_type, calc_time) + ); + CREATE INDEX idx_dws_percentile_history ON billiards_dws.dws_index_percentile_history (site_id, index_type, calc_time DESC); + """ +] + +# 初始化参数 +SEED_PARAMS = """ +INSERT INTO billiards_dws.cfg_index_parameters + (index_type, param_name, param_value, description, effective_from) +VALUES + ('RECALL', 'lookback_days', 60, '回溯窗口（天）', CURRENT_DATE), + ('RECALL', 'sigma_min', 2.0, '波动下限（天）', CURRENT_DATE), + ('RECALL', 'halflife_new', 7, '新客户半衰期（天）', CURRENT_DATE), + ('RECALL', 'halflife_recharge', 10, '刚充值半衰期（天）', CURRENT_DATE), + ('RECALL', 'weight_overdue', 3.0, '超期紧急性权重', CURRENT_DATE), + ('RECALL', 'weight_new', 1.0, '新客户权重', CURRENT_DATE), + ('RECALL', 'weight_recharge', 1.0, '刚充值权重', CURRENT_DATE), + ('RECALL', 'weight_hot', 1.0, '热度断档权重', CURRENT_DATE), + ('RECALL', 'percentile_lower', 5, '下锚分位数', CURRENT_DATE), + ('RECALL', 'percentile_upper', 95, '上锚分位数', CURRENT_DATE), + ('RECALL', 'ewma_alpha', 0.2, 'EWMA平滑系数', CURRENT_DATE), + ('INTIMACY', 'lookback_days', 60, '回溯窗口（天）', CURRENT_DATE), + ('INTIMACY', 'session_merge_hours', 4, '会话合并间隔（小时）', CURRENT_DATE), + ('INTIMACY', 'recharge_attribute_hours', 1, '充值归因窗口（小时）', CURRENT_DATE), + ('INTIMACY', 'amount_base', 500, '金额压缩基准（元）', CURRENT_DATE), + ('INTIMACY', 'incentive_weight', 1.5, '附加课权重倍数', CURRENT_DATE), + ('INTIMACY', 'halflife_session', 14, '会话衰减半衰期（天）', CURRENT_DATE), + ('INTIMACY', 'halflife_last', 10, '最近一次半衰期（天）', CURRENT_DATE), + ('INTIMACY', 'halflife_recharge', 21, '充值衰减半衰期（天）', CURRENT_DATE), + ('INTIMACY', 'halflife_short', 7, '短期激增检测半衰期（天）', CURRENT_DATE), + ('INTIMACY', 'halflife_long', 30, '长期激增检测半衰期（天）', CURRENT_DATE), + ('INTIMACY', 'weight_frequency', 2.0, '频次权重', CURRENT_DATE), + ('INTIMACY', 'weight_recency', 1.5, '最近一次权重', CURRENT_DATE), + ('INTIMACY', 'weight_recharge', 2.0, '归因充值权重', CURRENT_DATE), + ('INTIMACY', 'weight_duration', 0.5, '时长权重', CURRENT_DATE), + ('INTIMACY', 'burst_gamma', 0.6, '激增放大系数', CURRENT_DATE), + ('INTIMACY', 'percentile_lower', 5, '下锚分位数', CURRENT_DATE), + ('INTIMACY', 'percentile_upper', 95, '上锚分位数', CURRENT_DATE), + ('INTIMACY', 'ewma_alpha', 0.2, 'EWMA平滑系数', CURRENT_DATE) +ON CONFLICT (index_type, param_name, effective_from) DO NOTHING; +""" + +def main(): + print("创建指数算法相关表...") + + config = AppConfig.load() + db_conn = DatabaseConnection(config.config["db"]["dsn"]) + + try: + with db_conn.conn.cursor() as cur: + # 创建表 + for i, ddl in enumerate(DDL_STATEMENTS, 1): + print(f" 执行DDL {i}/{len(DDL_STATEMENTS)}...") + cur.execute(ddl) + + # 初始化参数 + print(" 初始化算法参数...") + cur.execute(SEED_PARAMS) + + db_conn.conn.commit() + print("完成!") + + # 验证 + cur.execute("SELECT COUNT(*) FROM billiards_dws.cfg_index_parameters") + count = cur.fetchone()[0] + print(f" 已插入 {count} 个参数配置") + + finally: + db_conn.close() + +if __name__ == '__main__': + main() diff --git a/etl_billiards/scripts/import_dws_excel.py b/etl_billiards/scripts/import_dws_excel.py new file mode 100644 index 0000000..f3dac1a --- /dev/null +++ b/etl_billiards/scripts/import_dws_excel.py @@ -0,0 +1,602 @@ +# -*- coding: utf-8 -*- +""" +DWS Excel导入脚本 + +功能说明： + 支持三类Excel数据的导入： + 1. 支出结构（dws_finance_expense_summary） + 2. 平台结算（dws_platform_settlement） + 3. 充值提成（dws_assistant_recharge_commission） + +导入规范： + - 字段定义：按照目标表字段要求 + - 时间粒度：支出按月，平台结算按日，充值提成按月 + - 门店维度：使用配置的site_id + - 去重规则：按import_batch_no去重 + - 校验规则：金额字段非负，日期格式校验 + +使用方式： + python import_dws_excel.py --type expense --file expenses.xlsx + python import_dws_excel.py --type platform --file platform_settlement.xlsx + python import_dws_excel.py --type commission --file recharge_commission.xlsx + +作者：ETL团队 +创建日期：2026-02-01 +""" + +import argparse +import os +import sys +import uuid +from datetime import date, datetime +from decimal import Decimal, InvalidOperation +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +# 添加项目根目录到Python路径 +project_root = Path(__file__).parent.parent.parent +sys.path.insert(0, str(project_root)) + +try: + import pandas as pd +except ImportError: + print("请安装 pandas: pip install pandas openpyxl") + sys.exit(1) + +from etl_billiards.utils.config import Config +from etl_billiards.utils.db import DatabaseConnection + + +# ============================================================================= +# 常量定义 +# ============================================================================= + +# 支出类型枚举 +EXPENSE_TYPES = { + '房租': 'RENT', + '水电费': 'UTILITY', + '物业费': 'PROPERTY', + '工资': 'SALARY', + '报销': 'REIMBURSE', + '平台服务费': 'PLATFORM_FEE', + '其他': 'OTHER', +} + +# 支出大类映射 +EXPENSE_CATEGORIES = { + 'RENT': 'FIXED_COST', + 'UTILITY': 'VARIABLE_COST', + 'PROPERTY': 'FIXED_COST', + 'SALARY': 'FIXED_COST', + 'REIMBURSE': 'VARIABLE_COST', + 'PLATFORM_FEE': 'VARIABLE_COST', + 'OTHER': 'OTHER', +} + +# 平台类型枚举 +PLATFORM_TYPES = { + '美团': 'MEITUAN', + '抖音': 'DOUYIN', + '大众点评': 'DIANPING', + '其他': 'OTHER', +} + + +# ============================================================================= +# 导入基类 +# ============================================================================= + +class BaseImporter: + """导入基类""" + + def __init__(self, config: Config, db: DatabaseConnection): + self.config = config + self.db = db + self.site_id = config.get("app.store_id") + self.tenant_id = config.get("app.tenant_id", self.site_id) + self.batch_no = self._generate_batch_no() + + def _generate_batch_no(self) -> str: + """生成导入批次号""" + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + unique_id = str(uuid.uuid4())[:8] + return f"{timestamp}_{unique_id}" + + def _safe_decimal(self, value: Any, default: Decimal = Decimal('0')) -> Decimal: + """安全转换为Decimal""" + if value is None or pd.isna(value): + return default + try: + return Decimal(str(value)) + except (ValueError, InvalidOperation): + return default + + def _safe_date(self, value: Any) -> Optional[date]: + """安全转换为日期""" + if value is None or pd.isna(value): + return None + if isinstance(value, datetime): + return value.date() + if isinstance(value, date): + return value + try: + return pd.to_datetime(value).date() + except: + return None + + def _safe_month(self, value: Any) -> Optional[date]: + """安全转换为月份（月第一天）""" + dt = self._safe_date(value) + if dt: + return dt.replace(day=1) + return None + + def import_file(self, file_path: str) -> Dict[str, Any]: + """导入文件""" + raise NotImplementedError + + def validate_row(self, row: Dict[str, Any], row_idx: int) -> List[str]: + """校验行数据，返回错误列表""" + return [] + + def transform_row(self, row: Dict[str, Any]) -> Dict[str, Any]: + """转换行数据""" + raise NotImplementedError + + def insert_records(self, records: List[Dict[str, Any]]) -> int: + """插入记录""" + raise NotImplementedError + + +# ============================================================================= +# 支出导入 +# ============================================================================= + +class ExpenseImporter(BaseImporter): + """ + 支出导入 + + Excel格式要求： + - 月份: 2026-01 或 2026/01/01 格式 + - 支出类型: 房租/水电费/物业费/工资/报销/平台服务费/其他 + - 金额: 数字 + - 备注: 可选 + """ + + TARGET_TABLE = "billiards_dws.dws_finance_expense_summary" + + REQUIRED_COLUMNS = ['月份', '支出类型', '金额'] + OPTIONAL_COLUMNS = ['明细', '备注'] + + def import_file(self, file_path: str) -> Dict[str, Any]: + """导入支出Excel""" + print(f"开始导入支出文件: {file_path}") + + # 读取Excel + df = pd.read_excel(file_path) + + # 校验必要列 + missing_cols = [c for c in self.REQUIRED_COLUMNS if c not in df.columns] + if missing_cols: + return {"status": "ERROR", "message": f"缺少必要列: {missing_cols}"} + + # 处理数据 + records = [] + errors = [] + + for idx, row in df.iterrows(): + row_dict = row.to_dict() + row_errors = self.validate_row(row_dict, idx + 2) # Excel行号从2开始 + + if row_errors: + errors.extend(row_errors) + continue + + record = self.transform_row(row_dict) + records.append(record) + + if errors: + print(f"校验错误: {len(errors)} 条") + for err in errors[:10]: + print(f" - {err}") + + # 插入数据 + inserted = 0 + if records: + inserted = self.insert_records(records) + + return { + "status": "SUCCESS" if not errors else "PARTIAL", + "batch_no": self.batch_no, + "total_rows": len(df), + "inserted": inserted, + "errors": len(errors), + "error_messages": errors[:10] + } + + def validate_row(self, row: Dict[str, Any], row_idx: int) -> List[str]: + errors = [] + + # 校验月份 + month = self._safe_month(row.get('月份')) + if not month: + errors.append(f"行{row_idx}: 月份格式错误") + + # 校验支出类型 + expense_type = row.get('支出类型', '').strip() + if expense_type not in EXPENSE_TYPES: + errors.append(f"行{row_idx}: 支出类型无效 '{expense_type}'") + + # 校验金额 + amount = self._safe_decimal(row.get('金额')) + if amount < 0: + errors.append(f"行{row_idx}: 金额不能为负数") + + return errors + + def transform_row(self, row: Dict[str, Any]) -> Dict[str, Any]: + expense_type_name = row.get('支出类型', '').strip() + expense_type_code = EXPENSE_TYPES.get(expense_type_name, 'OTHER') + expense_category = EXPENSE_CATEGORIES.get(expense_type_code, 'OTHER') + + return { + 'site_id': self.site_id, + 'tenant_id': self.tenant_id, + 'expense_month': self._safe_month(row.get('月份')), + 'expense_type_code': expense_type_code, + 'expense_type_name': expense_type_name, + 'expense_category': expense_category, + 'expense_amount': self._safe_decimal(row.get('金额')), + 'expense_detail': row.get('明细'), + 'import_batch_no': self.batch_no, + 'import_file_name': os.path.basename(str(row.get('_file_path', ''))), + 'import_time': datetime.now(), + 'import_user': os.getenv('USERNAME', 'system'), + 'remark': row.get('备注'), + } + + def insert_records(self, records: List[Dict[str, Any]]) -> int: + columns = [ + 'site_id', 'tenant_id', 'expense_month', 'expense_type_code', + 'expense_type_name', 'expense_category', 'expense_amount', + 'expense_detail', 'import_batch_no', 'import_file_name', + 'import_time', 'import_user', 'remark' + ] + + cols_str = ", ".join(columns) + placeholders = ", ".join(["%s"] * len(columns)) + sql = f"INSERT INTO {self.TARGET_TABLE} ({cols_str}) VALUES ({placeholders})" + + inserted = 0 + with self.db.conn.cursor() as cur: + for record in records: + values = [record.get(col) for col in columns] + cur.execute(sql, values) + inserted += cur.rowcount + + self.db.commit() + return inserted + + +# ============================================================================= +# 平台结算导入 +# ============================================================================= + +class PlatformSettlementImporter(BaseImporter): + """ + 平台结算导入 + + Excel格式要求： + - 回款日期: 日期格式 + - 平台类型: 美团/抖音/大众点评/其他 + - 平台订单号: 字符串 + - 订单原始金额: 数字 + - 佣金: 数字 + - 服务费: 数字 + - 回款金额: 数字 + - 备注: 可选 + """ + + TARGET_TABLE = "billiards_dws.dws_platform_settlement" + + REQUIRED_COLUMNS = ['回款日期', '平台类型', '回款金额'] + OPTIONAL_COLUMNS = ['平台订单号', '订单原始金额', '佣金', '服务费', '关联订单ID', '备注'] + + def import_file(self, file_path: str) -> Dict[str, Any]: + print(f"开始导入平台结算文件: {file_path}") + + df = pd.read_excel(file_path) + + missing_cols = [c for c in self.REQUIRED_COLUMNS if c not in df.columns] + if missing_cols: + return {"status": "ERROR", "message": f"缺少必要列: {missing_cols}"} + + records = [] + errors = [] + + for idx, row in df.iterrows(): + row_dict = row.to_dict() + row_errors = self.validate_row(row_dict, idx + 2) + + if row_errors: + errors.extend(row_errors) + continue + + record = self.transform_row(row_dict) + records.append(record) + + if errors: + print(f"校验错误: {len(errors)} 条") + for err in errors[:10]: + print(f" - {err}") + + inserted = 0 + if records: + inserted = self.insert_records(records) + + return { + "status": "SUCCESS" if not errors else "PARTIAL", + "batch_no": self.batch_no, + "total_rows": len(df), + "inserted": inserted, + "errors": len(errors), + } + + def validate_row(self, row: Dict[str, Any], row_idx: int) -> List[str]: + errors = [] + + settlement_date = self._safe_date(row.get('回款日期')) + if not settlement_date: + errors.append(f"行{row_idx}: 回款日期格式错误") + + platform_type = row.get('平台类型', '').strip() + if platform_type not in PLATFORM_TYPES: + errors.append(f"行{row_idx}: 平台类型无效 '{platform_type}'") + + amount = self._safe_decimal(row.get('回款金额')) + if amount < 0: + errors.append(f"行{row_idx}: 回款金额不能为负数") + + return errors + + def transform_row(self, row: Dict[str, Any]) -> Dict[str, Any]: + platform_name = row.get('平台类型', '').strip() + platform_type = PLATFORM_TYPES.get(platform_name, 'OTHER') + + return { + 'site_id': self.site_id, + 'tenant_id': self.tenant_id, + 'settlement_date': self._safe_date(row.get('回款日期')), + 'platform_type': platform_type, + 'platform_name': platform_name, + 'platform_order_no': row.get('平台订单号'), + 'order_settle_id': row.get('关联订单ID'), + 'settlement_amount': self._safe_decimal(row.get('回款金额')), + 'commission_amount': self._safe_decimal(row.get('佣金')), + 'service_fee': self._safe_decimal(row.get('服务费')), + 'gross_amount': self._safe_decimal(row.get('订单原始金额')), + 'import_batch_no': self.batch_no, + 'import_file_name': os.path.basename(str(row.get('_file_path', ''))), + 'import_time': datetime.now(), + 'import_user': os.getenv('USERNAME', 'system'), + 'remark': row.get('备注'), + } + + def insert_records(self, records: List[Dict[str, Any]]) -> int: + columns = [ + 'site_id', 'tenant_id', 'settlement_date', 'platform_type', + 'platform_name', 'platform_order_no', 'order_settle_id', + 'settlement_amount', 'commission_amount', 'service_fee', + 'gross_amount', 'import_batch_no', 'import_file_name', + 'import_time', 'import_user', 'remark' + ] + + cols_str = ", ".join(columns) + placeholders = ", ".join(["%s"] * len(columns)) + sql = f"INSERT INTO {self.TARGET_TABLE} ({cols_str}) VALUES ({placeholders})" + + inserted = 0 + with self.db.conn.cursor() as cur: + for record in records: + values = [record.get(col) for col in columns] + cur.execute(sql, values) + inserted += cur.rowcount + + self.db.commit() + return inserted + + +# ============================================================================= +# 充值提成导入 +# ============================================================================= + +class RechargeCommissionImporter(BaseImporter): + """ + 充值提成导入 + + Excel格式要求： + - 月份: 2026-01 格式 + - 助教ID: 数字 + - 助教花名: 字符串 + - 充值订单金额: 数字 + - 提成金额: 数字 + - 充值订单号: 可选 + - 备注: 可选 + """ + + TARGET_TABLE = "billiards_dws.dws_assistant_recharge_commission" + + REQUIRED_COLUMNS = ['月份', '助教ID', '提成金额'] + OPTIONAL_COLUMNS = ['助教花名', '充值订单金额', '充值订单ID', '充值订单号', '备注'] + + def import_file(self, file_path: str) -> Dict[str, Any]: + print(f"开始导入充值提成文件: {file_path}") + + df = pd.read_excel(file_path) + + missing_cols = [c for c in self.REQUIRED_COLUMNS if c not in df.columns] + if missing_cols: + return {"status": "ERROR", "message": f"缺少必要列: {missing_cols}"} + + records = [] + errors = [] + + for idx, row in df.iterrows(): + row_dict = row.to_dict() + row_errors = self.validate_row(row_dict, idx + 2) + + if row_errors: + errors.extend(row_errors) + continue + + record = self.transform_row(row_dict) + records.append(record) + + if errors: + print(f"校验错误: {len(errors)} 条") + for err in errors[:10]: + print(f" - {err}") + + inserted = 0 + if records: + inserted = self.insert_records(records) + + return { + "status": "SUCCESS" if not errors else "PARTIAL", + "batch_no": self.batch_no, + "total_rows": len(df), + "inserted": inserted, + "errors": len(errors), + } + + def validate_row(self, row: Dict[str, Any], row_idx: int) -> List[str]: + errors = [] + + month = self._safe_month(row.get('月份')) + if not month: + errors.append(f"行{row_idx}: 月份格式错误") + + assistant_id = row.get('助教ID') + if assistant_id is None or pd.isna(assistant_id): + errors.append(f"行{row_idx}: 助教ID不能为空") + + amount = self._safe_decimal(row.get('提成金额')) + if amount < 0: + errors.append(f"行{row_idx}: 提成金额不能为负数") + + return errors + + def transform_row(self, row: Dict[str, Any]) -> Dict[str, Any]: + recharge_amount = self._safe_decimal(row.get('充值订单金额')) + commission_amount = self._safe_decimal(row.get('提成金额')) + commission_ratio = commission_amount / recharge_amount if recharge_amount > 0 else None + + return { + 'site_id': self.site_id, + 'tenant_id': self.tenant_id, + 'assistant_id': int(row.get('助教ID')), + 'assistant_nickname': row.get('助教花名'), + 'commission_month': self._safe_month(row.get('月份')), + 'recharge_order_id': row.get('充值订单ID'), + 'recharge_order_no': row.get('充值订单号'), + 'recharge_amount': recharge_amount, + 'commission_amount': commission_amount, + 'commission_ratio': commission_ratio, + 'import_batch_no': self.batch_no, + 'import_file_name': os.path.basename(str(row.get('_file_path', ''))), + 'import_time': datetime.now(), + 'import_user': os.getenv('USERNAME', 'system'), + 'remark': row.get('备注'), + } + + def insert_records(self, records: List[Dict[str, Any]]) -> int: + columns = [ + 'site_id', 'tenant_id', 'assistant_id', 'assistant_nickname', + 'commission_month', 'recharge_order_id', 'recharge_order_no', + 'recharge_amount', 'commission_amount', 'commission_ratio', + 'import_batch_no', 'import_file_name', 'import_time', + 'import_user', 'remark' + ] + + cols_str = ", ".join(columns) + placeholders = ", ".join(["%s"] * len(columns)) + sql = f"INSERT INTO {self.TARGET_TABLE} ({cols_str}) VALUES ({placeholders})" + + inserted = 0 + with self.db.conn.cursor() as cur: + for record in records: + values = [record.get(col) for col in columns] + cur.execute(sql, values) + inserted += cur.rowcount + + self.db.commit() + return inserted + + +# ============================================================================= +# 主函数 +# ============================================================================= + +def main(): + parser = argparse.ArgumentParser(description='DWS Excel导入工具') + parser.add_argument( + '--type', '-t', + choices=['expense', 'platform', 'commission'], + required=True, + help='导入类型: expense(支出), platform(平台结算), commission(充值提成)' + ) + parser.add_argument( + '--file', '-f', + required=True, + help='Excel文件路径' + ) + + args = parser.parse_args() + + # 检查文件 + if not os.path.exists(args.file): + print(f"文件不存在: {args.file}") + sys.exit(1) + + # 加载配置 + config = Config() + db = DatabaseConnection(config) + + try: + # 选择导入器 + if args.type == 'expense': + importer = ExpenseImporter(config, db) + elif args.type == 'platform': + importer = PlatformSettlementImporter(config, db) + elif args.type == 'commission': + importer = RechargeCommissionImporter(config, db) + else: + print(f"未知的导入类型: {args.type}") + sys.exit(1) + + # 执行导入 + result = importer.import_file(args.file) + + # 输出结果 + print("\n" + "=" * 50) + print("导入结果:") + print(f" 状态: {result.get('status')}") + print(f" 批次号: {result.get('batch_no')}") + print(f" 总行数: {result.get('total_rows')}") + print(f" 插入行数: {result.get('inserted')}") + print(f" 错误行数: {result.get('errors')}") + + if result.get('status') == 'ERROR': + print(f" 错误信息: {result.get('message')}") + sys.exit(1) + + except Exception as e: + print(f"导入失败: {e}") + db.rollback() + sys.exit(1) + finally: + db.close() + + +if __name__ == "__main__": + main() diff --git a/etl_billiards/scripts/run_seed_dws_config.py b/etl_billiards/scripts/run_seed_dws_config.py new file mode 100644 index 0000000..e024ef4 --- /dev/null +++ b/etl_billiards/scripts/run_seed_dws_config.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +"""执行DWS配置数据导入""" + +import os +from pathlib import Path +from dotenv import load_dotenv +import psycopg2 + +def main(): + # 加载.env配置 + env_path = Path(__file__).parent.parent / ".env" + load_dotenv(env_path) + + dsn = os.getenv("PG_DSN") + if not dsn: + print("错误: 未找到 PG_DSN 配置") + return + + # 读取SQL文件 + sql_file = Path(__file__).parent.parent / "database" / "seed_dws_config.sql" + sql_content = sql_file.read_text(encoding="utf-8") + + print(f"连接数据库...") + conn = psycopg2.connect(dsn) + conn.autocommit = True + + with conn.cursor() as cur: + print(f"执行SQL文件: {sql_file}") + cur.execute(sql_content) + print("DWS配置数据导入成功!") + + conn.close() + +if __name__ == "__main__": + main() diff --git a/etl_billiards/scripts/show_area_category.py b/etl_billiards/scripts/show_area_category.py new file mode 100644 index 0000000..d30fe94 --- /dev/null +++ b/etl_billiards/scripts/show_area_category.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +"""显示台区分类映射数据""" + +import os +from pathlib import Path +from dotenv import load_dotenv +import psycopg2 + +def main(): + load_dotenv(Path(__file__).parent.parent / ".env") + dsn = os.getenv("PG_DSN") + conn = psycopg2.connect(dsn) + + print("cfg_area_category 数据内容:") + print("=" * 90) + print(f"{'source_area_name':<15} {'category_code':<15} {'category_name':<12} {'match_type':<10} {'priority':<8}") + print("-" * 90) + + with conn.cursor() as cur: + cur.execute(""" + SELECT source_area_name, category_code, category_name, match_type, match_priority + FROM billiards_dws.cfg_area_category + ORDER BY match_priority, category_code, source_area_name + """) + for row in cur.fetchall(): + print(f"{row[0]:<15} {row[1]:<15} {row[2]:<12} {row[3]:<10} {row[4]:<8}") + + print("=" * 90) + print("\n分类汇总:") + with conn.cursor() as cur: + cur.execute(""" + SELECT category_code, category_name, COUNT(*) as cnt + FROM billiards_dws.cfg_area_category + GROUP BY category_code, category_name + ORDER BY category_code + """) + for row in cur.fetchall(): + print(f" {row[0]:<15} {row[1]:<12} {row[2]} 条规则") + + conn.close() + +if __name__ == "__main__": + main() diff --git a/etl_billiards/scripts/show_performance_tier.py b/etl_billiards/scripts/show_performance_tier.py new file mode 100644 index 0000000..b3448a0 --- /dev/null +++ b/etl_billiards/scripts/show_performance_tier.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +"""显示绩效档位配置数据""" + +import os +from pathlib import Path +from dotenv import load_dotenv +import psycopg2 + +def main(): + load_dotenv(Path(__file__).parent.parent / ".env") + dsn = os.getenv("PG_DSN") + conn = psycopg2.connect(dsn) + + print("cfg_performance_tier 数据内容:") + print("=" * 110) + print(f"{'tier_code':<8} {'tier_name':<18} {'min_hours':<10} {'max_hours':<10} {'base_ded':<10} {'bonus_ded':<10} {'vacation':<10}") + print("-" * 110) + + with conn.cursor() as cur: + cur.execute(""" + SELECT tier_code, tier_name, min_hours, max_hours, + base_deduction, bonus_deduction_ratio, + vacation_days, vacation_unlimited + FROM billiards_dws.cfg_performance_tier + ORDER BY tier_level + """) + for row in cur.fetchall(): + max_h = str(row[3]) if row[3] else "NULL" + vac = "自由" if row[7] else str(row[6]) + "天" + print(f"{row[0]:<8} {row[1]:<18} {row[2]:<10} {max_h:<10} {row[4]:<10} {row[5]*100:.0f}%{'':<7} {vac:<10}") + + print("=" * 110) + print("\n数据来源依据: DWS 数据库处理需求.md 第35-41行") + print(""" +| 档位 | 总业绩小时数阈值 | 专业课抽成 | 打赏课抽成 | 次月休假 | +|------|------------------|-----------|-----------|----------| +| 0档 | H < 100 | 28元/小时 | 50% | 3天 | +| 1档 | 100 ≤ H < 130 | 18元/小时 | 40% | 4天 | +| 2档 | 130 ≤ H < 160 | 15元/小时 | 38% | 4天 | +| 3档 | 160 ≤ H < 190 | 13元/小时 | 35% | 5天 | +| 4档 | 190 ≤ H < 220 | 10元/小时 | 33% | 6天 | +| 5档 | H ≥ 220 | 8元/小时 | 30% | 休假自由 | +""") + + conn.close() + +if __name__ == "__main__": + main() diff --git a/etl_billiards/scripts/test_index_tasks.py b/etl_billiards/scripts/test_index_tasks.py new file mode 100644 index 0000000..4d47c0a --- /dev/null +++ b/etl_billiards/scripts/test_index_tasks.py @@ -0,0 +1,222 @@ +# -*- coding: utf-8 -*- +""" +测试指数算法任务 +""" +import sys +import os + +# 添加项目路径 +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import logging +from config.settings import AppConfig +from database.connection import DatabaseConnection +from database.operations import DatabaseOperations +from tasks.dws.index import RecallIndexTask, IntimacyIndexTask + +# 配置日志 +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) +logger = logging.getLogger('test_index') + +def test_recall_index(): + """测试召回指数任务""" + logger.info("=" * 60) + logger.info("测试客户召回指数任务 (DWS_RECALL_INDEX)") + logger.info("=" * 60) + + # 加载配置 + config = AppConfig.load() + + # 连接数据库 + db_conn = DatabaseConnection(config.config["db"]["dsn"]) + db = DatabaseOperations(db_conn) + + try: + # 创建任务实例 + task = RecallIndexTask(config, db, None, logger) + + # 执行任务 + result = task.execute(None) + + logger.info("任务执行结果: %s", result) + + # 查询结果 + if result.get('status') == 'success': + sql = """ + SELECT + COUNT(*) as total_count, + ROUND(AVG(display_score)::numeric, 2) as avg_score, + ROUND(MIN(display_score)::numeric, 2) as min_score, + ROUND(MAX(display_score)::numeric, 2) as max_score, + ROUND(AVG(raw_score)::numeric, 4) as avg_raw_score, + ROUND(AVG(score_overdue)::numeric, 4) as avg_overdue, + ROUND(AVG(score_new_bonus)::numeric, 4) as avg_new_bonus, + ROUND(AVG(score_recharge_bonus)::numeric, 4) as avg_recharge_bonus, + ROUND(AVG(score_hot_drop)::numeric, 4) as avg_hot_drop + FROM billiards_dws.dws_member_recall_index + """ + rows = db.query(sql) + if rows: + stats = dict(rows[0]) + logger.info("-" * 40) + logger.info("召回指数统计:") + logger.info(" 总记录数: %s", stats['total_count']) + logger.info(" Display Score: 平均=%.2f, 最小=%.2f, 最大=%.2f", + stats['avg_score'] or 0, stats['min_score'] or 0, stats['max_score'] or 0) + logger.info(" Raw Score 平均: %.4f", stats['avg_raw_score'] or 0) + logger.info(" 分项得分平均:") + logger.info(" - 超期紧急性: %.4f", stats['avg_overdue'] or 0) + logger.info(" - 新客户加分: %.4f", stats['avg_new_bonus'] or 0) + logger.info(" - 充值加分: %.4f", stats['avg_recharge_bonus'] or 0) + logger.info(" - 热度断档: %.4f", stats['avg_hot_drop'] or 0) + + # 查询Top 5 + logger.info("-" * 40) + logger.info("召回优先级 Top 5:") + top_sql = """ + SELECT member_id, display_score, raw_score, + days_since_last_visit, visit_interval_median + FROM billiards_dws.dws_member_recall_index + ORDER BY display_score DESC + LIMIT 5 + """ + top_rows = db.query(top_sql) + for i, row in enumerate(top_rows or [], 1): + r = dict(row) + logger.info(" %d. 会员%s: %.2f分 (Raw=%.4f, 最近到店=%s天前, 周期=%.1f天)", + i, r['member_id'], r['display_score'] or 0, r['raw_score'] or 0, + r['days_since_last_visit'], r['visit_interval_median'] or 0) + + return result + + finally: + db_conn.close() + + +def test_intimacy_index(): + """测试亲密指数任务""" + logger.info("") + logger.info("=" * 60) + logger.info("测试客户-助教亲密指数任务 (DWS_INTIMACY_INDEX)") + logger.info("=" * 60) + + # 加载配置 + config = AppConfig.load() + + # 连接数据库 + db_conn = DatabaseConnection(config.config["db"]["dsn"]) + db = DatabaseOperations(db_conn) + + try: + # 创建任务实例 + task = IntimacyIndexTask(config, db, None, logger) + + # 执行任务 + result = task.execute(None) + + logger.info("任务执行结果: %s", result) + + # 查询结果 + if result.get('status') == 'success': + sql = """ + SELECT + COUNT(*) as total_count, + COUNT(DISTINCT member_id) as unique_members, + COUNT(DISTINCT assistant_id) as unique_assistants, + ROUND(AVG(display_score)::numeric, 2) as avg_score, + ROUND(MIN(display_score)::numeric, 2) as min_score, + ROUND(MAX(display_score)::numeric, 2) as max_score, + ROUND(AVG(raw_score)::numeric, 4) as avg_raw_score, + ROUND(AVG(score_frequency)::numeric, 4) as avg_frequency, + ROUND(AVG(score_recency)::numeric, 4) as avg_recency, + ROUND(AVG(score_recharge)::numeric, 4) as avg_recharge, + ROUND(AVG(burst_multiplier)::numeric, 4) as avg_burst + FROM billiards_dws.dws_member_assistant_intimacy + """ + rows = db.query(sql) + if rows: + stats = dict(rows[0]) + logger.info("-" * 40) + logger.info("亲密指数统计:") + logger.info(" 总记录数: %s (客户-助教对)", stats['total_count']) + logger.info(" 唯一会员: %s, 唯一助教: %s", stats['unique_members'], stats['unique_assistants']) + logger.info(" Display Score: 平均=%.2f, 最小=%.2f, 最大=%.2f", + stats['avg_score'] or 0, stats['min_score'] or 0, stats['max_score'] or 0) + logger.info(" Raw Score 平均: %.4f", stats['avg_raw_score'] or 0) + logger.info(" 分项得分平均:") + logger.info(" - 频次强度: %.4f", stats['avg_frequency'] or 0) + logger.info(" - 最近温度: %.4f", stats['avg_recency'] or 0) + logger.info(" - 充值强度: %.4f", stats['avg_recharge'] or 0) + logger.info(" - 激增放大: %.4f", stats['avg_burst'] or 0) + + # 查询Top亲密关系 + logger.info("-" * 40) + logger.info("亲密度 Top 5 客户-助教对:") + top_sql = """ + SELECT member_id, assistant_id, display_score, raw_score, + session_count, attributed_recharge_amount + FROM billiards_dws.dws_member_assistant_intimacy + ORDER BY display_score DESC + LIMIT 5 + """ + top_rows = db.query(top_sql) + for i, row in enumerate(top_rows or [], 1): + r = dict(row) + logger.info(" %d. 会员%s-助教%s: %.2f分 (会话%d次, 归因充值%.2f元)", + i, r['member_id'], r['assistant_id'], + r['display_score'] or 0, r['session_count'] or 0, + r['attributed_recharge_amount'] or 0) + + return result + + finally: + db_conn.close() + + +if __name__ == '__main__': + print("=" * 60) + print("指数算法任务测试") + print("=" * 60) + print() + + # 先检查表是否存在 + config = AppConfig.load() + db_conn = DatabaseConnection(config.config["db"]["dsn"]) + db = DatabaseOperations(db_conn) + + check_sql = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'billiards_dws' + AND table_name IN ('dws_member_recall_index', 'dws_member_assistant_intimacy', 'cfg_index_parameters') + """ + tables = db.query(check_sql) + existing_tables = [dict(r)['table_name'] for r in (tables or [])] + + if 'cfg_index_parameters' not in existing_tables: + print("警告: cfg_index_parameters 表不存在，请先执行 schema_dws.sql") + print("需要执行的表:") + print(" - cfg_index_parameters") + print(" - dws_member_recall_index") + print(" - dws_member_assistant_intimacy") + print(" - dws_index_percentile_history") + db_conn.close() + sys.exit(1) + + db_conn.close() + + # 测试召回指数 + recall_result = test_recall_index() + + # 测试亲密指数 + intimacy_result = test_intimacy_index() + + print() + print("=" * 60) + print("测试完成") + print("=" * 60) + print(f"召回指数: {recall_result.get('status', 'unknown')}") + print(f"亲密指数: {intimacy_result.get('status', 'unknown')}") diff --git a/etl_billiards/scripts/verify_dws_config.py b/etl_billiards/scripts/verify_dws_config.py new file mode 100644 index 0000000..cc69ebd --- /dev/null +++ b/etl_billiards/scripts/verify_dws_config.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +"""验证DWS配置数据""" + +import os +from pathlib import Path +from dotenv import load_dotenv +import psycopg2 + +def main(): + load_dotenv(Path(__file__).parent.parent / ".env") + dsn = os.getenv("PG_DSN") + conn = psycopg2.connect(dsn) + + tables = [ + "cfg_performance_tier", + "cfg_assistant_level_price", + "cfg_bonus_rules", + "cfg_area_category", + "cfg_skill_type" + ] + + print("DWS 配置表数据统计:") + print("-" * 40) + + with conn.cursor() as cur: + for t in tables: + cur.execute(f"SELECT COUNT(*) FROM billiards_dws.{t}") + cnt = cur.fetchone()[0] + print(f"{t}: {cnt} 行") + + conn.close() + +if __name__ == "__main__": + main() diff --git a/etl_billiards/tasks/dwd_load_task.py b/etl_billiards/tasks/dwd_load_task.py index 816c1d9..bd4853b 100644 --- a/etl_billiards/tasks/dwd_load_task.py +++ b/etl_billiards/tasks/dwd_load_task.py @@ -155,6 +155,7 @@ class DwdLoadTask(BaseTask): ("table_id", "id", None), ("site_table_area_name", "areaname", None), ("tenant_table_area_id", "site_table_area_id", None), + ("order_id", "order_id", None), ], "billiards_dwd.dim_table_ex": [ ("table_id", "id", None), @@ -167,12 +168,23 @@ class DwdLoadTask(BaseTask): ("group_name", "group_name", None), ("light_equipment_id", "light_equipment_id", None), ], - "billiards_dwd.dim_member": [("member_id", "id", None)], + "billiards_dwd.dim_member": [ + ("member_id", "id", None), + ("pay_money_sum", "pay_money_sum", None), + ("recharge_money_sum", "recharge_money_sum", None), + ], "billiards_dwd.dim_member_ex": [ ("member_id", "id", None), ("register_site_name", "site_name", None), + ("person_tenant_org_id", "person_tenant_org_id", None), + ("person_tenant_org_name", "person_tenant_org_name", None), + ("register_source", "register_source", None), + ], + "billiards_dwd.dim_member_card_account": [ + ("member_card_id", "id", None), + ("principal_balance", "principal_balance", None), + ("member_grade", "member_grade", None), ], - "billiards_dwd.dim_member_card_account": [("member_card_id", "id", None)], "billiards_dwd.dim_member_card_account_ex": [ ("member_card_id", "id", None), ("tenant_name", "tenantname", None), @@ -182,10 +194,16 @@ class DwdLoadTask(BaseTask): ("use_scene", "use_scene", None), ("tableareaid", "tableareaid", None), ("goodscategoryid", "goodscategoryid", None), + ("able_share_member_discount", "able_share_member_discount", "boolean"), + ("electricity_deduct_radio", "electricity_deduct_radio", None), + ("electricity_discount", "electricity_discount", None), + ("electricity_card_deduct", "electricitycarddeduct", "boolean"), + ("recharge_freeze_balance", "rechargefreezebalance", None), ], "billiards_dwd.dim_tenant_goods": [ ("tenant_goods_id", "id", None), ("category_name", "categoryname", None), + ("not_sale", "not_sale", None), ], "billiards_dwd.dim_tenant_goods_ex": [ ("tenant_goods_id", "id", None), @@ -204,6 +222,8 @@ class DwdLoadTask(BaseTask): ("batch_stock_qty", "stock", None), ("sale_qty", "sale_num", None), ("total_sales_qty", "total_sales", None), + ("commodity_code", "commodity_code", None), + ("not_sale", "not_sale", None), ], "billiards_dwd.dim_store_goods_ex": [ ("site_goods_id", "id", None), @@ -239,6 +259,8 @@ class DwdLoadTask(BaseTask): ("package_template_id", "package_id", None), ("coupon_face_value", "coupon_money", None), ("duration_seconds", "duration", None), + ("sort", "sort", None), + ("is_first_limit", "is_first_limit", "boolean"), ], "billiards_dwd.dim_groupbuy_package_ex": [ ("groupbuy_package_id", "id", None), @@ -247,12 +269,18 @@ class DwdLoadTask(BaseTask): ("usable_range", "usable_range", None), ("table_area_id_list", "table_area_id_list", None), ("package_type", "type", None), + ("tenant_coupon_sale_order_item_id", "tenantcouponsaleorderitemid", None), ], # 事实表主键及关键差异列 - "billiards_dwd.dwd_table_fee_log": [("table_fee_log_id", "id", None)], + "billiards_dwd.dwd_table_fee_log": [ + ("table_fee_log_id", "id", None), + ("activity_discount_amount", "activity_discount_amount", None), + ("real_service_money", "real_service_money", None), + ], "billiards_dwd.dwd_table_fee_log_ex": [ ("table_fee_log_id", "id", None), ("salesman_name", "salesman_name", None), + ("order_consumption_type", "order_consumption_type", None), ], "billiards_dwd.dwd_table_fee_adjust": [ ("table_fee_adjust_id", "id", None), @@ -260,12 +288,24 @@ class DwdLoadTask(BaseTask): ("table_area_id", "tenant_table_area_id", None), ("table_area_name", "tableprofile->>'table_area_name'", None), ("adjust_time", "create_time", None), + ("table_name", "table_name", None), + ("table_price", "table_price", None), + ("charge_free", "charge_free", "boolean"), ], "billiards_dwd.dwd_table_fee_adjust_ex": [ ("table_fee_adjust_id", "id", None), ("ledger_name", "ledger_name", None), + ("area_type_id", "area_type_id", None), + ("site_table_area_id", "site_table_area_id", None), + ("site_table_area_name", "site_table_area_name", None), + ("site_name", "sitename", None), + ("tenant_name", "tenant_name", None), + ], + "billiards_dwd.dwd_store_goods_sale": [ + ("store_goods_sale_id", "id", None), + ("discount_price", "discount_money", None), + ("coupon_share_money", "coupon_share_money", None), ], - "billiards_dwd.dwd_store_goods_sale": [("store_goods_sale_id", "id", None), ("discount_price", "discount_money", None)], "billiards_dwd.dwd_store_goods_sale_ex": [ ("store_goods_sale_id", "id", None), ("option_value_name", "option_value_name", None), @@ -282,6 +322,7 @@ class DwdLoadTask(BaseTask): ("site_assistant_id", "order_assistant_id", None), ("level_name", "levelname", None), ("skill_name", "skillname", None), + ("real_service_money", "real_service_money", None), ], "billiards_dwd.dwd_assistant_service_log_ex": [ ("assistant_service_id", "id", None), @@ -291,6 +332,7 @@ class DwdLoadTask(BaseTask): ("trash_reason", "trash_reason", None), ("salesman_name", "salesman_name", None), ("table_name", "tablename", None), + ("assistant_team_name", "assistantteamname", None), ], "billiards_dwd.dwd_assistant_trash_event": [ ("assistant_trash_event_id", "id", None), @@ -303,6 +345,7 @@ class DwdLoadTask(BaseTask): ("assistant_name", "assistantname", None), ("trash_reason", "trashreason", None), ("create_time", "createtime", None), + ("tenant_id", "tenant_id", None), ], "billiards_dwd.dwd_assistant_trash_event_ex": [ ("assistant_trash_event_id", "id", None), @@ -318,13 +361,20 @@ class DwdLoadTask(BaseTask): ("change_time", "create_time", None), ("member_name", "membername", None), ("member_mobile", "membermobile", None), + ("principal_before", "principal_before", None), + ("principal_after", "principal_after", None), ], "billiards_dwd.dwd_member_balance_change_ex": [ ("balance_change_id", "id", None), ("pay_site_name", "paysitename", None), ("register_site_name", "registersitename", None), + ("principal_data", "principal_data", None), + ], + "billiards_dwd.dwd_groupbuy_redemption": [ + ("redemption_id", "id", None), + ("member_discount_money", "member_discount_money", None), + ("coupon_sale_id", "coupon_sale_id", None), ], - "billiards_dwd.dwd_groupbuy_redemption": [("redemption_id", "id", None)], "billiards_dwd.dwd_groupbuy_redemption_ex": [ ("redemption_id", "id", None), ("table_area_name", "tableareaname", None), @@ -334,13 +384,24 @@ class DwdLoadTask(BaseTask): ("salesman_name", "salesman_name", None), ("salesman_org_id", "sales_man_org_id", None), ("ledger_group_name", "ledger_group_name", None), + ("table_share_money", "table_share_money", None), + ("table_service_share_money", "table_service_share_money", None), + ("goods_share_money", "goods_share_money", None), + ("good_service_share_money", "good_service_share_money", None), + ("assistant_share_money", "assistant_share_money", None), + ("assistant_service_share_money", "assistant_service_share_money", None), + ("recharge_share_money", "recharge_share_money", None), ], "billiards_dwd.dwd_platform_coupon_redemption": [("platform_coupon_redemption_id", "id", None)], "billiards_dwd.dwd_platform_coupon_redemption_ex": [ ("platform_coupon_redemption_id", "id", None), ("coupon_cover", "coupon_cover", None), ], - "billiards_dwd.dwd_payment": [("payment_id", "id", None), ("pay_date", "pay_time", "date")], + "billiards_dwd.dwd_payment": [ + ("payment_id", "id", None), + ("pay_date", "pay_time", "date"), + ("tenant_id", "tenant_id", None), + ], "billiards_dwd.dwd_refund": [("refund_id", "id", None)], "billiards_dwd.dwd_refund_ex": [ ("refund_id", "id", None), @@ -382,6 +443,11 @@ class DwdLoadTask(BaseTask): ("coupon_amount", "couponamount", None), ("rounding_amount", "roundingamount", None), ("point_amount", "pointamount", None), + ("electricity_money", "electricitymoney", None), + ("real_electricity_money", "realelectricitymoney", None), + ("electricity_adjust_money", "electricityadjustmoney", None), + ("pl_coupon_sale_amount", "plcouponsaleamount", None), + ("mervou_sales_amount", "mervousalesamount", None), ], "billiards_dwd.dwd_settlement_head_ex": [ ("order_settle_id", "id", None), @@ -414,6 +480,7 @@ class DwdLoadTask(BaseTask): ("order_remark", "orderremark", None), ("operator_id", "operatorid", None), ("salesman_user_id", "salesmanuserid", None), + ("settle_list", "settlelist", None), ], # 充值结算：recharge_settlements（字段风格同 settlement_records） "billiards_dwd.dwd_recharge_order": [ diff --git a/etl_billiards/tasks/dws/__init__.py b/etl_billiards/tasks/dws/__init__.py new file mode 100644 index 0000000..ac2721c --- /dev/null +++ b/etl_billiards/tasks/dws/__init__.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +DWS层ETL任务模块 + +包含： +- BaseDwsTask: DWS任务基类 +- 助教维度任务 +- 客户维度任务 +- 财务维度任务 +- 指数算法任务 +""" + +from .base_dws_task import BaseDwsTask, TimeLayer, TimeWindow, CourseType, DiscountType +from .assistant_daily_task import AssistantDailyTask +from .assistant_monthly_task import AssistantMonthlyTask +from .assistant_customer_task import AssistantCustomerTask +from .assistant_salary_task import AssistantSalaryTask +from .assistant_finance_task import AssistantFinanceTask +from .member_consumption_task import MemberConsumptionTask +from .member_visit_task import MemberVisitTask +from .finance_daily_task import FinanceDailyTask +from .finance_recharge_task import FinanceRechargeTask +from .finance_income_task import FinanceIncomeStructureTask +from .finance_discount_task import FinanceDiscountDetailTask +from .retention_cleanup_task import DwsRetentionCleanupTask + +# 指数算法任务 +from .index import RecallIndexTask, IntimacyIndexTask + +__all__ = [ + # 基类 + "BaseDwsTask", + "TimeLayer", + "TimeWindow", + "CourseType", + "DiscountType", + # 助教维度 + "AssistantDailyTask", + "AssistantMonthlyTask", + "AssistantCustomerTask", + "AssistantSalaryTask", + "AssistantFinanceTask", + # 客户维度 + "MemberConsumptionTask", + "MemberVisitTask", + # 财务维度 + "FinanceDailyTask", + "FinanceRechargeTask", + "FinanceIncomeStructureTask", + "FinanceDiscountDetailTask", + "DwsRetentionCleanupTask", + # 指数算法 + "RecallIndexTask", + "IntimacyIndexTask", +] diff --git a/etl_billiards/tasks/dws/assistant_customer_task.py b/etl_billiards/tasks/dws/assistant_customer_task.py new file mode 100644 index 0000000..08698f8 --- /dev/null +++ b/etl_billiards/tasks/dws/assistant_customer_task.py @@ -0,0 +1,333 @@ +# -*- coding: utf-8 -*- +""" +助教服务客户统计任务 + +功能说明： + 以"助教+客户"为粒度，统计服务关系和滚动窗口指标 + +数据来源： + - dwd_assistant_service_log: 助教服务流水 + - dim_member: 会员维度 + +目标表： + billiards_dws.dws_assistant_customer_stats + +更新策略： + - 更新频率：每日更新 + - 幂等方式：delete-before-insert（按统计日期） + +业务规则： + - 散客处理：member_id=0 不进入此表统计 + - 滚动窗口：7/10/15/30/60/90天 + - 活跃度：近7天/30天是否有服务 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Set, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class AssistantCustomerTask(BaseDwsTask): + """ + 助教服务客户统计任务 + + 统计每个助教与每个客户的服务关系： + - 首次/最近服务日期 + - 累计服务统计 + - 滚动窗口统计（7/10/15/30/60/90天） + - 活跃度指标 + """ + + def get_task_code(self) -> str: + return "DWS_ASSISTANT_CUSTOMER" + + def get_target_table(self) -> str: + return "dws_assistant_customer_stats" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "assistant_id", "member_id", "stat_date"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据 + """ + stat_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + self.logger.info( + "%s: 提取数据，统计日期 %s", + self.get_task_code(), stat_date + ) + + # 计算最大回溯日期（90天窗口） + lookback_start = stat_date - timedelta(days=90) + + # 1. 获取助教-客户服务记录（包含历史全量用于累计统计） + service_pairs = self._extract_service_pairs(site_id, stat_date) + + # 2. 获取会员信息 + member_info = self._extract_member_info(site_id) + + # 3. 获取助教信息 + assistant_info = self._extract_assistant_info(site_id) + + return { + 'service_pairs': service_pairs, + 'member_info': member_info, + 'assistant_info': assistant_info, + 'stat_date': stat_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据：计算各窗口统计 + """ + service_pairs = extracted['service_pairs'] + member_info = extracted['member_info'] + assistant_info = extracted['assistant_info'] + stat_date = extracted['stat_date'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，%d 条服务关系记录", + self.get_task_code(), len(service_pairs) + ) + + # 构建统计记录 + results = [] + + for pair in service_pairs: + assistant_id = pair.get('assistant_id') + member_id = pair.get('member_id') + + # 跳过散客 + if self.is_guest(member_id): + continue + + asst_info = assistant_info.get(assistant_id, {}) + memb_info = member_info.get(member_id, {}) + + # 构建记录 + record = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'assistant_id': assistant_id, + 'assistant_nickname': asst_info.get('nickname', pair.get('assistant_nickname')), + 'member_id': member_id, + 'member_nickname': memb_info.get('nickname'), + 'member_mobile': self._mask_mobile(memb_info.get('mobile')), + 'stat_date': stat_date, + # 全量累计统计 + 'first_service_date': pair.get('first_service_date'), + 'last_service_date': pair.get('last_service_date'), + 'total_service_count': self.safe_int(pair.get('total_service_count', 0)), + 'total_service_hours': self.safe_decimal(pair.get('total_service_hours', 0)), + 'total_service_amount': self.safe_decimal(pair.get('total_service_amount', 0)), + # 滚动窗口统计 + 'service_count_7d': self.safe_int(pair.get('service_count_7d', 0)), + 'service_count_10d': self.safe_int(pair.get('service_count_10d', 0)), + 'service_count_15d': self.safe_int(pair.get('service_count_15d', 0)), + 'service_count_30d': self.safe_int(pair.get('service_count_30d', 0)), + 'service_count_60d': self.safe_int(pair.get('service_count_60d', 0)), + 'service_count_90d': self.safe_int(pair.get('service_count_90d', 0)), + 'service_hours_7d': self.safe_decimal(pair.get('service_hours_7d', 0)), + 'service_hours_10d': self.safe_decimal(pair.get('service_hours_10d', 0)), + 'service_hours_15d': self.safe_decimal(pair.get('service_hours_15d', 0)), + 'service_hours_30d': self.safe_decimal(pair.get('service_hours_30d', 0)), + 'service_hours_60d': self.safe_decimal(pair.get('service_hours_60d', 0)), + 'service_hours_90d': self.safe_decimal(pair.get('service_hours_90d', 0)), + 'service_amount_7d': self.safe_decimal(pair.get('service_amount_7d', 0)), + 'service_amount_10d': self.safe_decimal(pair.get('service_amount_10d', 0)), + 'service_amount_15d': self.safe_decimal(pair.get('service_amount_15d', 0)), + 'service_amount_30d': self.safe_decimal(pair.get('service_amount_30d', 0)), + 'service_amount_60d': self.safe_decimal(pair.get('service_amount_60d', 0)), + 'service_amount_90d': self.safe_decimal(pair.get('service_amount_90d', 0)), + # 活跃度指标 + 'days_since_last': self._calc_days_since(stat_date, pair.get('last_service_date')), + 'is_active_7d': self.safe_int(pair.get('service_count_7d', 0)) > 0, + 'is_active_30d': self.safe_int(pair.get('service_count_30d', 0)) > 0, + } + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + # 删除已存在的数据 + deleted = self.delete_existing_data(context, date_col="stat_date") + + # 批量插入 + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_service_pairs( + self, + site_id: int, + stat_date: date + ) -> List[Dict[str, Any]]: + """ + 提取助教-客户服务统计（含滚动窗口） + """ + sql = """ + WITH service_base AS ( + SELECT + site_assistant_id AS assistant_id, + nickname AS assistant_nickname, + tenant_member_id AS member_id, + DATE(start_use_time) AS service_date, + income_seconds, + ledger_amount + FROM billiards_dwd.dwd_assistant_service_log + WHERE site_id = %s + AND tenant_member_id IS NOT NULL + AND tenant_member_id != 0 + ) + SELECT + assistant_id, + MAX(assistant_nickname) AS assistant_nickname, + member_id, + MIN(service_date) AS first_service_date, + MAX(service_date) AS last_service_date, + -- 全量累计 + COUNT(*) AS total_service_count, + SUM(income_seconds) / 3600.0 AS total_service_hours, + SUM(ledger_amount) AS total_service_amount, + -- 7天窗口 + COUNT(CASE WHEN service_date >= %s - INTERVAL '6 days' THEN 1 END) AS service_count_7d, + SUM(CASE WHEN service_date >= %s - INTERVAL '6 days' THEN income_seconds ELSE 0 END) / 3600.0 AS service_hours_7d, + SUM(CASE WHEN service_date >= %s - INTERVAL '6 days' THEN ledger_amount ELSE 0 END) AS service_amount_7d, + -- 10天窗口 + COUNT(CASE WHEN service_date >= %s - INTERVAL '9 days' THEN 1 END) AS service_count_10d, + SUM(CASE WHEN service_date >= %s - INTERVAL '9 days' THEN income_seconds ELSE 0 END) / 3600.0 AS service_hours_10d, + SUM(CASE WHEN service_date >= %s - INTERVAL '9 days' THEN ledger_amount ELSE 0 END) AS service_amount_10d, + -- 15天窗口 + COUNT(CASE WHEN service_date >= %s - INTERVAL '14 days' THEN 1 END) AS service_count_15d, + SUM(CASE WHEN service_date >= %s - INTERVAL '14 days' THEN income_seconds ELSE 0 END) / 3600.0 AS service_hours_15d, + SUM(CASE WHEN service_date >= %s - INTERVAL '14 days' THEN ledger_amount ELSE 0 END) AS service_amount_15d, + -- 30天窗口 + COUNT(CASE WHEN service_date >= %s - INTERVAL '29 days' THEN 1 END) AS service_count_30d, + SUM(CASE WHEN service_date >= %s - INTERVAL '29 days' THEN income_seconds ELSE 0 END) / 3600.0 AS service_hours_30d, + SUM(CASE WHEN service_date >= %s - INTERVAL '29 days' THEN ledger_amount ELSE 0 END) AS service_amount_30d, + -- 60天窗口 + COUNT(CASE WHEN service_date >= %s - INTERVAL '59 days' THEN 1 END) AS service_count_60d, + SUM(CASE WHEN service_date >= %s - INTERVAL '59 days' THEN income_seconds ELSE 0 END) / 3600.0 AS service_hours_60d, + SUM(CASE WHEN service_date >= %s - INTERVAL '59 days' THEN ledger_amount ELSE 0 END) AS service_amount_60d, + -- 90天窗口 + COUNT(CASE WHEN service_date >= %s - INTERVAL '89 days' THEN 1 END) AS service_count_90d, + SUM(CASE WHEN service_date >= %s - INTERVAL '89 days' THEN income_seconds ELSE 0 END) / 3600.0 AS service_hours_90d, + SUM(CASE WHEN service_date >= %s - INTERVAL '89 days' THEN ledger_amount ELSE 0 END) AS service_amount_90d + FROM service_base + GROUP BY assistant_id, member_id + HAVING MAX(service_date) >= %s - INTERVAL '90 days' + """ + # 构建参数（每个窗口需要3个日期参数） + params = [site_id] + for _ in range(6): # 6个窗口，每个3个参数 + params.extend([stat_date, stat_date, stat_date]) + params.append(stat_date) # HAVING条件 + + rows = self.db.query(sql, tuple(params)) + return [dict(row) for row in rows] if rows else [] + + def _extract_member_info(self, site_id: int) -> Dict[int, Dict[str, Any]]: + """ + 提取会员信息 + """ + sql = """ + SELECT + member_id, + nickname, + mobile + FROM billiards_dwd.dim_member + WHERE site_id = %s + """ + rows = self.db.query(sql, (site_id,)) + + result = {} + for row in (rows or []): + row_dict = dict(row) + result[row_dict['member_id']] = row_dict + return result + + def _extract_assistant_info(self, site_id: int) -> Dict[int, Dict[str, Any]]: + """ + 提取助教信息 + """ + sql = """ + SELECT + site_assistant_id AS assistant_id, + nickname + FROM billiards_dwd.dim_assistant + WHERE site_id = %s + AND valid_to IS NULL + """ + rows = self.db.query(sql, (site_id,)) + + result = {} + for row in (rows or []): + row_dict = dict(row) + result[row_dict['assistant_id']] = row_dict + return result + + # ========================================================================== + # 工具方法 + # ========================================================================== + + def _mask_mobile(self, mobile: Optional[str]) -> Optional[str]: + """ + 手机号脱敏 + """ + if not mobile or len(mobile) < 7: + return mobile + return mobile[:3] + "****" + mobile[-4:] + + def _calc_days_since(self, stat_date: date, last_date: Optional[date]) -> Optional[int]: + """ + 计算距离最近服务的天数 + """ + if not last_date: + return None + if isinstance(last_date, datetime): + last_date = last_date.date() + return (stat_date - last_date).days + + +# 便于外部导入 +__all__ = ['AssistantCustomerTask'] diff --git a/etl_billiards/tasks/dws/assistant_daily_task.py b/etl_billiards/tasks/dws/assistant_daily_task.py new file mode 100644 index 0000000..e842d90 --- /dev/null +++ b/etl_billiards/tasks/dws/assistant_daily_task.py @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- +""" +助教日度业绩明细任务 + +功能说明： + 以"助教+日期"为粒度，汇总每日业绩明细 + +数据来源： + - dwd_assistant_service_log: 助教服务流水 + - dwd_assistant_trash_event: 废除记录（排除） + - dim_assistant: 助教维度（SCD2，获取当日等级） + - cfg_skill_type: 技能→课程类型映射 + +目标表： + billiards_dws.dws_assistant_daily_detail + +更新策略： + - 更新频率：每小时增量更新 + - 幂等方式：delete-before-insert（按日期窗口） + +业务规则： + - 有效业绩：需排除dwd_assistant_trash_event中的废除记录 + - 助教等级：使用SCD2 as-of取值，获取统计日当日生效的等级 + - 课程类型：通过skill_id映射，分为基础课和附加课 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Set, Tuple + +from .base_dws_task import BaseDwsTask, CourseType, TaskContext + + +class AssistantDailyTask(BaseDwsTask): + """ + 助教日度业绩明细任务 + + 汇总每个助教每天的： + - 服务次数（总/基础课/附加课） + - 计费时长（秒/小时） + - 计费金额 + - 服务客户数（去重） + - 服务台桌数（去重） + - 被废除的记录统计 + """ + + def get_task_code(self) -> str: + return "DWS_ASSISTANT_DAILY" + + def get_target_table(self) -> str: + return "dws_assistant_daily_detail" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "assistant_id", "stat_date"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据：从DWD层读取助教服务记录 + """ + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + self.logger.info( + "%s: 提取数据，日期范围 %s ~ %s", + self.get_task_code(), start_date, end_date + ) + + # 1. 获取助教服务记录 + service_records = self._extract_service_records(site_id, start_date, end_date) + + # 2. 获取废除记录 + trash_records = self._extract_trash_records(site_id, start_date, end_date) + + # 3. 加载配置缓存 + self.load_config_cache() + + return { + 'service_records': service_records, + 'trash_records': trash_records, + 'start_date': start_date, + 'end_date': end_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据：按助教+日期聚合 + """ + service_records = extracted['service_records'] + trash_records = extracted['trash_records'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，服务记录 %d 条，废除记录 %d 条", + self.get_task_code(), len(service_records), len(trash_records) + ) + + # 构建废除记录索引（assistant_service_id -> trash_info） + trash_index = self._build_trash_index(trash_records) + + # 按助教+日期聚合 + aggregated = self._aggregate_by_assistant_date( + service_records, + trash_index, + site_id + ) + + return aggregated + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据：写入DWS表 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + # 删除已存在的数据（幂等） + deleted = self.delete_existing_data(context, date_col="stat_date") + + # 批量插入 + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_service_records( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取助教服务记录 + """ + sql = """ + SELECT + asl.assistant_service_id, + asl.order_settle_id, + asl.site_assistant_id AS assistant_id, + asl.nickname AS assistant_nickname, + asl.assistant_level, + asl.skill_id, + asl.skill_name, + asl.tenant_member_id AS member_id, + asl.site_table_id AS table_id, + asl.income_seconds, + asl.real_use_seconds, + asl.ledger_amount, + asl.ledger_unit_price, + DATE(asl.start_use_time) AS service_date + FROM billiards_dwd.dwd_assistant_service_log asl + WHERE asl.site_id = %s + AND DATE(asl.start_use_time) >= %s + AND DATE(asl.start_use_time) <= %s + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_trash_records( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取废除记录 + + 有效业绩的排除规则：仅对"助教废除表"的记录进行处理排除 + """ + sql = """ + SELECT + assistant_service_id, + trash_seconds, + trash_reason, + trash_time + FROM billiards_dwd.dwd_assistant_trash_event + WHERE site_id = %s + AND DATE(trash_time) >= %s + AND DATE(trash_time) <= %s + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + # ========================================================================== + # 数据转换方法 + # ========================================================================== + + def _build_trash_index( + self, + trash_records: List[Dict[str, Any]] + ) -> Dict[int, Dict[str, Any]]: + """ + 构建废除记录索引 + """ + index = {} + for record in trash_records: + service_id = record.get('assistant_service_id') + if service_id: + index[service_id] = record + return index + + def _aggregate_by_assistant_date( + self, + service_records: List[Dict[str, Any]], + trash_index: Dict[int, Dict[str, Any]], + site_id: int + ) -> List[Dict[str, Any]]: + """ + 按助教+日期聚合服务记录 + """ + # 聚合字典：(assistant_id, service_date) -> aggregated_data + agg_dict: Dict[Tuple[int, date], Dict[str, Any]] = {} + + for record in service_records: + assistant_id = record.get('assistant_id') + service_date = record.get('service_date') + + if not assistant_id or not service_date: + continue + + key = (assistant_id, service_date) + + # 初始化聚合数据 + if key not in agg_dict: + # 获取助教当日等级（SCD2 as-of） + level_info = self.get_assistant_level_asof(assistant_id, service_date) + + agg_dict[key] = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'assistant_id': assistant_id, + 'assistant_nickname': record.get('assistant_nickname'), + 'stat_date': service_date, + 'assistant_level_code': level_info.get('level_code') if level_info else record.get('assistant_level'), + 'assistant_level_name': level_info.get('level_name') if level_info else None, + 'total_service_count': 0, + 'base_service_count': 0, + 'bonus_service_count': 0, + 'total_seconds': 0, + 'base_seconds': 0, + 'bonus_seconds': 0, + 'total_hours': Decimal('0'), + 'base_hours': Decimal('0'), + 'bonus_hours': Decimal('0'), + 'total_ledger_amount': Decimal('0'), + 'base_ledger_amount': Decimal('0'), + 'bonus_ledger_amount': Decimal('0'), + 'unique_customers': set(), + 'unique_tables': set(), + 'trashed_seconds': 0, + 'trashed_count': 0, + } + + agg = agg_dict[key] + + # 获取服务信息 + service_id = record.get('assistant_service_id') + income_seconds = self.safe_int(record.get('income_seconds', 0)) + ledger_amount = self.safe_decimal(record.get('ledger_amount', 0)) + skill_id = record.get('skill_id') + member_id = record.get('member_id') + table_id = record.get('table_id') + + # 判断课程类型 + course_type = self.get_course_type(skill_id) if skill_id else CourseType.BASE + is_base = course_type == CourseType.BASE + + # 检查是否被废除 + is_trashed = service_id in trash_index + + if is_trashed: + # 废除记录单独统计 + trash_info = trash_index[service_id] + trash_seconds = self.safe_int(trash_info.get('trash_seconds', income_seconds)) + agg['trashed_seconds'] += trash_seconds + agg['trashed_count'] += 1 + else: + # 正常记录累加 + agg['total_service_count'] += 1 + agg['total_seconds'] += income_seconds + agg['total_ledger_amount'] += ledger_amount + + if is_base: + agg['base_service_count'] += 1 + agg['base_seconds'] += income_seconds + agg['base_ledger_amount'] += ledger_amount + else: + agg['bonus_service_count'] += 1 + agg['bonus_seconds'] += income_seconds + agg['bonus_ledger_amount'] += ledger_amount + + # 客户和台桌去重统计（不论是否废除） + if member_id and not self.is_guest(member_id): + agg['unique_customers'].add(member_id) + if table_id: + agg['unique_tables'].add(table_id) + + # 转换为列表并计算派生字段 + result = [] + for key, agg in agg_dict.items(): + # 计算小时数 + agg['total_hours'] = self.seconds_to_hours(agg['total_seconds']) + agg['base_hours'] = self.seconds_to_hours(agg['base_seconds']) + agg['bonus_hours'] = self.seconds_to_hours(agg['bonus_seconds']) + + # 转换set为count + agg['unique_customers'] = len(agg['unique_customers']) + agg['unique_tables'] = len(agg['unique_tables']) + + result.append(agg) + + return result + + +# 便于外部导入 +__all__ = ['AssistantDailyTask'] diff --git a/etl_billiards/tasks/dws/assistant_finance_task.py b/etl_billiards/tasks/dws/assistant_finance_task.py new file mode 100644 index 0000000..c805d86 --- /dev/null +++ b/etl_billiards/tasks/dws/assistant_finance_task.py @@ -0,0 +1,199 @@ +# -*- coding: utf-8 -*- +""" +助教收支分析任务 + +功能说明： + 以"日期+助教"为粒度，分析助教产出的收入和成本 + +数据来源： + - dwd_assistant_service_log: 助教服务流水（收入） + - dws_assistant_salary_calc: 工资计算（成本） + +目标表： + billiards_dws.dws_assistant_finance_analysis + +更新策略： + - 更新频率：每日更新 + - 幂等方式：delete-before-insert（按日期） + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_dws_task import BaseDwsTask, CourseType, TaskContext + + +class AssistantFinanceTask(BaseDwsTask): + """ + 助教收支分析任务 + """ + + def get_task_code(self) -> str: + return "DWS_ASSISTANT_FINANCE" + + def get_target_table(self) -> str: + return "dws_assistant_finance_analysis" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "stat_date", "assistant_id"] + + def extract(self, context: TaskContext) -> Dict[str, Any]: + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + # 获取助教日度收入 + daily_revenue = self._extract_daily_revenue(site_id, start_date, end_date) + + # 获取月度工资（用于计算日均成本） + monthly_salary = self._extract_monthly_salary(site_id, start_date, end_date) + + # 加载配置 + self.load_config_cache() + + return { + 'daily_revenue': daily_revenue, + 'monthly_salary': monthly_salary, + 'start_date': start_date, + 'end_date': end_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + daily_revenue = extracted['daily_revenue'] + monthly_salary = extracted['monthly_salary'] + site_id = extracted['site_id'] + + # 构建月度工资索引 + salary_index = {} + for sal in monthly_salary: + asst_id = sal.get('assistant_id') + month = sal.get('salary_month') + if asst_id and month: + salary_index[(asst_id, month)] = sal + + results = [] + for rev in daily_revenue: + assistant_id = rev.get('assistant_id') + stat_date = rev.get('stat_date') + + # 获取对应月份的工资 + month_start = stat_date.replace(day=1) if isinstance(stat_date, date) else None + salary = salary_index.get((assistant_id, month_start), {}) + + # 计算日均成本 + gross_salary = self.safe_decimal(salary.get('gross_salary', 0)) + work_days = self.safe_int(salary.get('work_days', 1)) or 1 + cost_daily = gross_salary / Decimal(str(work_days)) + + revenue_total = self.safe_decimal(rev.get('revenue_total', 0)) + gross_profit = revenue_total - cost_daily + gross_margin = gross_profit / revenue_total if revenue_total > 0 else Decimal('0') + + record = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'stat_date': stat_date, + 'assistant_id': assistant_id, + 'assistant_nickname': rev.get('assistant_nickname'), + 'revenue_total': revenue_total, + 'revenue_base': self.safe_decimal(rev.get('revenue_base', 0)), + 'revenue_bonus': self.safe_decimal(rev.get('revenue_bonus', 0)), + 'cost_daily': cost_daily, + 'gross_profit': gross_profit, + 'gross_margin': gross_margin, + 'service_count': self.safe_int(rev.get('service_count', 0)), + 'service_hours': self.safe_decimal(rev.get('service_hours', 0)), + 'unique_customers': self.safe_int(rev.get('unique_customers', 0)), + } + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + if not transformed: + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + deleted = self.delete_existing_data(context, date_col="stat_date") + inserted = self.bulk_insert(transformed) + + return { + "counts": {"fetched": len(transformed), "inserted": inserted, "updated": 0, "skipped": 0, "errors": 0}, + "extra": {"deleted": deleted} + } + + def _extract_daily_revenue(self, site_id: int, start_date: date, end_date: date) -> List[Dict[str, Any]]: + # 基础课skill_id + BASE_SKILL_ID = 2791903611396869 + + sql = """ + SELECT + DATE(start_use_time) AS stat_date, + site_assistant_id AS assistant_id, + MAX(nickname) AS assistant_nickname, + COUNT(*) AS service_count, + SUM(income_seconds) / 3600.0 AS service_hours, + SUM(ledger_amount) AS revenue_total, + SUM(CASE WHEN skill_id = %s THEN ledger_amount ELSE 0 END) AS revenue_base, + SUM(CASE WHEN skill_id != %s THEN ledger_amount ELSE 0 END) AS revenue_bonus, + COUNT(DISTINCT tenant_member_id) AS unique_customers + FROM billiards_dwd.dwd_assistant_service_log + WHERE site_id = %s + AND DATE(start_use_time) >= %s + AND DATE(start_use_time) <= %s + GROUP BY DATE(start_use_time), site_assistant_id + """ + rows = self.db.query(sql, (BASE_SKILL_ID, BASE_SKILL_ID, site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_monthly_salary(self, site_id: int, start_date: date, end_date: date) -> List[Dict[str, Any]]: + # 获取涉及的月份 + month_start = start_date.replace(day=1) + month_end = end_date.replace(day=1) + + sql = """ + SELECT + assistant_id, + salary_month, + gross_salary, + effective_hours + FROM billiards_dws.dws_assistant_salary_calc + WHERE site_id = %s + AND salary_month >= %s + AND salary_month <= %s + """ + rows = self.db.query(sql, (site_id, month_start, month_end)) + + # 获取每月工作天数 + work_days_sql = """ + SELECT + assistant_id, + DATE_TRUNC('month', stat_date)::DATE AS month, + COUNT(DISTINCT stat_date) AS work_days + FROM billiards_dws.dws_assistant_daily_detail + WHERE site_id = %s + AND stat_date >= %s + AND stat_date <= %s + GROUP BY assistant_id, DATE_TRUNC('month', stat_date) + """ + work_days_rows = self.db.query(work_days_sql, (site_id, start_date, end_date)) + work_days_index = {(r['assistant_id'], r['month']): r['work_days'] for r in (work_days_rows or [])} + + results = [] + for row in (rows or []): + row_dict = dict(row) + asst_id = row_dict.get('assistant_id') + month = row_dict.get('salary_month') + row_dict['work_days'] = work_days_index.get((asst_id, month), 20) + results.append(row_dict) + + return results + + +__all__ = ['AssistantFinanceTask'] diff --git a/etl_billiards/tasks/dws/assistant_monthly_task.py b/etl_billiards/tasks/dws/assistant_monthly_task.py new file mode 100644 index 0000000..dd4c5d6 --- /dev/null +++ b/etl_billiards/tasks/dws/assistant_monthly_task.py @@ -0,0 +1,444 @@ +# -*- coding: utf-8 -*- +""" +助教月度业绩汇总任务 + +功能说明： + 以"助教+月份"为粒度，汇总月度业绩及档位计算 + +数据来源： + - dws_assistant_daily_detail: 日度明细（聚合） + - dim_assistant: 助教维度（入职日期、等级） + - cfg_performance_tier: 绩效档位配置 + +目标表： + billiards_dws.dws_assistant_monthly_summary + +更新策略： + - 更新频率：每日更新当月数据 + - 幂等方式：delete-before-insert（按月份） + +业务规则： + - 新入职判断：入职日期在月1日0点之后则为新入职 + - 有效业绩：total_hours - trashed_hours + - 档位匹配：根据有效业绩小时数匹配cfg_performance_tier + - 排名计算：按有效业绩小时数降序，考虑并列（如2个第一则都是1，下一个是3） + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Set, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class AssistantMonthlyTask(BaseDwsTask): + """ + 助教月度业绩汇总任务 + + 汇总每个助教每月的： + - 工作天数、服务次数、时长 + - 有效业绩（扣除废除记录后） + - 档位匹配 + - 月度排名（用于Top3奖金） + """ + + def get_task_code(self) -> str: + return "DWS_ASSISTANT_MONTHLY" + + def get_target_table(self) -> str: + return "dws_assistant_monthly_summary" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "assistant_id", "stat_month"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据：从日度明细表聚合 + """ + # 确定月份范围 + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + # 获取涉及的月份列表 + months = self._get_months_in_range(start_date, end_date) + + self.logger.info( + "%s: 提取数据，月份范围 %s", + self.get_task_code(), [str(m) for m in months] + ) + + # 1. 获取日度明细聚合数据 + daily_aggregates = self._extract_daily_aggregates(site_id, months) + + # 2. 获取助教基本信息 + assistant_info = self._extract_assistant_info(site_id) + + # 3. 加载配置缓存 + self.load_config_cache() + + return { + 'daily_aggregates': daily_aggregates, + 'assistant_info': assistant_info, + 'months': months, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据：计算月度汇总、档位匹配、排名 + """ + daily_aggregates = extracted['daily_aggregates'] + assistant_info = extracted['assistant_info'] + months = extracted['months'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，%d 个月份，%d 条聚合记录", + self.get_task_code(), len(months), len(daily_aggregates) + ) + + # 按月份处理 + all_results = [] + for month in months: + month_results = self._process_month( + daily_aggregates, + assistant_info, + month, + site_id + ) + all_results.extend(month_results) + + return all_results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据：写入DWS表 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + # 删除已存在的数据（按月份） + deleted = self._delete_by_months(context, transformed) + + # 批量插入 + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _get_months_in_range(self, start_date: date, end_date: date) -> List[date]: + """ + 获取日期范围内的所有月份（月第一天） + """ + months = [] + current = start_date.replace(day=1) + end_month = end_date.replace(day=1) + + while current <= end_month: + months.append(current) + # 下个月 + if current.month == 12: + current = current.replace(year=current.year + 1, month=1) + else: + current = current.replace(month=current.month + 1) + + return months + + def _extract_daily_aggregates( + self, + site_id: int, + months: List[date] + ) -> List[Dict[str, Any]]: + """ + 从日度明细表提取并按月聚合 + """ + if not months: + return [] + + # 构建月份条件 + month_conditions = [] + for month in months: + next_month = (month.replace(day=28) + timedelta(days=4)).replace(day=1) + month_conditions.append(f"(stat_date >= '{month}' AND stat_date < '{next_month}')") + + month_where = " OR ".join(month_conditions) + + sql = f""" + SELECT + assistant_id, + assistant_nickname, + assistant_level_code, + assistant_level_name, + DATE_TRUNC('month', stat_date)::DATE AS stat_month, + COUNT(DISTINCT stat_date) AS work_days, + SUM(total_service_count) AS total_service_count, + SUM(base_service_count) AS base_service_count, + SUM(bonus_service_count) AS bonus_service_count, + SUM(total_hours) AS total_hours, + SUM(base_hours) AS base_hours, + SUM(bonus_hours) AS bonus_hours, + SUM(total_ledger_amount) AS total_ledger_amount, + SUM(base_ledger_amount) AS base_ledger_amount, + SUM(bonus_ledger_amount) AS bonus_ledger_amount, + SUM(unique_customers) AS total_unique_customers, + SUM(unique_tables) AS total_unique_tables, + SUM(trashed_seconds) AS trashed_seconds, + SUM(trashed_count) AS trashed_count + FROM billiards_dws.dws_assistant_daily_detail + WHERE site_id = %s AND ({month_where}) + GROUP BY assistant_id, assistant_nickname, assistant_level_code, assistant_level_name, + DATE_TRUNC('month', stat_date) + """ + + rows = self.db.query(sql, (site_id,)) + return [dict(row) for row in rows] if rows else [] + + def _extract_assistant_info(self, site_id: int) -> Dict[int, Dict[str, Any]]: + """ + 提取助教基本信息 + """ + sql = """ + SELECT + site_assistant_id AS assistant_id, + nickname, + assistant_level, + entry_date AS hire_date + FROM billiards_dwd.dim_assistant + WHERE site_id = %s + AND valid_to IS NULL -- 当前有效记录 + """ + rows = self.db.query(sql, (site_id,)) + + result = {} + for row in (rows or []): + row_dict = dict(row) + result[row_dict['assistant_id']] = row_dict + return result + + # ========================================================================== + # 数据转换方法 + # ========================================================================== + + def _process_month( + self, + daily_aggregates: List[Dict[str, Any]], + assistant_info: Dict[int, Dict[str, Any]], + month: date, + site_id: int + ) -> List[Dict[str, Any]]: + """ + 处理单个月份的数据 + """ + # 筛选该月份的数据 + month_data = [ + agg for agg in daily_aggregates + if agg.get('stat_month') == month + ] + + if not month_data: + return [] + + # 构建月度汇总记录 + month_records = [] + + for agg in month_data: + assistant_id = agg.get('assistant_id') + asst_info = assistant_info.get(assistant_id, {}) + + # 计算有效业绩 + total_hours = self.safe_decimal(agg.get('total_hours', 0)) + trashed_hours = self.seconds_to_hours(self.safe_int(agg.get('trashed_seconds', 0))) + effective_hours = total_hours - trashed_hours + + # 判断是否新入职 + hire_date = asst_info.get('hire_date') + is_new_hire = False + if hire_date: + if isinstance(hire_date, datetime): + hire_date = hire_date.date() + is_new_hire = self.is_new_hire_in_month(hire_date, month) + + # 匹配档位 + tier_hours = effective_hours + max_tier_level = None + if is_new_hire: + tier_hours = self._calc_new_hire_tier_hours(effective_hours, self.safe_int(agg.get('work_days', 0))) + if hire_date and hire_date.day > 25: + max_tier_level = 3 + tier = self.get_performance_tier( + tier_hours, + is_new_hire, + effective_date=month, + max_tier_level=max_tier_level + ) + + # 获取月末的等级信息（用于记录） + month_end = self._get_month_end(month) + level_info = self.get_assistant_level_asof(assistant_id, month_end) + + record = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'assistant_id': assistant_id, + 'assistant_nickname': agg.get('assistant_nickname'), + 'stat_month': month, + 'assistant_level_code': level_info.get('level_code') if level_info else agg.get('assistant_level_code'), + 'assistant_level_name': level_info.get('level_name') if level_info else agg.get('assistant_level_name'), + 'hire_date': hire_date, + 'is_new_hire': is_new_hire, + 'work_days': self.safe_int(agg.get('work_days', 0)), + 'total_service_count': self.safe_int(agg.get('total_service_count', 0)), + 'base_service_count': self.safe_int(agg.get('base_service_count', 0)), + 'bonus_service_count': self.safe_int(agg.get('bonus_service_count', 0)), + 'total_hours': total_hours, + 'base_hours': self.safe_decimal(agg.get('base_hours', 0)), + 'bonus_hours': self.safe_decimal(agg.get('bonus_hours', 0)), + 'effective_hours': effective_hours, + 'trashed_hours': trashed_hours, + 'total_ledger_amount': self.safe_decimal(agg.get('total_ledger_amount', 0)), + 'base_ledger_amount': self.safe_decimal(agg.get('base_ledger_amount', 0)), + 'bonus_ledger_amount': self.safe_decimal(agg.get('bonus_ledger_amount', 0)), + 'unique_customers': self.safe_int(agg.get('total_unique_customers', 0)), + 'unique_tables': self.safe_int(agg.get('total_unique_tables', 0)), + 'avg_service_seconds': self._calc_avg_service_seconds(agg), + 'tier_id': tier.get('tier_id') if tier else None, + 'tier_code': tier.get('tier_code') if tier else None, + 'tier_name': tier.get('tier_name') if tier else None, + 'rank_by_hours': None, # 后面计算 + 'rank_with_ties': None, # 后面计算 + } + month_records.append(record) + + # 计算排名 + self._calculate_ranks(month_records) + + return month_records + + def _get_month_end(self, month: date) -> date: + """ + 获取月末日期 + """ + if month.month == 12: + next_month = month.replace(year=month.year + 1, month=1, day=1) + else: + next_month = month.replace(month=month.month + 1, day=1) + return next_month - timedelta(days=1) + + def _calc_avg_service_seconds(self, agg: Dict[str, Any]) -> Decimal: + """ + 计算平均单次服务时长 + """ + total_count = self.safe_int(agg.get('total_service_count', 0)) + if total_count == 0: + return Decimal('0') + + total_hours = self.safe_decimal(agg.get('total_hours', 0)) + total_seconds = total_hours * Decimal('3600') + return total_seconds / Decimal(str(total_count)) + + def _calc_new_hire_tier_hours(self, effective_hours: Decimal, work_days: int) -> Decimal: + """ + 新入职定档：日均 * 30（仅用于定档，不影响奖金与排名） + """ + if work_days <= 0: + return Decimal('0') + return (effective_hours / Decimal(str(work_days))) * Decimal('30') + + def _calculate_ranks(self, records: List[Dict[str, Any]]) -> None: + """ + 计算排名（考虑并列） + + Top3排名口径：按有效业绩总小时数排名， + 如遇并列则都算，比如2个第一，则记为2个第一，一个第三 + """ + if not records: + return + + # 按有效业绩降序排序 + sorted_records = sorted( + records, + key=lambda x: x.get('effective_hours', Decimal('0')), + reverse=True + ) + + # 计算考虑并列的排名 + values = [ + (r.get('assistant_id'), r.get('effective_hours', Decimal('0'))) + for r in sorted_records + ] + ranked = self.calculate_rank_with_ties(values) + + # 创建排名映射 + rank_map = { + assistant_id: (rank, dense_rank) + for assistant_id, rank, dense_rank in ranked + } + + # 更新记录 + for record in records: + assistant_id = record.get('assistant_id') + if assistant_id in rank_map: + rank, _ = rank_map[assistant_id] + record['rank_by_hours'] = rank + record['rank_with_ties'] = rank # 使用考虑并列的排名 + + def _delete_by_months( + self, + context: TaskContext, + records: List[Dict[str, Any]] + ) -> int: + """ + 按月份删除已存在的数据 + """ + # 获取涉及的月份 + months = set(r.get('stat_month') for r in records if r.get('stat_month')) + + if not months: + return 0 + + target_table = self.get_target_table() + full_table = f"{self.DWS_SCHEMA}.{target_table}" + + total_deleted = 0 + with self.db.conn.cursor() as cur: + for month in months: + sql = f""" + DELETE FROM {full_table} + WHERE site_id = %s AND stat_month = %s + """ + cur.execute(sql, (context.store_id, month)) + total_deleted += cur.rowcount + + return total_deleted + + +# 便于外部导入 +__all__ = ['AssistantMonthlyTask'] diff --git a/etl_billiards/tasks/dws/assistant_salary_task.py b/etl_billiards/tasks/dws/assistant_salary_task.py new file mode 100644 index 0000000..2893b8a --- /dev/null +++ b/etl_billiards/tasks/dws/assistant_salary_task.py @@ -0,0 +1,403 @@ +# -*- coding: utf-8 -*- +""" +助教工资计算任务 + +功能说明： + 以"助教+月份"为粒度，计算月度工资明细 + +数据来源： + - dws_assistant_monthly_summary: 月度业绩汇总 + - dws_assistant_recharge_commission: 充值提成（Excel导入） + - cfg_performance_tier: 绩效档位配置 + - cfg_assistant_level_price: 等级定价配置 + - cfg_bonus_rules: 奖金规则配置 + +目标表： + billiards_dws.dws_assistant_salary_calc + +更新策略： + - 更新频率：月初计算上月工资 + - 幂等方式：delete-before-insert（按月份） + +业务规则（来自DWS数据库处理需求.md）： + - 基础课收入 = 基础课小时数 × (客户支付价格 - 专业课抽成) + 例：中级助教基础课170小时，3档 = 170 × (108 - 13) = 16,150元 + - 附加课收入 = 附加课小时数 × 附加课价格 × (1 - 打赏课抽成比例) + 例：附加课15小时，3档 = 15 × 190 × (1 - 0.35) = 1,852.5元 + - 冲刺奖金：H>=190:300, H>=220:800（不累计，取最高档） + - Top3奖金：1st:1000, 2nd:600, 3rd:400（并列都算） + - 充值提成：来自dws_assistant_recharge_commission + - SCD2口径：等级定价使用月份对应的历史值 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class AssistantSalaryTask(BaseDwsTask): + """ + 助教工资计算任务 + + 计算每个助教每月的工资明细： + - 课时收入（基础课+附加课） + - 扣款（档位扣款+其他） + - 奖金（档位奖金+冲刺+Top3+充值提成+其他） + - 应发工资 + """ + + def get_task_code(self) -> str: + return "DWS_ASSISTANT_SALARY" + + def get_target_table(self) -> str: + return "dws_assistant_salary_calc" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "assistant_id", "salary_month"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据 + """ + # 确定工资月份（通常是上月） + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + salary_month = self._get_salary_month(end_date) + site_id = context.store_id + + self.logger.info( + "%s: 提取数据，工资月份 %s", + self.get_task_code(), salary_month + ) + + # 1. 获取月度业绩汇总 + monthly_summary = self._extract_monthly_summary(site_id, salary_month) + + # 2. 获取充值提成 + recharge_commission = self._extract_recharge_commission(site_id, salary_month) + + # 3. 加载配置缓存 + self.load_config_cache() + + return { + 'monthly_summary': monthly_summary, + 'recharge_commission': recharge_commission, + 'salary_month': salary_month, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据：计算工资 + """ + monthly_summary = extracted['monthly_summary'] + recharge_commission = extracted['recharge_commission'] + salary_month = extracted['salary_month'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，%d 条月度汇总记录", + self.get_task_code(), len(monthly_summary) + ) + + # 构建充值提成索引 + commission_index = {} + for comm in recharge_commission: + asst_id = comm.get('assistant_id') + if asst_id: + commission_index[asst_id] = commission_index.get(asst_id, Decimal('0')) + \ + self.safe_decimal(comm.get('commission_amount', 0)) + + # 计算工资 + results = [] + for summary in monthly_summary: + record = self._calculate_salary(summary, commission_index, salary_month, site_id) + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + # 删除已存在的数据 + deleted = self._delete_by_month(context, transformed) + + # 批量插入 + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _get_salary_month(self, end_date: date) -> date: + """ + 获取工资月份（默认为上月） + """ + # 如果是月初，计算上月工资 + if end_date.day <= 5: + if end_date.month == 1: + return date(end_date.year - 1, 12, 1) + else: + return date(end_date.year, end_date.month - 1, 1) + else: + # 否则计算当月（可能是调整） + return end_date.replace(day=1) + + def _extract_monthly_summary( + self, + site_id: int, + salary_month: date + ) -> List[Dict[str, Any]]: + """ + 提取月度业绩汇总 + """ + sql = """ + SELECT + assistant_id, + assistant_nickname, + stat_month, + assistant_level_code, + assistant_level_name, + hire_date, + is_new_hire, + effective_hours, + base_hours, + bonus_hours, + tier_id, + tier_code, + tier_name, + rank_with_ties + FROM billiards_dws.dws_assistant_monthly_summary + WHERE site_id = %s AND stat_month = %s + """ + rows = self.db.query(sql, (site_id, salary_month)) + return [dict(row) for row in rows] if rows else [] + + def _extract_recharge_commission( + self, + site_id: int, + salary_month: date + ) -> List[Dict[str, Any]]: + """ + 提取充值提成 + """ + sql = """ + SELECT + assistant_id, + commission_amount + FROM billiards_dws.dws_assistant_recharge_commission + WHERE site_id = %s AND commission_month = %s + """ + rows = self.db.query(sql, (site_id, salary_month)) + return [dict(row) for row in rows] if rows else [] + + # ========================================================================== + # 工资计算方法 + # ========================================================================== + + def _calculate_salary( + self, + summary: Dict[str, Any], + commission_index: Dict[int, Decimal], + salary_month: date, + site_id: int + ) -> Dict[str, Any]: + """ + 计算单个助教的月度工资 + """ + assistant_id = summary.get('assistant_id') + level_code = summary.get('assistant_level_code') + effective_hours = self.safe_decimal(summary.get('effective_hours', 0)) + base_hours = self.safe_decimal(summary.get('base_hours', 0)) + bonus_hours = self.safe_decimal(summary.get('bonus_hours', 0)) + is_new_hire = summary.get('is_new_hire', False) + rank = summary.get('rank_with_ties') + + # 获取等级定价（SCD2口径，按月份取值） + # base_course_price: 客户支付价格（初级98/中级108/高级118/星级138） + # bonus_course_price: 附加课客户支付价格（固定190元） + level_price = self.get_level_price(level_code, salary_month) + base_course_price = self.safe_decimal( + level_price.get('base_course_price', 98) if level_price else 98 + ) + bonus_course_price = self.safe_decimal( + level_price.get('bonus_course_price', 190) if level_price else 190 + ) + + # 获取档位配置 + # base_deduction: 专业课抽成（元/小时），球房从每小时扣除 + # bonus_deduction_ratio: 打赏课抽成比例，球房从附加课收入扣除的比例 + tier = self.get_performance_tier_by_id(summary.get('tier_id'), salary_month) + if not tier: + tier = self.get_performance_tier( + effective_hours, + is_new_hire, + effective_date=salary_month + ) + base_deduction = self.safe_decimal(tier.get('base_deduction', 18)) if tier else Decimal('18') + bonus_deduction_ratio = self.safe_decimal(tier.get('bonus_deduction_ratio', 0.40)) if tier else Decimal('0.40') + vacation_days = tier.get('vacation_days', 0) if tier else 0 + vacation_unlimited = tier.get('vacation_unlimited', False) if tier else False + + # ============================================================ + # 工资计算公式（来自DWS数据库处理需求.md） + # ============================================================ + # 基础课收入 = 基础课小时数 × (客户支付价格 - 专业课抽成) + # 例：中级助教170小时，3档 = 170 × (108 - 13) = 16,150元 + base_income = base_hours * (base_course_price - base_deduction) + + # 附加课收入 = 附加课小时数 × 附加课价格 × (1 - 打赏课抽成比例) + # 例：15小时，3档 = 15 × 190 × (1 - 0.35) = 1,852.5元 + bonus_income = bonus_hours * bonus_course_price * (Decimal('1') - bonus_deduction_ratio) + + # 课时收入合计 + total_course_income = base_income + bonus_income + + # 计算冲刺奖金（H>=190:300, H>=220:800，不累计取最高） + sprint_bonus = self.calculate_sprint_bonus(effective_hours, salary_month) + + # 计算Top3排名奖金（1st:1000, 2nd:600, 3rd:400，并列都算） + top_rank_bonus = Decimal('0') + if rank and rank <= 3: + top_rank_bonus = self.calculate_top_rank_bonus(rank, salary_month) + + # 获取充值提成 + recharge_commission = commission_index.get(assistant_id, Decimal('0')) + + # 汇总奖金 + other_bonus = Decimal('0') # 预留其他奖金 + total_bonus = sprint_bonus + top_rank_bonus + recharge_commission + other_bonus + + # 计算应发工资 = 课时收入 + 奖金 + gross_salary = total_course_income + total_bonus + + # 构建记录 + return { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'assistant_id': assistant_id, + 'assistant_nickname': summary.get('assistant_nickname'), + 'salary_month': salary_month, + 'assistant_level_code': level_code, + 'assistant_level_name': summary.get('assistant_level_name'), + 'hire_date': summary.get('hire_date'), + 'is_new_hire': is_new_hire, + 'effective_hours': effective_hours, + 'base_hours': base_hours, + 'bonus_hours': bonus_hours, + 'tier_id': summary.get('tier_id'), + 'tier_code': tier.get('tier_code') if tier else None, + 'tier_name': tier.get('tier_name') if tier else None, + 'rank_with_ties': rank, + # 定价信息 + 'base_course_price': base_course_price, + 'bonus_course_price': bonus_course_price, + 'base_deduction': base_deduction, + 'bonus_deduction_ratio': bonus_deduction_ratio, + # 收入明细 + 'base_income': base_income, + 'bonus_income': bonus_income, + 'total_course_income': total_course_income, + # 奖金明细 + 'sprint_bonus': sprint_bonus, + 'top_rank_bonus': top_rank_bonus, + 'recharge_commission': recharge_commission, + 'other_bonus': other_bonus, + 'total_bonus': total_bonus, + # 应发工资 + 'gross_salary': gross_salary, + # 假期 + 'vacation_days': vacation_days, + 'vacation_unlimited': vacation_unlimited, + 'calc_notes': self._build_calc_notes(summary, tier, sprint_bonus, top_rank_bonus), + } + + def _build_calc_notes( + self, + summary: Dict[str, Any], + tier: Optional[Dict[str, Any]], + sprint_bonus: Decimal, + top_rank_bonus: Decimal + ) -> Optional[str]: + """ + 构建计算备注 + """ + notes = [] + + if summary.get('is_new_hire'): + notes.append("新入职首月") + + if tier: + notes.append(f"档位: {tier.get('tier_name', 'N/A')}") + + if sprint_bonus > 0: + notes.append(f"冲刺奖金: {sprint_bonus}") + + if top_rank_bonus > 0: + rank = summary.get('rank_with_ties') + notes.append(f"Top{rank}奖金: {top_rank_bonus}") + + return "; ".join(notes) if notes else None + + def _delete_by_month( + self, + context: TaskContext, + records: List[Dict[str, Any]] + ) -> int: + """ + 按月份删除已存在的数据 + """ + months = set(r.get('salary_month') for r in records if r.get('salary_month')) + + if not months: + return 0 + + target_table = self.get_target_table() + full_table = f"{self.DWS_SCHEMA}.{target_table}" + + total_deleted = 0 + with self.db.conn.cursor() as cur: + for month in months: + sql = f""" + DELETE FROM {full_table} + WHERE site_id = %s AND salary_month = %s + """ + cur.execute(sql, (context.store_id, month)) + total_deleted += cur.rowcount + + return total_deleted + + +# 便于外部导入 +__all__ = ['AssistantSalaryTask'] diff --git a/etl_billiards/tasks/dws/base_dws_task.py b/etl_billiards/tasks/dws/base_dws_task.py new file mode 100644 index 0000000..27488e3 --- /dev/null +++ b/etl_billiards/tasks/dws/base_dws_task.py @@ -0,0 +1,1223 @@ +# -*- coding: utf-8 -*- +""" +DWS层任务基类 + +功能说明： + - 提供从DWD层读取数据的标准方法 + - 提供时间分层查询功能（近2天/近1月/近3月/近6月/全量） + - 提供配置表读取方法 + - 提供幂等更新机制（delete-before-insert） + - 提供SCD2维度as-of取值方法 + - 提供滚动窗口统计方法 + +时间口径说明： + - 周起始日：周一 + - 月/季度起始：第一天0点 + - 环比规则：对比上一个等长区间 + - 前3个月：含/不含本月（用于财务筛选） + - 最近半年：不含本月 + +更新频率： + - 日度表：每日更新 + - 实时表：每小时更新 + - 月度表：每日更新当月数据 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +import calendar +from abc import abstractmethod +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from decimal import Decimal +from enum import Enum +from typing import Any, Dict, Iterator, List, Optional, Tuple, TypeVar + +from ..base_task import BaseTask, TaskContext + +# ============================================================================= +# 类型定义 +# ============================================================================= + +T = TypeVar('T') + + +class TimeLayer(Enum): + """时间分层枚举（用于数据筛选）""" + LAST_2_DAYS = "LAST_2_DAYS" # 近2天 + LAST_1_MONTH = "LAST_1_MONTH" # 近1月 + LAST_3_MONTHS = "LAST_3_MONTHS" # 近3月 + LAST_6_MONTHS = "LAST_6_MONTHS" # 近6月（不含本月） + ALL = "ALL" # 全量 + + +class TimeWindow(Enum): + """时间窗口类型枚举（用于财务报表）""" + THIS_WEEK = "THIS_WEEK" # 本周（周一起始） + LAST_WEEK = "LAST_WEEK" # 上周 + THIS_MONTH = "THIS_MONTH" # 本月 + LAST_MONTH = "LAST_MONTH" # 上月 + LAST_3_MONTHS_EXCL_CURRENT = "LAST_3_MONTHS_EXCL_CURRENT" # 前3个月不含本月 + LAST_3_MONTHS_INCL_CURRENT = "LAST_3_MONTHS_INCL_CURRENT" # 前3个月含本月 + THIS_QUARTER = "THIS_QUARTER" # 本季度 + LAST_QUARTER = "LAST_QUARTER" # 上季度 + LAST_6_MONTHS = "LAST_6_MONTHS" # 最近半年（不含本月） + + +class CourseType(Enum): + """课程类型枚举""" + BASE = "BASE" # 基础课/陪打 + BONUS = "BONUS" # 附加课/超休 + + +class DiscountType(Enum): + """优惠类型枚举""" + GROUPBUY = "GROUPBUY" # 团购优惠 + VIP = "VIP" # 会员折扣 + GIFT_CARD = "GIFT_CARD" # 赠送卡抵扣 + MANUAL = "MANUAL" # 手动调整 + ROUNDING = "ROUNDING" # 抹零 + BIG_CUSTOMER = "BIG_CUSTOMER" # 大客户优惠 + OTHER = "OTHER" # 其他优惠 + + +@dataclass +class TimeRange: + """时间范围数据类""" + start: date + end: date + + +@dataclass +class ConfigCache: + """配置缓存数据类""" + performance_tiers: List[Dict[str, Any]] # 绩效档位配置 + level_prices: List[Dict[str, Any]] # 等级定价配置 + bonus_rules: List[Dict[str, Any]] # 奖金规则配置 + area_categories: Dict[str, Dict[str, Any]] # 区域分类映射 + skill_types: Dict[int, Dict[str, Any]] # 技能类型映射 + loaded_at: datetime # 加载时间 + + +# ============================================================================= +# DWS任务基类 +# ============================================================================= + +class BaseDwsTask(BaseTask): + """ + DWS层任务基类 + + 提供DWS层通用功能： + 1. DWD数据读取方法 + 2. 时间分层与窗口计算 + 3. 配置表缓存与读取 + 4. SCD2维度as-of取值 + 5. 幂等更新机制 + 6. 滚动窗口统计 + """ + + # 类级别的配置缓存 + _config_cache: Optional[ConfigCache] = None + _config_cache_ttl: int = 300 # 缓存有效期（秒） + + # DWS Schema名称 + DWS_SCHEMA = "billiards_dws" + DWD_SCHEMA = "billiards_dwd" + + # 滚动窗口天数列表 + ROLLING_WINDOWS = [7, 10, 15, 30, 60, 90] + + # ========================================================================== + # 抽象方法（子类必须实现） + # ========================================================================== + + @abstractmethod + def get_target_table(self) -> str: + """ + 获取目标表名（不含schema） + + Returns: + 目标表名，如 'dws_assistant_daily_detail' + """ + raise NotImplementedError("子类需实现 get_target_table 方法") + + @abstractmethod + def get_primary_keys(self) -> List[str]: + """ + 获取主键字段列表（用于幂等更新） + + Returns: + 主键字段列表，如 ['site_id', 'assistant_id', 'stat_date'] + """ + raise NotImplementedError("子类需实现 get_primary_keys 方法") + + # ========================================================================== + # 时间计算方法 + # ========================================================================== + + def get_time_layer_range( + self, + layer: TimeLayer, + base_date: Optional[date] = None + ) -> TimeRange: + """ + 获取时间分层的日期范围 + + Args: + layer: 时间分层枚举 + base_date: 基准日期，默认为今天 + + Returns: + TimeRange对象，包含起止日期 + """ + if base_date is None: + base_date = date.today() + + if layer == TimeLayer.LAST_2_DAYS: + return TimeRange( + start=base_date - timedelta(days=1), + end=base_date + ) + elif layer == TimeLayer.LAST_1_MONTH: + return TimeRange( + start=base_date - timedelta(days=30), + end=base_date + ) + elif layer == TimeLayer.LAST_3_MONTHS: + return TimeRange( + start=base_date - timedelta(days=90), + end=base_date + ) + elif layer == TimeLayer.LAST_6_MONTHS: + # 不含本月，从上月末往前6个月 + month_start = self.get_month_first_day(base_date) + end = month_start - timedelta(days=1) + start = self.get_month_first_day(self._shift_months(month_start, -6)) + return TimeRange(start=start, end=end) + else: # ALL + return TimeRange( + start=date(2000, 1, 1), + end=base_date + ) + + def get_time_window_range( + self, + window: TimeWindow, + base_date: Optional[date] = None + ) -> TimeRange: + """ + 获取时间窗口的日期范围（用于财务报表） + + 时间口径说明： + - 周起始日为周一 + - 月/季度起始为第一天0点 + + Args: + window: 时间窗口枚举 + base_date: 基准日期，默认为今天 + + Returns: + TimeRange对象 + """ + if base_date is None: + base_date = date.today() + + if window == TimeWindow.THIS_WEEK: + # 本周（周一起始） + days_since_monday = base_date.weekday() + start = base_date - timedelta(days=days_since_monday) + return TimeRange(start=start, end=base_date) + + elif window == TimeWindow.LAST_WEEK: + # 上周 + days_since_monday = base_date.weekday() + this_monday = base_date - timedelta(days=days_since_monday) + end = this_monday - timedelta(days=1) # 上周日 + start = end - timedelta(days=6) # 上周一 + return TimeRange(start=start, end=end) + + elif window == TimeWindow.THIS_MONTH: + # 本月 + start = base_date.replace(day=1) + return TimeRange(start=start, end=base_date) + + elif window == TimeWindow.LAST_MONTH: + # 上月 + month_start = base_date.replace(day=1) + end = month_start - timedelta(days=1) + start = end.replace(day=1) + return TimeRange(start=start, end=end) + + elif window == TimeWindow.LAST_3_MONTHS_EXCL_CURRENT: + # 前3个月（不含本月）：从三个月前月初到上月月末 + current_month_start = self.get_month_first_day(base_date) + end = current_month_start - timedelta(days=1) + start = self.get_month_first_day(self._shift_months(current_month_start, -3)) + return TimeRange(start=start, end=end) + + elif window == TimeWindow.LAST_3_MONTHS_INCL_CURRENT: + # 前3个月（含本月）：从两个月前月初到当前日期 + current_month_start = self.get_month_first_day(base_date) + start = self.get_month_first_day(self._shift_months(current_month_start, -2)) + return TimeRange(start=start, end=base_date) + + elif window == TimeWindow.THIS_QUARTER: + # 本季度 + quarter = (base_date.month - 1) // 3 + start_month = quarter * 3 + 1 + start = base_date.replace(month=start_month, day=1) + return TimeRange(start=start, end=base_date) + + elif window == TimeWindow.LAST_QUARTER: + # 上季度 + quarter = (base_date.month - 1) // 3 + start_month = quarter * 3 + 1 + this_quarter_start = base_date.replace(month=start_month, day=1) + end = this_quarter_start - timedelta(days=1) + prev_quarter = (end.month - 1) // 3 + prev_start_month = prev_quarter * 3 + 1 + start = end.replace(month=prev_start_month, day=1) + return TimeRange(start=start, end=end) + + elif window == TimeWindow.LAST_6_MONTHS: + # 最近半年（不含本月） + month_start = self.get_month_first_day(base_date) + end = month_start - timedelta(days=1) + start = self.get_month_first_day(self._shift_months(month_start, -6)) + return TimeRange(start=start, end=end) + + raise ValueError(f"不支持的时间窗口类型: {window}") + + def get_comparison_range(self, time_range: TimeRange) -> TimeRange: + """ + 计算环比区间（上一个等长区间） + + 环比规则：对比上一个等长区间 + + Args: + time_range: 当前时间范围 + + Returns: + 环比时间范围 + """ + duration = (time_range.end - time_range.start).days + 1 + prev_end = time_range.start - timedelta(days=1) + prev_start = prev_end - timedelta(days=duration - 1) + return TimeRange(start=prev_start, end=prev_end) + + def get_month_first_day(self, dt: date) -> date: + """获取月第一天""" + return dt.replace(day=1) + + def get_month_last_day(self, dt: date) -> date: + """获取月最后一天""" + last_day = calendar.monthrange(dt.year, dt.month)[1] + return dt.replace(day=last_day) + + def _shift_months(self, base_date: date, months: int) -> date: + """ + 按月偏移日期（保持日不越界） + """ + total_months = base_date.year * 12 + (base_date.month - 1) + months + year = total_months // 12 + month = total_months % 12 + 1 + last_day = calendar.monthrange(year, month)[1] + day = min(base_date.day, last_day) + return date(year, month, day) + + def is_new_hire_in_month(self, hire_date: date, stat_month: date) -> bool: + """ + 判断是否为新入职（月1日0点后入职） + + 新入职定档规则：月1日0点之后入职的，计算为新入职 + + Args: + hire_date: 入职日期 + stat_month: 统计月份（月第一天） + + Returns: + 是否为新入职 + """ + month_start = self.get_month_first_day(stat_month) + return hire_date >= month_start + + # ========================================================================== + # 配置表读取方法 + # ========================================================================== + + def load_config_cache(self, force_reload: bool = False) -> ConfigCache: + """ + 加载配置表缓存 + + Args: + force_reload: 是否强制重新加载 + + Returns: + ConfigCache对象 + """ + now = datetime.now(self.tz) + + # 检查缓存是否有效 + if ( + not force_reload + and self._config_cache is not None + and (now - self._config_cache.loaded_at).total_seconds() < self._config_cache_ttl + ): + return self._config_cache + + self.logger.debug("重新加载DWS配置表缓存") + + # 加载绩效档位配置 + performance_tiers = self._load_performance_tiers() + + # 加载等级定价配置 + level_prices = self._load_level_prices() + + # 加载奖金规则配置 + bonus_rules = self._load_bonus_rules() + + # 加载区域分类映射 + area_categories = self._load_area_categories() + + # 加载技能类型映射 + skill_types = self._load_skill_types() + + self._config_cache = ConfigCache( + performance_tiers=performance_tiers, + level_prices=level_prices, + bonus_rules=bonus_rules, + area_categories=area_categories, + skill_types=skill_types, + loaded_at=now + ) + + return self._config_cache + + def _load_performance_tiers(self) -> List[Dict[str, Any]]: + """ + 加载绩效档位配置 + + 字段说明（来自DWS数据库处理需求.md）： + - base_deduction: 专业课抽成（元/小时），球房从基础课每小时扣除的金额 + - bonus_deduction_ratio: 打赏课抽成比例，球房从附加课收入中扣除的比例 + - vacation_days: 次月可休假天数 + - vacation_unlimited: 休假自由标记（5档为TRUE） + """ + sql = """ + SELECT + tier_id, tier_code, tier_name, tier_level, + min_hours, max_hours, + base_deduction, bonus_deduction_ratio, + vacation_days, vacation_unlimited, + is_new_hire_tier, effective_from, effective_to + FROM billiards_dws.cfg_performance_tier + ORDER BY tier_level ASC, effective_from ASC + """ + rows = self.db.query(sql) + return [dict(row) for row in rows] if rows else [] + + def _load_level_prices(self) -> List[Dict[str, Any]]: + """加载等级定价配置""" + sql = """ + SELECT + price_id, level_code, level_name, + base_course_price, bonus_course_price, + effective_from, effective_to + FROM billiards_dws.cfg_assistant_level_price + ORDER BY level_code ASC, effective_from DESC + """ + rows = self.db.query(sql) + return [dict(row) for row in rows] if rows else [] + + def _load_bonus_rules(self) -> List[Dict[str, Any]]: + """加载奖金规则配置""" + sql = """ + SELECT + rule_id, rule_type, rule_code, rule_name, + threshold_hours, rank_position, bonus_amount, + is_cumulative, priority, + effective_from, effective_to + FROM billiards_dws.cfg_bonus_rules + ORDER BY rule_type, priority DESC, effective_from DESC + """ + rows = self.db.query(sql) + return [dict(row) for row in rows] if rows else [] + + def _load_area_categories(self) -> Dict[str, Dict[str, Any]]: + """加载区域分类映射""" + sql = """ + SELECT + source_area_name, category_code, category_name, + match_type, match_priority + FROM billiards_dws.cfg_area_category + WHERE is_active = TRUE + ORDER BY match_priority ASC + """ + rows = self.db.query(sql) + if not rows: + return {} + + result = {} + for row in rows: + row_dict = dict(row) + result[row_dict['source_area_name']] = row_dict + return result + + def _load_skill_types(self) -> Dict[int, Dict[str, Any]]: + """加载技能类型映射""" + sql = """ + SELECT + skill_id, skill_name, + course_type_code, course_type_name + FROM billiards_dws.cfg_skill_type + WHERE is_active = TRUE + """ + rows = self.db.query(sql) + if not rows: + return {} + + result = {} + for row in rows: + row_dict = dict(row) + result[int(row_dict['skill_id'])] = row_dict + return result + + # ========================================================================== + # 配置应用方法 + # ========================================================================== + + def _filter_by_effective_date( + self, + items: List[Dict[str, Any]], + effective_date: Optional[date] + ) -> List[Dict[str, Any]]: + """ + 按生效期过滤配置项 + """ + ref_date = effective_date or date.today() + results: List[Dict[str, Any]] = [] + for item in items: + eff_from = item.get('effective_from') + eff_to = item.get('effective_to') + if eff_from and ref_date < eff_from: + continue + if eff_to and ref_date > eff_to: + continue + results.append(item) + return results + + def get_performance_tier( + self, + effective_hours: Decimal, + is_new_hire: bool, + effective_date: Optional[date] = None, + max_tier_level: Optional[int] = None + ) -> Optional[Dict[str, Any]]: + """ + 根据有效业绩小时数匹配绩效档位 + + Args: + effective_hours: 有效业绩小时数 + is_new_hire: 是否为新入职 + effective_date: 生效日期（用于历史月份） + + Returns: + 匹配的档位配置，如果没有匹配则返回None + """ + config = self.load_config_cache() + tiers = self._filter_by_effective_date(config.performance_tiers, effective_date) + + if max_tier_level is not None: + tiers = [ + t for t in tiers + if t.get('tier_level') is None or int(t.get('tier_level')) <= max_tier_level + ] + + # 新入职使用专用档位 + if is_new_hire: + new_hire_tiers = [t for t in tiers if t.get('is_new_hire_tier')] + if new_hire_tiers: + new_hire_tiers.sort(key=lambda x: x.get('tier_level') or 0) + return new_hire_tiers[0] + + # 按阈值匹配档位 + for tier in tiers: + if tier.get('is_new_hire_tier'): + continue + min_hours = Decimal(str(tier.get('min_hours', 0))) + max_hours = tier.get('max_hours') + if max_hours is not None: + max_hours = Decimal(str(max_hours)) + + if effective_hours >= min_hours: + if max_hours is None or effective_hours < max_hours: + return tier + + return None + + def get_performance_tier_by_id( + self, + tier_id: Optional[int], + effective_date: Optional[date] = None + ) -> Optional[Dict[str, Any]]: + """ + 通过档位ID获取配置（支持生效期筛选） + """ + if not tier_id: + return None + + config = self.load_config_cache() + tiers = self._filter_by_effective_date(config.performance_tiers, effective_date) + for tier in tiers: + if tier.get('tier_id') == tier_id: + return tier + return None + + def get_level_price( + self, + level_code: int, + effective_date: Optional[date] = None + ) -> Optional[Dict[str, Any]]: + """ + 获取助教等级对应的单价（SCD2口径，按生效日期取值） + + Args: + level_code: 等级代码 + effective_date: 生效日期 + + Returns: + 等级定价配置 + """ + config = self.load_config_cache() + prices = self._filter_by_effective_date(config.level_prices, effective_date) + + for price in prices: + if price.get('level_code') == level_code: + return price + + return None + + def get_course_type(self, skill_id: int) -> CourseType: + """ + 根据skill_id获取课程类型 + + Args: + skill_id: 技能ID + + Returns: + CourseType枚举 + """ + config = self.load_config_cache() + skill_config = config.skill_types.get(skill_id) + + if skill_config: + code = skill_config.get('course_type_code', 'BASE') + return CourseType.BONUS if code == 'BONUS' else CourseType.BASE + + # 默认为基础课 + return CourseType.BASE + + def get_area_category(self, area_name: Optional[str]) -> Dict[str, str]: + """ + 获取区域分类（支持精确匹配、模糊匹配、兜底） + + Args: + area_name: 原始区域名称 + + Returns: + 包含 category_code 和 category_name 的字典 + """ + config = self.load_config_cache() + + if not area_name: + # 无区域名称，返回默认 + return {'category_code': 'OTHER', 'category_name': '其他区域'} + + # 1. 精确匹配 + if area_name in config.area_categories: + cat = config.area_categories[area_name] + if cat.get('match_type') == 'EXACT': + return { + 'category_code': cat['category_code'], + 'category_name': cat['category_name'] + } + + # 2. 模糊匹配（按优先级） + for key, cat in config.area_categories.items(): + if cat.get('match_type') == 'LIKE': + pattern = key.replace('%', '') + if pattern and pattern in area_name: + return { + 'category_code': cat['category_code'], + 'category_name': cat['category_name'] + } + + # 3. 兜底 + if 'DEFAULT' in config.area_categories: + cat = config.area_categories['DEFAULT'] + return { + 'category_code': cat['category_code'], + 'category_name': cat['category_name'] + } + + return {'category_code': 'OTHER', 'category_name': '其他区域'} + + def calculate_sprint_bonus( + self, + effective_hours: Decimal, + effective_date: Optional[date] = None + ) -> Decimal: + """ + 计算冲刺奖金（不累计，取最高档） + + 冲刺奖金规则： + - H>=190: 300元 + - H>=220: 800元 + - 不累计，取最高档 + + Args: + effective_hours: 有效业绩小时数 + effective_date: 生效日期 + + Returns: + 冲刺奖金金额 + """ + config = self.load_config_cache() + bonus_rules = self._filter_by_effective_date(config.bonus_rules, effective_date) + + # 筛选冲刺奖金规则，按优先级降序 + sprint_rules = [ + r for r in bonus_rules + if r.get('rule_type') == 'SPRINT' + ] + sprint_rules.sort(key=lambda x: x.get('priority', 0), reverse=True) + + # 取满足条件的最高档 + for rule in sprint_rules: + threshold = Decimal(str(rule.get('threshold_hours', 0))) + if effective_hours >= threshold: + return Decimal(str(rule.get('bonus_amount', 0))) + + return Decimal('0') + + def calculate_top_rank_bonus( + self, + rank: int, + effective_date: Optional[date] = None + ) -> Decimal: + """ + 计算Top排名奖金 + + Top3奖金规则： + - 第1名: 1000元 + - 第2名: 600元 + - 第3名: 400元 + - 并列都算 + + Args: + rank: 排名（考虑并列后的排名） + effective_date: 生效日期 + + Returns: + 排名奖金金额 + """ + config = self.load_config_cache() + bonus_rules = self._filter_by_effective_date(config.bonus_rules, effective_date) + + if rank > 3: + return Decimal('0') + + for rule in bonus_rules: + if rule.get('rule_type') == 'TOP_RANK': + if rule.get('rank_position') == rank: + return Decimal(str(rule.get('bonus_amount', 0))) + + return Decimal('0') + + # ========================================================================== + # DWD数据读取方法 + # ========================================================================== + + def iter_dwd_rows( + self, + table_name: str, + columns: List[str], + start_date: date, + end_date: date, + date_col: str = "created_at", + where_clause: str = "", + order_by: str = "", + batch_size: int = 1000 + ) -> Iterator[List[Dict[str, Any]]]: + """ + 分批迭代读取DWD表数据 + + Args: + table_name: DWD表名（不含schema） + columns: 需要查询的字段列表 + start_date: 开始日期（含） + end_date: 结束日期（含） + date_col: 日期过滤字段 + where_clause: 额外的WHERE条件（不含WHERE关键字） + order_by: 排序字段（不含ORDER BY关键字） + batch_size: 批次大小 + + Yields: + 每批次的数据行列表 + """ + offset = 0 + cols_str = ", ".join(columns) + + # 构建WHERE条件 + where_parts = [f"DATE({date_col}) >= %s", f"DATE({date_col}) <= %s"] + params: List[Any] = [start_date, end_date] + + if where_clause: + where_parts.append(f"({where_clause})") + + where_str = " AND ".join(where_parts) + + # 构建ORDER BY + order_str = f"ORDER BY {order_by}" if order_by else f"ORDER BY {date_col} ASC" + + while True: + sql = f""" + SELECT {cols_str} + FROM {self.DWD_SCHEMA}.{table_name} + WHERE {where_str} + {order_str} + LIMIT %s OFFSET %s + """ + + rows = self.db.query(sql, (*params, batch_size, offset)) + + if not rows: + break + + yield [dict(row) for row in rows] + + if len(rows) < batch_size: + break + + offset += batch_size + + def query_dwd( + self, + sql: str, + params: Optional[Tuple] = None + ) -> List[Dict[str, Any]]: + """ + 直接执行DWD查询 + + Args: + sql: SQL语句 + params: 参数元组 + + Returns: + 查询结果列表 + """ + rows = self.db.query(sql, params) + return [dict(row) for row in rows] if rows else [] + + # ========================================================================== + # SCD2维度取值方法 + # ========================================================================== + + def get_assistant_level_asof( + self, + assistant_id: int, + asof_date: date + ) -> Optional[Dict[str, Any]]: + """ + 获取助教在指定日期的等级（SCD2 as-of取值） + + 助教等级是SCD2维度，历史月份不能直接用"当前等级"。 + 需要按有效期as-of join取数。 + + Args: + assistant_id: 助教ID + asof_date: 取值日期 + + Returns: + 助教等级信息，包含level_code和level_name + """ + sql = """ + SELECT + site_assistant_id, + nickname, + assistant_level AS level_code, + CASE assistant_level + WHEN 8 THEN '初级' + WHEN 10 THEN '中级' + WHEN 20 THEN '高级' + WHEN 30 THEN '星级' + WHEN 40 THEN '金牌' + ELSE '未知' + END AS level_name, + valid_from, + valid_to + FROM billiards_dwd.dim_assistant + WHERE site_assistant_id = %s + AND valid_from <= %s + AND (valid_to IS NULL OR valid_to > %s) + ORDER BY valid_from DESC + LIMIT 1 + """ + rows = self.db.query(sql, (assistant_id, asof_date, asof_date)) + return dict(rows[0]) if rows else None + + def get_member_card_balance_asof( + self, + member_id: int, + asof_date: date + ) -> Dict[str, Decimal]: + """ + 获取会员在指定日期的卡余额（SCD2 as-of取值） + + Args: + member_id: 会员ID + asof_date: 取值日期 + + Returns: + 卡余额字典，包含cash_balance和gift_balance + """ + sql = """ + SELECT + card_type_id, + balance + FROM billiards_dwd.dim_member_card_account + WHERE tenant_member_id = %s + AND valid_from <= %s + AND (valid_to IS NULL OR valid_to > %s) + """ + rows = self.db.query(sql, (member_id, asof_date, asof_date)) + + # 卡类型ID映射 + CASH_CARD_TYPE_ID = 2793249295533893 + GIFT_CARD_TYPE_IDS = [ + 2791990152417157, # 台费卡 + 2793266846533445, # 活动抵用券 + 2794699703437125, # 酒水卡 + ] + + cash_balance = Decimal('0') + gift_balance = Decimal('0') + + for row in (rows or []): + row_dict = dict(row) + card_type_id = row_dict.get('card_type_id') + balance = Decimal(str(row_dict.get('balance', 0))) + + if card_type_id == CASH_CARD_TYPE_ID: + cash_balance += balance + elif card_type_id in GIFT_CARD_TYPE_IDS: + gift_balance += balance + + return { + 'cash_balance': cash_balance, + 'gift_balance': gift_balance, + 'total_balance': cash_balance + gift_balance + } + + # ========================================================================== + # 幂等更新方法 + # ========================================================================== + + def delete_existing_data( + self, + context: TaskContext, + date_col: str = "stat_date", + extra_conditions: Optional[Dict[str, Any]] = None + ) -> int: + """ + 删除已存在的数据（实现幂等更新） + + Args: + context: 任务上下文 + date_col: 日期字段名 + extra_conditions: 额外的删除条件 + + Returns: + 删除的行数 + """ + target_table = self.get_target_table() + full_table = f"{self.DWS_SCHEMA}.{target_table}" + + # 构建WHERE条件 + where_parts = [f"site_id = %s"] + params: List[Any] = [context.store_id] + + # 日期范围条件 + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + + where_parts.append(f"{date_col} >= %s") + params.append(start_date) + where_parts.append(f"{date_col} <= %s") + params.append(end_date) + + # 额外条件 + if extra_conditions: + for col, val in extra_conditions.items(): + where_parts.append(f"{col} = %s") + params.append(val) + + where_str = " AND ".join(where_parts) + + sql = f"DELETE FROM {full_table} WHERE {where_str}" + + with self.db.conn.cursor() as cur: + cur.execute(sql, params) + deleted = cur.rowcount + + self.logger.debug( + "%s: 删除已存在数据 %d 行，条件: %s", + self.get_task_code(), deleted, where_str + ) + + return deleted + + def bulk_insert( + self, + rows: List[Dict[str, Any]], + columns: Optional[List[str]] = None + ) -> int: + """ + 批量插入数据 + + Args: + rows: 数据行列表 + columns: 字段列表（如果为None则从第一行获取） + + Returns: + 插入的行数 + """ + if not rows: + return 0 + + target_table = self.get_target_table() + full_table = f"{self.DWS_SCHEMA}.{target_table}" + + if columns is None: + columns = list(rows[0].keys()) + + cols_str = ", ".join(columns) + placeholders = ", ".join(["%s"] * len(columns)) + + sql = f"INSERT INTO {full_table} ({cols_str}) VALUES ({placeholders})" + + inserted = 0 + with self.db.conn.cursor() as cur: + for row in rows: + values = [row.get(col) for col in columns] + cur.execute(sql, values) + inserted += cur.rowcount + + return inserted + + def upsert( + self, + rows: List[Dict[str, Any]], + columns: Optional[List[str]] = None, + update_columns: Optional[List[str]] = None + ) -> Tuple[int, int]: + """ + 批量upsert（插入或更新） + + Args: + rows: 数据行列表 + columns: 全部字段列表 + update_columns: 需要更新的字段列表 + + Returns: + (inserted, updated) 元组 + """ + if not rows: + return 0, 0 + + target_table = self.get_target_table() + full_table = f"{self.DWS_SCHEMA}.{target_table}" + primary_keys = self.get_primary_keys() + + if columns is None: + columns = list(rows[0].keys()) + + if update_columns is None: + update_columns = [c for c in columns if c not in primary_keys and c not in ('created_at',)] + + cols_str = ", ".join(columns) + placeholders = ", ".join(["%s"] * len(columns)) + conflict_cols = ", ".join(primary_keys) + + update_parts = [f"{col} = EXCLUDED.{col}" for col in update_columns] + update_parts.append("updated_at = NOW()") + update_str = ", ".join(update_parts) + + sql = f""" + INSERT INTO {full_table} ({cols_str}) + VALUES ({placeholders}) + ON CONFLICT ({conflict_cols}) + DO UPDATE SET {update_str} + """ + + inserted = 0 + updated = 0 + + with self.db.conn.cursor() as cur: + for row in rows: + values = [row.get(col) for col in columns] + cur.execute(sql, values) + # PostgreSQL的INSERT ON CONFLICT返回1表示有操作 + if cur.rowcount > 0: + # 无法精确区分insert和update，统计为inserted + inserted += 1 + + return inserted, updated + + # ========================================================================== + # 滚动窗口统计方法 + # ========================================================================== + + def calculate_rolling_stats( + self, + base_date: date, + entity_id: int, + entity_type: str, + stat_sql: str, + windows: Optional[List[int]] = None + ) -> Dict[str, Any]: + """ + 计算滚动窗口统计 + + Args: + base_date: 基准日期 + entity_id: 实体ID（如assistant_id或member_id） + entity_type: 实体类型（用于SQL参数名） + stat_sql: 统计SQL模板，需要包含 {window_days} 和 {entity_id} 占位符 + windows: 窗口天数列表，默认为 [7, 10, 15, 30, 60, 90] + + Returns: + 各窗口的统计结果字典 + """ + if windows is None: + windows = self.ROLLING_WINDOWS + + results = {} + + for days in windows: + start_date = base_date - timedelta(days=days - 1) + + # 替换SQL中的占位符 + sql = stat_sql.format( + window_days=days, + entity_id=entity_id, + start_date=start_date, + end_date=base_date + ) + + rows = self.db.query(sql) + if rows: + for key, value in dict(rows[0]).items(): + results[f"{key}_{days}d"] = value + + return results + + # ========================================================================== + # 排名计算方法 + # ========================================================================== + + def calculate_rank_with_ties( + self, + values: List[Tuple[int, Decimal]] + ) -> List[Tuple[int, int, int]]: + """ + 计算考虑并列的排名 + + Top3排名口径：按绩效总小时数，如遇并列则都算， + 比如2个第一，则记为2个第一，一个第三。 + + Args: + values: (entity_id, score) 元组列表 + + Returns: + (entity_id, rank, dense_rank) 元组列表 + rank: 考虑并列的排名（如2个第一，下一个是3） + dense_rank: 密集排名（如2个第一，下一个是2） + """ + if not values: + return [] + + # 按分数降序排序 + sorted_values = sorted(values, key=lambda x: x[1], reverse=True) + + results = [] + prev_score = None + prev_rank = 0 + count = 0 + + for entity_id, score in sorted_values: + count += 1 + + if score != prev_score: + # 新的分数，rank为当前计数 + current_rank = count + prev_score = score + else: + # 相同分数，rank保持不变 + current_rank = prev_rank + + prev_rank = current_rank + results.append((entity_id, current_rank, count)) + + return results + + # ========================================================================== + # 散客过滤 + # ========================================================================== + + def is_guest(self, member_id: Optional[int]) -> bool: + """ + 判断是否为散客 + + 散客处理：member_id=0 的客户是散客，不进入客户维度统计 + + Args: + member_id: 会员ID + + Returns: + 是否为散客 + """ + return member_id is None or member_id == 0 + + # ========================================================================== + # 工具方法 + # ========================================================================== + + def safe_decimal(self, value: Any, default: Decimal = Decimal('0')) -> Decimal: + """安全转换为Decimal""" + if value is None: + return default + try: + return Decimal(str(value)) + except (ValueError, TypeError): + return default + + def safe_int(self, value: Any, default: int = 0) -> int: + """安全转换为int""" + if value is None: + return default + try: + return int(value) + except (ValueError, TypeError): + return default + + def seconds_to_hours(self, seconds: int) -> Decimal: + """秒转换为小时""" + return Decimal(str(seconds)) / Decimal('3600') + + def hours_to_seconds(self, hours: Decimal) -> int: + """小时转换为秒""" + return int(hours * Decimal('3600')) diff --git a/etl_billiards/tasks/dws/finance_daily_task.py b/etl_billiards/tasks/dws/finance_daily_task.py new file mode 100644 index 0000000..6175693 --- /dev/null +++ b/etl_billiards/tasks/dws/finance_daily_task.py @@ -0,0 +1,574 @@ +# -*- coding: utf-8 -*- +""" +财务日度汇总任务 + +功能说明： + 以"日期"为粒度，汇总当日财务数据 + +数据来源： + - dwd_settlement_head: 结账单头表 + - dwd_groupbuy_redemption: 团购核销 + - dwd_recharge_order: 充值订单 + - dws_finance_expense_summary: 支出汇总（Excel导入） + - dws_platform_settlement: 平台回款/服务费（Excel导入） + +目标表： + billiards_dws.dws_finance_daily_summary + +更新策略： + - 更新频率：每小时更新当日数据 + - 幂等方式：delete-before-insert（按日期） + +业务规则： + - 发生额：table_charge_money + goods_money + assistant_pd_money + assistant_cx_money + - 团购优惠：coupon_amount - 团购支付金额 + - 团购支付：pl_coupon_sale_amount 或关联 groupbuy_redemption.ledger_unit_price + - 首充/续充：通过 is_first 字段区分 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +import calendar +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class FinanceDailyTask(BaseDwsTask): + """ + 财务日度汇总任务 + + 汇总每日的： + - 发生额（正价） + - 优惠拆分 + - 确认收入 + - 现金流（流入/流出） + - 充值统计（首充/续充） + - 订单统计 + """ + + def get_task_code(self) -> str: + return "DWS_FINANCE_DAILY" + + def get_target_table(self) -> str: + return "dws_finance_daily_summary" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "stat_date"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据 + """ + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + self.logger.info( + "%s: 提取数据，日期范围 %s ~ %s", + self.get_task_code(), start_date, end_date + ) + + # 1. 获取结账单汇总 + settlement_summary = self._extract_settlement_summary(site_id, start_date, end_date) + + # 2. 获取团购核销汇总 + groupbuy_summary = self._extract_groupbuy_summary(site_id, start_date, end_date) + + # 3. 获取充值汇总 + recharge_summary = self._extract_recharge_summary(site_id, start_date, end_date) + + # 4. 获取支出汇总（来自导入表） + expense_summary = self._extract_expense_summary(site_id, start_date, end_date) + + # 5. 获取平台回款汇总（来自导入表） + platform_summary = self._extract_platform_summary(site_id, start_date, end_date) + + # 6. 获取大客户优惠明细（用于拆分手动优惠） + big_customer_summary = self._extract_big_customer_discounts(site_id, start_date, end_date) + + return { + 'settlement_summary': settlement_summary, + 'groupbuy_summary': groupbuy_summary, + 'recharge_summary': recharge_summary, + 'expense_summary': expense_summary, + 'platform_summary': platform_summary, + 'big_customer_summary': big_customer_summary, + 'start_date': start_date, + 'end_date': end_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据：按日期聚合 + """ + settlement_summary = extracted['settlement_summary'] + groupbuy_summary = extracted['groupbuy_summary'] + recharge_summary = extracted['recharge_summary'] + expense_summary = extracted['expense_summary'] + platform_summary = extracted['platform_summary'] + big_customer_summary = extracted['big_customer_summary'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，%d 天结账数据，%d 天充值数据", + self.get_task_code(), len(settlement_summary), len(recharge_summary) + ) + + # 按日期合并数据 + dates = set() + for item in settlement_summary + recharge_summary + expense_summary + platform_summary: + stat_date = item.get('stat_date') + if stat_date: + dates.add(stat_date) + + # 构建索引 + settle_index = {s['stat_date']: s for s in settlement_summary} + groupbuy_index = {g['stat_date']: g for g in groupbuy_summary} + recharge_index = {r['stat_date']: r for r in recharge_summary} + expense_index = {e['stat_date']: e for e in expense_summary} + platform_index = {p['stat_date']: p for p in platform_summary} + big_customer_index = {b['stat_date']: b for b in big_customer_summary} + + results = [] + for stat_date in sorted(dates): + settle = settle_index.get(stat_date, {}) + groupbuy = groupbuy_index.get(stat_date, {}) + recharge = recharge_index.get(stat_date, {}) + expense = expense_index.get(stat_date, {}) + platform = platform_index.get(stat_date, {}) + big_customer = big_customer_index.get(stat_date, {}) + + record = self._build_daily_record( + stat_date, settle, groupbuy, recharge, expense, platform, big_customer, site_id + ) + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + deleted = self.delete_existing_data(context, date_col="stat_date") + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_settlement_summary( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取结账单日汇总 + """ + sql = """ + SELECT + DATE(create_time) AS stat_date, + COUNT(*) AS order_count, + COUNT(CASE WHEN member_id != 0 AND member_id IS NOT NULL THEN 1 END) AS member_order_count, + COUNT(CASE WHEN member_id = 0 OR member_id IS NULL THEN 1 END) AS guest_order_count, + -- 发生额（正价） + SUM(table_charge_money) AS table_fee_amount, + SUM(goods_money) AS goods_amount, + SUM(assistant_pd_money) AS assistant_pd_amount, + SUM(assistant_cx_money) AS assistant_cx_amount, + SUM(table_charge_money + goods_money + assistant_pd_money + assistant_cx_money) AS gross_amount, + -- 支付 + SUM(pay_amount) AS cash_pay_amount, + SUM(recharge_card_amount) AS card_pay_amount, + SUM(balance_amount) AS balance_pay_amount, + SUM(gift_card_amount) AS gift_card_pay_amount, + -- 优惠 + SUM(coupon_amount) AS coupon_amount, + SUM(adjust_amount) AS adjust_amount, + SUM(member_discount_amount) AS member_discount_amount, + SUM(rounding_amount) AS rounding_amount, + SUM(pl_coupon_sale_amount) AS pl_coupon_sale_amount, + -- 消费金额 + SUM(consume_money) AS total_consume + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %s + AND DATE(create_time) >= %s + AND DATE(create_time) <= %s + GROUP BY DATE(create_time) + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_groupbuy_summary( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取团购核销日汇总 + """ + sql = """ + SELECT + DATE(redeem_time) AS stat_date, + COUNT(*) AS groupbuy_count, + SUM(ledger_unit_price) AS groupbuy_pay_total + FROM billiards_dwd.dwd_groupbuy_redemption + WHERE site_id = %s + AND DATE(redeem_time) >= %s + AND DATE(redeem_time) <= %s + GROUP BY DATE(redeem_time) + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_recharge_summary( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取充值日汇总 + """ + sql = """ + SELECT + DATE(create_time) AS stat_date, + COUNT(*) AS recharge_count, + SUM(pay_money + gift_money) AS recharge_total, + SUM(pay_money) AS recharge_cash, + SUM(gift_money) AS recharge_gift, + COUNT(CASE WHEN is_first = 1 THEN 1 END) AS first_recharge_count, + SUM(CASE WHEN is_first = 1 THEN pay_money + gift_money ELSE 0 END) AS first_recharge_total, + SUM(CASE WHEN is_first = 1 THEN pay_money ELSE 0 END) AS first_recharge_cash, + SUM(CASE WHEN is_first = 1 THEN gift_money ELSE 0 END) AS first_recharge_gift, + COUNT(CASE WHEN is_first = 0 OR is_first IS NULL THEN 1 END) AS renewal_count, + SUM(CASE WHEN is_first = 0 OR is_first IS NULL THEN pay_money + gift_money ELSE 0 END) AS renewal_total, + SUM(CASE WHEN is_first = 0 OR is_first IS NULL THEN pay_money ELSE 0 END) AS renewal_cash, + SUM(CASE WHEN is_first = 0 OR is_first IS NULL THEN gift_money ELSE 0 END) AS renewal_gift, + COUNT(DISTINCT member_id) AS recharge_member_count + FROM billiards_dwd.dwd_recharge_order + WHERE site_id = %s + AND DATE(create_time) >= %s + AND DATE(create_time) <= %s + GROUP BY DATE(create_time) + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_expense_summary( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取支出汇总（来自导入表，按月分摊到日） + """ + if start_date > end_date: + return [] + + start_month = start_date.replace(day=1) + end_month = end_date.replace(day=1) + + sql = """ + SELECT + expense_month, + SUM(expense_amount) AS expense_amount + FROM billiards_dws.dws_finance_expense_summary + WHERE site_id = %s + AND expense_month >= %s + AND expense_month <= %s + GROUP BY expense_month + """ + rows = self.db.query(sql, (site_id, start_month, end_month)) + if not rows: + return [] + + daily_totals: Dict[date, Decimal] = {} + for row in rows: + row_dict = dict(row) + month_date = row_dict.get('expense_month') + if not month_date: + continue + amount = self.safe_decimal(row_dict.get('expense_amount', 0)) + days_in_month = calendar.monthrange(month_date.year, month_date.month)[1] + daily_amount = amount / Decimal(str(days_in_month)) if days_in_month > 0 else Decimal('0') + + for day in range(1, days_in_month + 1): + stat_date = date(month_date.year, month_date.month, day) + if stat_date < start_date or stat_date > end_date: + continue + daily_totals[stat_date] = daily_totals.get(stat_date, Decimal('0')) + daily_amount + + return [ + {'stat_date': stat_date, 'expense_amount': amount} + for stat_date, amount in sorted(daily_totals.items()) + ] + + def _extract_platform_summary( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取平台回款/服务费汇总（来自导入表） + """ + sql = """ + SELECT + settlement_date AS stat_date, + SUM(settlement_amount) AS settlement_amount, + SUM(commission_amount) AS commission_amount, + SUM(service_fee) AS service_fee + FROM billiards_dws.dws_platform_settlement + WHERE site_id = %s + AND settlement_date >= %s + AND settlement_date <= %s + GROUP BY settlement_date + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_big_customer_discounts( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取大客户优惠（用于拆分手动调整） + """ + member_ids = self._parse_id_list(self.config.get("dws.discount.big_customer_member_ids")) + order_ids = self._parse_id_list(self.config.get("dws.discount.big_customer_order_ids")) + if not member_ids and not order_ids: + return [] + + sql = """ + SELECT + pay_time::DATE AS stat_date, + order_settle_id, + member_id, + adjust_amount + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %s + AND pay_time >= %s + AND pay_time < %s + INTERVAL '1 day' + AND adjust_amount != 0 + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + if not rows: + return [] + + result: Dict[date, Dict[str, Any]] = {} + for row in rows: + row_dict = dict(row) + stat_date = row_dict.get('stat_date') + if not stat_date: + continue + order_id = row_dict.get('order_settle_id') + member_id = row_dict.get('member_id') + if order_id not in order_ids and member_id not in member_ids: + continue + amount = abs(self.safe_decimal(row_dict.get('adjust_amount', 0))) + entry = result.setdefault(stat_date, {'stat_date': stat_date, 'big_customer_amount': Decimal('0'), 'big_customer_count': 0}) + entry['big_customer_amount'] += amount + entry['big_customer_count'] += 1 + + return list(result.values()) + + def _parse_id_list(self, value: Any) -> set: + if not value: + return set() + if isinstance(value, str): + items = [v.strip() for v in value.split(",") if v.strip()] + return {int(v) for v in items if v.isdigit()} + if isinstance(value, (list, tuple, set)): + result = set() + for item in value: + if item is None: + continue + try: + result.add(int(item)) + except (ValueError, TypeError): + continue + return result + return set() + + # ========================================================================== + # 数据转换方法 + # ========================================================================== + + def _build_daily_record( + self, + stat_date: date, + settle: Dict[str, Any], + groupbuy: Dict[str, Any], + recharge: Dict[str, Any], + expense: Dict[str, Any], + platform: Dict[str, Any], + big_customer: Dict[str, Any], + site_id: int + ) -> Dict[str, Any]: + """ + 构建日度财务记录 + """ + # 发生额 + gross_amount = self.safe_decimal(settle.get('gross_amount', 0)) + table_fee_amount = self.safe_decimal(settle.get('table_fee_amount', 0)) + goods_amount = self.safe_decimal(settle.get('goods_amount', 0)) + assistant_pd_amount = self.safe_decimal(settle.get('assistant_pd_amount', 0)) + assistant_cx_amount = self.safe_decimal(settle.get('assistant_cx_amount', 0)) + + # 支付 + cash_pay_amount = self.safe_decimal(settle.get('cash_pay_amount', 0)) + card_pay_amount = self.safe_decimal(settle.get('card_pay_amount', 0)) + balance_pay_amount = self.safe_decimal(settle.get('balance_pay_amount', 0)) + gift_card_pay_amount = self.safe_decimal(settle.get('gift_card_pay_amount', 0)) + + # 优惠 + coupon_amount = self.safe_decimal(settle.get('coupon_amount', 0)) + pl_coupon_sale = self.safe_decimal(settle.get('pl_coupon_sale_amount', 0)) + groupbuy_pay = self.safe_decimal(groupbuy.get('groupbuy_pay_total', 0)) + + # 团购支付金额：优先使用pl_coupon_sale_amount，否则使用groupbuy核销金额 + if pl_coupon_sale > 0: + groupbuy_pay_amount = pl_coupon_sale + else: + groupbuy_pay_amount = groupbuy_pay + + # 团购优惠 = 团购抵消台费 - 团购支付金额 + discount_groupbuy = coupon_amount - groupbuy_pay_amount + if discount_groupbuy < 0: + discount_groupbuy = Decimal('0') + + adjust_amount = self.safe_decimal(settle.get('adjust_amount', 0)) + member_discount = self.safe_decimal(settle.get('member_discount_amount', 0)) + rounding_amount = self.safe_decimal(settle.get('rounding_amount', 0)) + big_customer_amount = self.safe_decimal(big_customer.get('big_customer_amount', 0)) + other_discount = adjust_amount - big_customer_amount + if other_discount < 0: + other_discount = Decimal('0') + + # 优惠合计 + discount_total = discount_groupbuy + member_discount + gift_card_pay_amount + adjust_amount + rounding_amount + + # 确认收入 + confirmed_income = gross_amount - discount_total + + # 现金流 + platform_settlement_amount = self.safe_decimal(platform.get('settlement_amount', 0)) + platform_fee_amount = ( + self.safe_decimal(platform.get('commission_amount', 0)) + + self.safe_decimal(platform.get('service_fee', 0)) + ) + recharge_cash_inflow = self.safe_decimal(recharge.get('recharge_cash', 0)) + platform_inflow = platform_settlement_amount if platform_settlement_amount > 0 else groupbuy_pay_amount + cash_inflow_total = cash_pay_amount + platform_inflow + recharge_cash_inflow + + cash_outflow_total = self.safe_decimal(expense.get('expense_amount', 0)) + platform_fee_amount + cash_balance_change = cash_inflow_total - cash_outflow_total + + # 卡消费 + cash_card_consume = card_pay_amount + balance_pay_amount + gift_card_consume = gift_card_pay_amount + card_consume_total = cash_card_consume + gift_card_consume + + # 充值统计 + recharge_count = self.safe_int(recharge.get('recharge_count', 0)) + recharge_total = self.safe_decimal(recharge.get('recharge_total', 0)) + recharge_cash = self.safe_decimal(recharge.get('recharge_cash', 0)) + recharge_gift = self.safe_decimal(recharge.get('recharge_gift', 0)) + first_recharge_count = self.safe_int(recharge.get('first_recharge_count', 0)) + first_recharge_amount = self.safe_decimal(recharge.get('first_recharge_total', 0)) + renewal_count = self.safe_int(recharge.get('renewal_count', 0)) + renewal_amount = self.safe_decimal(recharge.get('renewal_total', 0)) + + # 订单统计 + order_count = self.safe_int(settle.get('order_count', 0)) + member_order_count = self.safe_int(settle.get('member_order_count', 0)) + guest_order_count = self.safe_int(settle.get('guest_order_count', 0)) + avg_order_amount = gross_amount / order_count if order_count > 0 else Decimal('0') + + return { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'stat_date': stat_date, + # 发生额 + 'gross_amount': gross_amount, + 'table_fee_amount': table_fee_amount, + 'goods_amount': goods_amount, + 'assistant_pd_amount': assistant_pd_amount, + 'assistant_cx_amount': assistant_cx_amount, + # 优惠 + 'discount_total': discount_total, + 'discount_groupbuy': discount_groupbuy, + 'discount_vip': member_discount, + 'discount_gift_card': gift_card_pay_amount, + 'discount_manual': adjust_amount, + 'discount_rounding': rounding_amount, + 'discount_other': other_discount, + # 确认收入 + 'confirmed_income': confirmed_income, + # 现金流 + 'cash_inflow_total': cash_inflow_total, + 'cash_pay_amount': cash_pay_amount, + 'groupbuy_pay_amount': groupbuy_pay_amount, + 'platform_settlement_amount': platform_settlement_amount, + 'platform_fee_amount': platform_fee_amount, + 'recharge_cash_inflow': recharge_cash_inflow, + 'card_consume_total': card_consume_total, + 'cash_card_consume': cash_card_consume, + 'gift_card_consume': gift_card_consume, + 'cash_outflow_total': cash_outflow_total, + 'cash_balance_change': cash_balance_change, + # 充值统计 + 'recharge_count': recharge_count, + 'recharge_total': recharge_total, + 'recharge_cash': recharge_cash, + 'recharge_gift': recharge_gift, + 'first_recharge_count': first_recharge_count, + 'first_recharge_amount': first_recharge_amount, + 'renewal_count': renewal_count, + 'renewal_amount': renewal_amount, + # 订单统计 + 'order_count': order_count, + 'member_order_count': member_order_count, + 'guest_order_count': guest_order_count, + 'avg_order_amount': avg_order_amount, + } + + +# 便于外部导入 +__all__ = ['FinanceDailyTask'] diff --git a/etl_billiards/tasks/dws/finance_discount_task.py b/etl_billiards/tasks/dws/finance_discount_task.py new file mode 100644 index 0000000..30a305b --- /dev/null +++ b/etl_billiards/tasks/dws/finance_discount_task.py @@ -0,0 +1,410 @@ +# -*- coding: utf-8 -*- +""" +优惠明细分析任务 + +功能说明： + 以"日期+优惠类型"为粒度，分析优惠构成 + +数据来源： + - dwd_settlement_head: 结账单头表（优惠字段） + - dwd_groupbuy_redemption: 团购核销（团购实付金额） + - dwd_member_balance_change: 余额变动（赠送卡消费） + +目标表： + billiards_dws.dws_finance_discount_detail + +更新策略： + - 更新频率：每日更新 + - 幂等方式：delete-before-insert（按日期） + +业务规则： + - 团购优惠 (GROUPBUY): coupon_amount - 团购实付金额 + - 会员折扣 (VIP): member_discount_amount + - 赠送卡抵扣 (GIFT_CARD): gift_card_amount + - 抹零 (ROUNDING): rounding_amount + - 大客户优惠 (BIG_CUSTOMER): 手动调整中标记的大客户订单 + - 其他优惠 (OTHER): 手动调整中除大客户外的部分 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class FinanceDiscountDetailTask(BaseDwsTask): + """ + 优惠明细分析任务 + + 分析各类优惠的使用情况： + - 团购优惠 + - 会员折扣 + - 赠送卡抵扣 + - 手动调整 + - 抹零 + - 其他优惠 + """ + + def get_task_code(self) -> str: + return "DWS_FINANCE_DISCOUNT_DETAIL" + + def get_target_table(self) -> str: + return "dws_finance_discount_detail" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "stat_date", "discount_type_code"] + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 抽取优惠相关数据 + + 数据来源： + 1. settlement_head: 各类优惠字段 + 2. groupbuy_redemption: 团购实付金额 + """ + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + # 从settlement_head抽取优惠数据 + discount_summary = self._extract_discount_summary(site_id, start_date, end_date) + + # 从groupbuy_redemption获取团购实付金额 + groupbuy_payments = self._extract_groupbuy_payments(site_id, start_date, end_date) + + # 提取大客户优惠（拆分手动调整） + big_customer_summary = self._extract_big_customer_discounts(site_id, start_date, end_date) + + return { + 'discount_summary': discount_summary, + 'groupbuy_payments': groupbuy_payments, + 'big_customer_summary': big_customer_summary, + } + + def _extract_discount_summary( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 从结账单头表抽取优惠汇总 + + 字段说明： + - coupon_amount: 团购抵消台费金额 + - adjust_amount: 手动调整金额（台费打折） + - member_discount_amount: 会员折扣 + - rounding_amount: 抹零金额 + - gift_card_amount: 赠送卡支付 + - pl_coupon_sale_amount: 平台券销售金额（团购实付路径1） + """ + sql = """ + SELECT + pay_time::DATE AS stat_date, + -- 团购相关 + COALESCE(SUM(coupon_amount), 0) AS coupon_amount_total, + COALESCE(SUM(pl_coupon_sale_amount), 0) AS pl_coupon_sale_total, + COUNT(CASE WHEN coupon_amount > 0 THEN 1 END) AS coupon_order_count, + -- 手动调整 + COALESCE(SUM(adjust_amount), 0) AS adjust_amount_total, + COUNT(CASE WHEN adjust_amount != 0 THEN 1 END) AS adjust_order_count, + -- 会员折扣 + COALESCE(SUM(member_discount_amount), 0) AS member_discount_total, + COUNT(CASE WHEN member_discount_amount > 0 THEN 1 END) AS member_discount_order_count, + -- 抹零 + COALESCE(SUM(rounding_amount), 0) AS rounding_amount_total, + COUNT(CASE WHEN rounding_amount != 0 THEN 1 END) AS rounding_order_count, + -- 赠送卡 + COALESCE(SUM(gift_card_amount), 0) AS gift_card_amount_total, + COUNT(CASE WHEN gift_card_amount > 0 THEN 1 END) AS gift_card_order_count, + -- 总订单数 + COUNT(*) AS total_orders + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %(site_id)s + AND pay_time >= %(start_date)s + AND pay_time < %(end_date)s + INTERVAL '1 day' + AND settle_status = 1 -- 已结账 + GROUP BY pay_time::DATE + ORDER BY stat_date + """ + rows = self.db.query(sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + return [dict(row) for row in rows] if rows else [] + + def _extract_groupbuy_payments( + self, + site_id: int, + start_date: date, + end_date: date + ) -> Dict[date, Decimal]: + """ + 从团购核销表获取团购实付金额 + + 团购实付金额计算： + - 若 pl_coupon_sale_amount > 0，使用该值 + - 否则使用 groupbuy_redemption.ledger_unit_price + + 返回：{日期: 团购实付总额} + """ + sql = """ + SELECT + sh.pay_time::DATE AS stat_date, + SUM( + CASE + WHEN sh.pl_coupon_sale_amount > 0 THEN sh.pl_coupon_sale_amount + ELSE COALESCE(gr.ledger_unit_price, 0) + END + ) AS groupbuy_payment + FROM billiards_dwd.dwd_settlement_head sh + LEFT JOIN billiards_dwd.dwd_groupbuy_redemption gr + ON gr.order_settle_id = sh.order_settle_id + WHERE sh.site_id = %(site_id)s + AND sh.pay_time >= %(start_date)s + AND sh.pay_time < %(end_date)s + INTERVAL '1 day' + AND sh.settle_status = 1 + AND sh.coupon_amount > 0 -- 只统计有团购的订单 + GROUP BY sh.pay_time::DATE + """ + rows = self.db.query(sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + + result = {} + if rows: + for row in rows: + result[row['stat_date']] = self.safe_decimal(row.get('groupbuy_payment', 0)) + return result + + def transform(self, data: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据 + + 将抽取的数据转换为目标表格式： + - 每种优惠类型一条记录 + - 计算团购优惠（coupon_amount - 团购实付） + - 计算优惠占比 + """ + site_id = context.store_id + tenant_id = self.config.get("app.tenant_id", site_id) + + discount_summary = data.get('discount_summary', []) + groupbuy_payments = data.get('groupbuy_payments', {}) + big_customer_summary = {r['stat_date']: r for r in data.get('big_customer_summary', [])} + + records = [] + + # 优惠类型定义 + # (type_code, type_name, amount_field, count_field, special_calc) + discount_types = [ + ('GROUPBUY', '团购优惠', 'coupon_amount_total', 'coupon_order_count', True), + ('VIP', '会员折扣', 'member_discount_total', 'member_discount_order_count', False), + ('ROUNDING', '抹零', 'rounding_amount_total', 'rounding_order_count', False), + ('GIFT_CARD', '赠送卡抵扣', 'gift_card_amount_total', 'gift_card_order_count', False), + ] + + for daily_data in discount_summary: + stat_date = daily_data.get('stat_date') + + # 计算各类优惠金额 + daily_discounts = {} + total_discount = Decimal('0') + + for type_code, type_name, amount_field, count_field, special_calc in discount_types: + if special_calc and type_code == 'GROUPBUY': + # 团购优惠 = 团购抵消台费 - 团购实付 + coupon_amount = self.safe_decimal(daily_data.get(amount_field, 0)) + groupbuy_paid = groupbuy_payments.get(stat_date, Decimal('0')) + discount_amount = coupon_amount - groupbuy_paid + # 确保优惠金额为正数 + discount_amount = max(discount_amount, Decimal('0')) + else: + discount_amount = abs(self.safe_decimal(daily_data.get(amount_field, 0))) + + usage_count = daily_data.get(count_field, 0) or 0 + + daily_discounts[type_code] = { + 'type_name': type_name, + 'amount': discount_amount, + 'count': usage_count, + } + total_discount += discount_amount + + # 拆分手动调整为大客户/其他 + adjust_amount = abs(self.safe_decimal(daily_data.get('adjust_amount_total', 0))) + adjust_count = daily_data.get('adjust_order_count', 0) or 0 + big_customer_info = big_customer_summary.get(stat_date, {}) + big_customer_amount = self.safe_decimal(big_customer_info.get('big_customer_amount', 0)) + big_customer_count = big_customer_info.get('big_customer_count', 0) or 0 + other_amount = adjust_amount - big_customer_amount + if other_amount < 0: + other_amount = Decimal('0') + other_count = adjust_count - big_customer_count + if other_count < 0: + other_count = 0 + + daily_discounts['BIG_CUSTOMER'] = { + 'type_name': '大客户优惠', + 'amount': big_customer_amount, + 'count': big_customer_count, + } + daily_discounts['OTHER'] = { + 'type_name': '其他优惠', + 'amount': other_amount, + 'count': other_count, + } + total_discount += big_customer_amount + other_amount + + # 为每种优惠类型生成记录 + for type_code, discount_info in daily_discounts.items(): + discount_amount = discount_info['amount'] + usage_count = discount_info['count'] + + # 计算占比（避免除零） + discount_ratio = (discount_amount / total_discount) if total_discount > 0 else Decimal('0') + + records.append({ + 'site_id': site_id, + 'tenant_id': tenant_id, + 'stat_date': stat_date, + 'discount_type_code': type_code, + 'discount_type_name': discount_info['type_name'], + 'discount_amount': discount_amount, + 'discount_ratio': round(discount_ratio, 4), + 'usage_count': usage_count, + 'affected_orders': usage_count, # 简化：使用次数=影响订单数 + }) + + return records + + def _extract_big_customer_discounts( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取大客户优惠（基于手动调整） + """ + member_ids = self._parse_id_list(self.config.get("dws.discount.big_customer_member_ids")) + order_ids = self._parse_id_list(self.config.get("dws.discount.big_customer_order_ids")) + if not member_ids and not order_ids: + return [] + + sql = """ + SELECT + pay_time::DATE AS stat_date, + order_settle_id, + member_id, + adjust_amount + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %(site_id)s + AND pay_time >= %(start_date)s + AND pay_time < %(end_date)s + INTERVAL '1 day' + AND adjust_amount != 0 + """ + rows = self.db.query(sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + if not rows: + return [] + + result: Dict[date, Dict[str, Any]] = {} + for row in rows: + row_dict = dict(row) + stat_date = row_dict.get('stat_date') + if not stat_date: + continue + order_id = row_dict.get('order_settle_id') + member_id = row_dict.get('member_id') + if order_id not in order_ids and member_id not in member_ids: + continue + amount = abs(self.safe_decimal(row_dict.get('adjust_amount', 0))) + entry = result.setdefault(stat_date, {'stat_date': stat_date, 'big_customer_amount': Decimal('0'), 'big_customer_count': 0}) + entry['big_customer_amount'] += amount + entry['big_customer_count'] += 1 + + return list(result.values()) + + def _parse_id_list(self, value: Any) -> set: + if not value: + return set() + if isinstance(value, str): + items = [v.strip() for v in value.split(",") if v.strip()] + return {int(v) for v in items if v.isdigit()} + if isinstance(value, (list, tuple, set)): + result = set() + for item in value: + if item is None: + continue + try: + result.add(int(item)) + except (ValueError, TypeError): + continue + return result + return set() + + def load(self, records: List[Dict[str, Any]], context: TaskContext) -> Dict[str, Any]: + """ + 加载数据到目标表 + + 使用幂等方式：delete-before-insert（按日期范围） + """ + if not records: + return {'inserted': 0, 'deleted': 0} + + site_id = context.store_id + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + + # 删除窗口内的旧数据 + delete_sql = """ + DELETE FROM billiards_dws.dws_finance_discount_detail + WHERE site_id = %(site_id)s + AND stat_date >= %(start_date)s + AND stat_date <= %(end_date)s + """ + deleted = self.db.execute(delete_sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + + # 批量插入新数据 + insert_sql = """ + INSERT INTO billiards_dws.dws_finance_discount_detail ( + site_id, tenant_id, stat_date, + discount_type_code, discount_type_name, + discount_amount, discount_ratio, + usage_count, affected_orders, + created_at, updated_at + ) VALUES ( + %(site_id)s, %(tenant_id)s, %(stat_date)s, + %(discount_type_code)s, %(discount_type_name)s, + %(discount_amount)s, %(discount_ratio)s, + %(usage_count)s, %(affected_orders)s, + NOW(), NOW() + ) + """ + + inserted = 0 + for record in records: + self.db.execute(insert_sql, record) + inserted += 1 + + return { + 'deleted': deleted or 0, + 'inserted': inserted, + } diff --git a/etl_billiards/tasks/dws/finance_income_task.py b/etl_billiards/tasks/dws/finance_income_task.py new file mode 100644 index 0000000..a2f40b8 --- /dev/null +++ b/etl_billiards/tasks/dws/finance_income_task.py @@ -0,0 +1,437 @@ +# -*- coding: utf-8 -*- +""" +收入结构分析任务 + +功能说明： + 以"日期+区域/类型"为粒度，分析收入结构 + +数据来源： + - dwd_settlement_head: 结账单头表（台费、商品、助教正价） + - dwd_table_fee_log: 台费流水（区域关联） + - dwd_assistant_service_log: 助教服务流水（区域关联） + - cfg_area_category: 区域分类映射 + +目标表： + billiards_dws.dws_finance_income_structure + +更新策略： + - 更新频率：每日更新 + - 幂等方式：delete-before-insert（按日期+类型） + +业务规则： + - 结构类型1（INCOME_TYPE）：按收入类型分析（台费/商品/助教基础课/助教附加课） + - 结构类型2（AREA）：按区域分析（普通台球区/VIP包厢/斯诺克/麻将/KTV等） + - 区域映射使用cfg_area_category配置 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class FinanceIncomeStructureTask(BaseDwsTask): + """ + 收入结构分析任务 + + 分析收入的两种维度： + 1. INCOME_TYPE: 按收入类型（台费/商品/助教基础课/助教附加课） + 2. AREA: 按区域（使用cfg_area_category映射） + """ + + def get_task_code(self) -> str: + return "DWS_FINANCE_INCOME_STRUCTURE" + + def get_target_table(self) -> str: + return "dws_finance_income_structure" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "stat_date", "structure_type", "category_code"] + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 抽取数据 + + 分两条路径抽取： + 1. 按收入类型汇总（来自settlement_head） + 2. 按区域汇总（来自table_fee_log和assistant_service_log） + """ + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + # 按收入类型汇总 + income_by_type = self._extract_income_by_type(site_id, start_date, end_date) + + # 按区域汇总 + income_by_area = self._extract_income_by_area(site_id, start_date, end_date) + + return { + 'income_by_type': income_by_type, + 'income_by_area': income_by_area, + } + + def _extract_income_by_type( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 按收入类型汇总 + + 收入类型分类： + - TABLE_FEE: 台费收入 (table_charge_money) + - GOODS: 商品收入 (goods_money) + - ASSISTANT_BASE: 助教基础课 (assistant_pd_money) + - ASSISTANT_BONUS: 助教附加课 (assistant_cx_money) + """ + sql = """ + SELECT + pay_time::DATE AS stat_date, + -- 台费收入 + COALESCE(SUM(table_charge_money), 0) AS table_fee_income, + COUNT(CASE WHEN table_charge_money > 0 THEN 1 END) AS table_fee_orders, + -- 商品收入 + COALESCE(SUM(goods_money), 0) AS goods_income, + COUNT(CASE WHEN goods_money > 0 THEN 1 END) AS goods_orders, + -- 助教基础课收入（PD=陪打） + COALESCE(SUM(assistant_pd_money), 0) AS assistant_base_income, + COUNT(CASE WHEN assistant_pd_money > 0 THEN 1 END) AS assistant_base_orders, + -- 助教附加课收入（CX=超休/促销） + COALESCE(SUM(assistant_cx_money), 0) AS assistant_bonus_income, + COUNT(CASE WHEN assistant_cx_money > 0 THEN 1 END) AS assistant_bonus_orders, + -- 总订单数 + COUNT(*) AS total_orders + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %(site_id)s + AND pay_time >= %(start_date)s + AND pay_time < %(end_date)s + INTERVAL '1 day' + AND settle_status = 1 -- 已结账 + GROUP BY pay_time::DATE + ORDER BY stat_date + """ + rows = self.db.query(sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + return [dict(row) for row in rows] if rows else [] + + def _extract_income_by_area( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 按区域汇总收入 + + 关联dim_table获取区域名称，再映射到cfg_area_category + """ + sql = """ + WITH + -- 台费按区域汇总 + table_fee_by_area AS ( + SELECT + tfl.pay_time::DATE AS stat_date, + dt.site_table_area_name AS area_name, + COALESCE(SUM(tfl.ledger_amount), 0) AS income_amount, + COALESCE(SUM(tfl.ledger_time_seconds), 0) AS duration_seconds, + COUNT(DISTINCT tfl.order_settle_id) AS order_count + FROM billiards_dwd.dwd_table_fee_log tfl + LEFT JOIN billiards_dwd.dim_table dt + ON dt.site_table_id = tfl.site_table_id + WHERE tfl.site_id = %(site_id)s + AND tfl.pay_time >= %(start_date)s + AND tfl.pay_time < %(end_date)s + INTERVAL '1 day' + GROUP BY tfl.pay_time::DATE, dt.site_table_area_name + ), + -- 助教服务按区域汇总 + assistant_by_area AS ( + SELECT + asl.start_use_time::DATE AS stat_date, + dt.site_table_area_name AS area_name, + COALESCE(SUM(asl.ledger_amount), 0) AS income_amount, + COALESCE(SUM(asl.income_seconds), 0) AS duration_seconds, + COUNT(DISTINCT asl.order_settle_id) AS order_count + FROM billiards_dwd.dwd_assistant_service_log asl + LEFT JOIN billiards_dwd.dim_table dt + ON dt.site_table_id = asl.site_table_id + WHERE asl.site_id = %(site_id)s + AND asl.start_use_time >= %(start_date)s + AND asl.start_use_time < %(end_date)s + INTERVAL '1 day' + GROUP BY asl.start_use_time::DATE, dt.site_table_area_name + ) + -- 合并台费和助教服务 + SELECT + COALESCE(t.stat_date, a.stat_date) AS stat_date, + COALESCE(t.area_name, a.area_name) AS area_name, + COALESCE(t.income_amount, 0) + COALESCE(a.income_amount, 0) AS income_amount, + COALESCE(t.duration_seconds, 0) + COALESCE(a.duration_seconds, 0) AS duration_seconds, + GREATEST(COALESCE(t.order_count, 0), COALESCE(a.order_count, 0)) AS order_count + FROM table_fee_by_area t + FULL OUTER JOIN assistant_by_area a + ON t.stat_date = a.stat_date AND t.area_name = a.area_name + ORDER BY stat_date, area_name + """ + rows = self.db.query(sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + return [dict(row) for row in rows] if rows else [] + + def transform(self, data: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据 + + 将抽取的数据转换为目标表格式： + 1. 按收入类型展开（每种类型一条记录） + 2. 按区域展开（每个区域一条记录） + 3. 计算占比 + """ + site_id = context.store_id + tenant_id = self.config.get("app.tenant_id", site_id) + + records = [] + + # 处理按收入类型的数据 + income_type_records = self._transform_income_by_type( + data.get('income_by_type', []), + site_id, + tenant_id + ) + records.extend(income_type_records) + + # 处理按区域的数据 + area_records = self._transform_income_by_area( + data.get('income_by_area', []), + site_id, + tenant_id + ) + records.extend(area_records) + + return records + + def _transform_income_by_type( + self, + income_data: List[Dict[str, Any]], + site_id: int, + tenant_id: int + ) -> List[Dict[str, Any]]: + """ + 转换按收入类型的数据 + + 将每日汇总数据展开为4条记录（台费/商品/基础课/附加课） + """ + # 收入类型定义 + income_types = [ + ('TABLE_FEE', '台费收入', 'table_fee_income', 'table_fee_orders'), + ('GOODS', '商品收入', 'goods_income', 'goods_orders'), + ('ASSISTANT_BASE', '助教基础课', 'assistant_base_income', 'assistant_base_orders'), + ('ASSISTANT_BONUS', '助教附加课', 'assistant_bonus_income', 'assistant_bonus_orders'), + ] + + records = [] + + for daily_data in income_data: + stat_date = daily_data.get('stat_date') + + # 计算当日总收入（用于计算占比） + total_income = sum( + self.safe_decimal(daily_data.get(field, 0)) + for _, _, field, _ in income_types + ) + + # 为每种收入类型生成一条记录 + for type_code, type_name, income_field, order_field in income_types: + income_amount = self.safe_decimal(daily_data.get(income_field, 0)) + order_count = daily_data.get(order_field, 0) or 0 + + # 计算占比（避免除零） + income_ratio = (income_amount / total_income) if total_income > 0 else Decimal('0') + + records.append({ + 'site_id': site_id, + 'tenant_id': tenant_id, + 'stat_date': stat_date, + 'structure_type': 'INCOME_TYPE', + 'category_code': type_code, + 'category_name': type_name, + 'income_amount': income_amount, + 'income_ratio': round(income_ratio, 4), + 'order_count': order_count, + 'duration_minutes': 0, # 收入类型维度不统计时长 + }) + + return records + + def _transform_income_by_area( + self, + area_data: List[Dict[str, Any]], + site_id: int, + tenant_id: int + ) -> List[Dict[str, Any]]: + """ + 转换按区域的数据 + + 将区域名称映射到cfg_area_category的category_code + """ + records = [] + + # 加载区域分类配置 + area_categories = self._get_config_cache().get('area_categories', {}) + + # 按日期分组计算总收入（用于计算占比） + daily_totals = {} + for row in area_data: + stat_date = row.get('stat_date') + income = self.safe_decimal(row.get('income_amount', 0)) + daily_totals[stat_date] = daily_totals.get(stat_date, Decimal('0')) + income + + # 按日期+区域聚合（相同category_code需要合并） + aggregated = {} + + for row in area_data: + stat_date = row.get('stat_date') + area_name = row.get('area_name') or '未知区域' + income_amount = self.safe_decimal(row.get('income_amount', 0)) + duration_seconds = row.get('duration_seconds', 0) or 0 + order_count = row.get('order_count', 0) or 0 + + # 映射区域名称到分类代码 + category = self._map_area_to_category(area_name, area_categories) + category_code = category.get('category_code', 'OTHER') + category_name = category.get('category_name', '其他区域') + + # 聚合键 + key = (stat_date, category_code) + + if key not in aggregated: + aggregated[key] = { + 'stat_date': stat_date, + 'category_code': category_code, + 'category_name': category_name, + 'income_amount': Decimal('0'), + 'duration_seconds': 0, + 'order_count': 0, + } + + aggregated[key]['income_amount'] += income_amount + aggregated[key]['duration_seconds'] += duration_seconds + aggregated[key]['order_count'] += order_count + + # 生成记录 + for key, agg_data in aggregated.items(): + stat_date = agg_data['stat_date'] + total_income = daily_totals.get(stat_date, Decimal('1')) + income_amount = agg_data['income_amount'] + + # 计算占比 + income_ratio = (income_amount / total_income) if total_income > 0 else Decimal('0') + + records.append({ + 'site_id': site_id, + 'tenant_id': tenant_id, + 'stat_date': stat_date, + 'structure_type': 'AREA', + 'category_code': agg_data['category_code'], + 'category_name': agg_data['category_name'], + 'income_amount': income_amount, + 'income_ratio': round(income_ratio, 4), + 'order_count': agg_data['order_count'], + 'duration_minutes': agg_data['duration_seconds'] // 60, + }) + + return records + + def _map_area_to_category( + self, + area_name: str, + area_categories: Dict[str, Dict[str, Any]] + ) -> Dict[str, Any]: + """ + 将区域名称映射到分类 + + 匹配规则： + 1. 精确匹配 match_pattern + 2. 模糊匹配（LIKE） + 3. 默认返回 OTHER + """ + if not area_name: + return {'category_code': 'OTHER', 'category_name': '其他区域'} + + # 遍历配置查找匹配 + for pattern, category in area_categories.items(): + match_type = category.get('match_type', 'exact') + + if match_type == 'exact': + if area_name == pattern: + return category + elif match_type == 'like': + # 简单的模糊匹配（包含关系） + if pattern.replace('%', '') in area_name: + return category + + # 默认分类 + return {'category_code': 'OTHER', 'category_name': '其他区域'} + + def load(self, records: List[Dict[str, Any]], context: TaskContext) -> Dict[str, Any]: + """ + 加载数据到目标表 + + 使用幂等方式：delete-before-insert（按日期范围） + """ + if not records: + return {'inserted': 0, 'deleted': 0} + + site_id = context.store_id + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + + # 删除窗口内的旧数据 + delete_sql = """ + DELETE FROM billiards_dws.dws_finance_income_structure + WHERE site_id = %(site_id)s + AND stat_date >= %(start_date)s + AND stat_date <= %(end_date)s + """ + deleted = self.db.execute(delete_sql, { + 'site_id': site_id, + 'start_date': start_date, + 'end_date': end_date, + }) + + # 批量插入新数据 + insert_sql = """ + INSERT INTO billiards_dws.dws_finance_income_structure ( + site_id, tenant_id, stat_date, + structure_type, category_code, category_name, + income_amount, income_ratio, + order_count, duration_minutes, + created_at, updated_at + ) VALUES ( + %(site_id)s, %(tenant_id)s, %(stat_date)s, + %(structure_type)s, %(category_code)s, %(category_name)s, + %(income_amount)s, %(income_ratio)s, + %(order_count)s, %(duration_minutes)s, + NOW(), NOW() + ) + """ + + inserted = 0 + for record in records: + self.db.execute(insert_sql, record) + inserted += 1 + + return { + 'deleted': deleted or 0, + 'inserted': inserted, + } diff --git a/etl_billiards/tasks/dws/finance_recharge_task.py b/etl_billiards/tasks/dws/finance_recharge_task.py new file mode 100644 index 0000000..55649ab --- /dev/null +++ b/etl_billiards/tasks/dws/finance_recharge_task.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +""" +充值统计任务 + +功能说明： + 以"日期"为粒度，统计充值数据 + +数据来源： + - dwd_recharge_order: 充值订单 + - dim_member_card_account: 会员卡账户（余额快照） + +目标表： + billiards_dws.dws_finance_recharge_summary + +更新策略： + - 更新频率：每日更新 + - 幂等方式：delete-before-insert（按日期） + +业务规则： + - 首充/续充：通过 is_first 字段区分 + - 现金/赠送：通过 pay_money/gift_money 区分 + - 卡余额：区分储值卡和赠送卡 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class FinanceRechargeTask(BaseDwsTask): + """ + 充值统计任务 + """ + + def get_task_code(self) -> str: + return "DWS_FINANCE_RECHARGE" + + def get_target_table(self) -> str: + return "dws_finance_recharge_summary" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "stat_date"] + + def extract(self, context: TaskContext) -> Dict[str, Any]: + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + recharge_summary = self._extract_recharge_summary(site_id, start_date, end_date) + card_balances = self._extract_card_balances(site_id, end_date) + + return { + 'recharge_summary': recharge_summary, + 'card_balances': card_balances, + 'start_date': start_date, + 'end_date': end_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + recharge_summary = extracted['recharge_summary'] + card_balances = extracted['card_balances'] + site_id = extracted['site_id'] + + results = [] + for recharge in recharge_summary: + stat_date = recharge.get('stat_date') + + # 获取当日卡余额快照 + balance = card_balances if stat_date == extracted['end_date'] else {} + + record = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'stat_date': stat_date, + 'recharge_count': self.safe_int(recharge.get('recharge_count', 0)), + 'recharge_total': self.safe_decimal(recharge.get('recharge_total', 0)), + 'recharge_cash': self.safe_decimal(recharge.get('recharge_cash', 0)), + 'recharge_gift': self.safe_decimal(recharge.get('recharge_gift', 0)), + 'first_recharge_count': self.safe_int(recharge.get('first_recharge_count', 0)), + 'first_recharge_cash': self.safe_decimal(recharge.get('first_recharge_cash', 0)), + 'first_recharge_gift': self.safe_decimal(recharge.get('first_recharge_gift', 0)), + 'first_recharge_total': self.safe_decimal(recharge.get('first_recharge_total', 0)), + 'renewal_count': self.safe_int(recharge.get('renewal_count', 0)), + 'renewal_cash': self.safe_decimal(recharge.get('renewal_cash', 0)), + 'renewal_gift': self.safe_decimal(recharge.get('renewal_gift', 0)), + 'renewal_total': self.safe_decimal(recharge.get('renewal_total', 0)), + 'recharge_member_count': self.safe_int(recharge.get('recharge_member_count', 0)), + 'new_member_count': self.safe_int(recharge.get('new_member_count', 0)), + 'total_card_balance': self.safe_decimal(balance.get('total_balance', 0)), + 'cash_card_balance': self.safe_decimal(balance.get('cash_balance', 0)), + 'gift_card_balance': self.safe_decimal(balance.get('gift_balance', 0)), + } + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + if not transformed: + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + deleted = self.delete_existing_data(context, date_col="stat_date") + inserted = self.bulk_insert(transformed) + + return { + "counts": {"fetched": len(transformed), "inserted": inserted, "updated": 0, "skipped": 0, "errors": 0}, + "extra": {"deleted": deleted} + } + + def _extract_recharge_summary(self, site_id: int, start_date: date, end_date: date) -> List[Dict[str, Any]]: + sql = """ + SELECT + DATE(create_time) AS stat_date, + COUNT(*) AS recharge_count, + SUM(pay_money + gift_money) AS recharge_total, + SUM(pay_money) AS recharge_cash, + SUM(gift_money) AS recharge_gift, + COUNT(CASE WHEN is_first = 1 THEN 1 END) AS first_recharge_count, + SUM(CASE WHEN is_first = 1 THEN pay_money ELSE 0 END) AS first_recharge_cash, + SUM(CASE WHEN is_first = 1 THEN gift_money ELSE 0 END) AS first_recharge_gift, + SUM(CASE WHEN is_first = 1 THEN pay_money + gift_money ELSE 0 END) AS first_recharge_total, + COUNT(CASE WHEN is_first != 1 OR is_first IS NULL THEN 1 END) AS renewal_count, + SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN pay_money ELSE 0 END) AS renewal_cash, + SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN gift_money ELSE 0 END) AS renewal_gift, + SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN pay_money + gift_money ELSE 0 END) AS renewal_total, + COUNT(DISTINCT member_id) AS recharge_member_count, + COUNT(DISTINCT CASE WHEN is_first = 1 THEN member_id END) AS new_member_count + FROM billiards_dwd.dwd_recharge_order + WHERE site_id = %s AND DATE(create_time) >= %s AND DATE(create_time) <= %s + GROUP BY DATE(create_time) + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_card_balances(self, site_id: int, stat_date: date) -> Dict[str, Decimal]: + CASH_CARD_TYPE_ID = 2793249295533893 + GIFT_CARD_TYPE_IDS = [2791990152417157, 2793266846533445, 2794699703437125] + + sql = """ + SELECT card_type_id, SUM(balance) AS total_balance + FROM billiards_dwd.dim_member_card_account + WHERE site_id = %s AND valid_to IS NULL + GROUP BY card_type_id + """ + rows = self.db.query(sql, (site_id,)) + + cash_balance = Decimal('0') + gift_balance = Decimal('0') + + for row in (rows or []): + card_type_id = row['card_type_id'] + balance = self.safe_decimal(row['total_balance']) + if card_type_id == CASH_CARD_TYPE_ID: + cash_balance += balance + elif card_type_id in GIFT_CARD_TYPE_IDS: + gift_balance += balance + + return { + 'cash_balance': cash_balance, + 'gift_balance': gift_balance, + 'total_balance': cash_balance + gift_balance + } + + +__all__ = ['FinanceRechargeTask'] diff --git a/etl_billiards/tasks/dws/index/__init__.py b/etl_billiards/tasks/dws/index/__init__.py new file mode 100644 index 0000000..6c19368 --- /dev/null +++ b/etl_billiards/tasks/dws/index/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +""" +指数算法任务模块 + +包含： +- RecallIndexTask: 客户召回指数计算任务 +- IntimacyIndexTask: 客户-助教亲密指数计算任务 +""" + +from .recall_index_task import RecallIndexTask +from .intimacy_index_task import IntimacyIndexTask + +__all__ = [ + 'RecallIndexTask', + 'IntimacyIndexTask', +] diff --git a/etl_billiards/tasks/dws/index/base_index_task.py b/etl_billiards/tasks/dws/index/base_index_task.py new file mode 100644 index 0000000..25f190e --- /dev/null +++ b/etl_billiards/tasks/dws/index/base_index_task.py @@ -0,0 +1,518 @@ +# -*- coding: utf-8 -*- +""" +指数算法任务基类 + +功能说明： + - 提供半衰期时间衰减函数 + - 提供分位数计算和分位截断 + - 提供0-10映射方法 + - 提供算法参数加载 + - 提供分位点历史记录（用于EWMA平滑） + +算法原理： + 1. 时间衰减函数（半衰期模型）：decay(d; h) = exp(-ln(2) * d / h) + 当 d=h 时权重衰减到 0.5，越近权重越大 + + 2. 0-10映射流程： + Raw Score → Winsorize(P5, P95) → [可选Log压缩] → MinMax(0, 10) + +作者：ETL团队 +创建日期：2026-02-03 +""" + +from __future__ import annotations + +import math +from abc import abstractmethod +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from ..base_dws_task import BaseDwsTask, TaskContext + + +# ============================================================================= +# 数据类定义 +# ============================================================================= + +@dataclass +class IndexParameters: + """指数算法参数数据类""" + params: Dict[str, float] + loaded_at: datetime + + +@dataclass +class PercentileHistory: + """分位点历史记录""" + percentile_5: float + percentile_95: float + percentile_5_smoothed: float + percentile_95_smoothed: float + record_count: int + calc_time: datetime + + +# ============================================================================= +# 指数任务基类 +# ============================================================================= + +class BaseIndexTask(BaseDwsTask): + """ + 指数算法任务基类 + + 提供指数计算通用功能： + 1. 半衰期时间衰减函数 + 2. 分位数计算与截断 + 3. 0-10归一化映射 + 4. 算法参数加载 + 5. 分位点历史管理（EWMA平滑） + """ + + # 子类需要定义的指数类型 + INDEX_TYPE: str = "" + + # 参数缓存 + _index_params_cache: Optional[IndexParameters] = None + _index_params_ttl: int = 300 # 缓存有效期（秒） + + # 默认参数 + DEFAULT_LOOKBACK_DAYS = 60 + DEFAULT_PERCENTILE_LOWER = 5 + DEFAULT_PERCENTILE_UPPER = 95 + DEFAULT_EWMA_ALPHA = 0.2 + + # ========================================================================== + # 抽象方法（子类需实现） + # ========================================================================== + + @abstractmethod + def get_index_type(self) -> str: + """获取指数类型（RECALL/INTIMACY）""" + raise NotImplementedError + + # ========================================================================== + # 时间衰减函数 + # ========================================================================== + + def decay(self, days: float, halflife: float) -> float: + """ + 半衰期衰减函数 + + 公式: decay(d; h) = exp(-ln(2) * d / h) + + 解释：当 d=h 时权重衰减到 0.5；越近权重越大，符合"近期更重要"的直觉 + + Args: + days: 事件距今天数 (d >= 0) + halflife: 半衰期 (h > 0)，单位：天 + + Returns: + 衰减后的权重，范围 (0, 1] + + Examples: + >>> decay(0, 7) # 今天，权重=1.0 + 1.0 + >>> decay(7, 7) # 7天前，半衰期=7，权重=0.5 + 0.5 + >>> decay(14, 7) # 14天前，权重=0.25 + 0.25 + """ + if halflife <= 0: + raise ValueError("半衰期必须大于0") + if days < 0: + days = 0 + return math.exp(-math.log(2) * days / halflife) + + # ========================================================================== + # 分位数计算 + # ========================================================================== + + def calculate_percentiles( + self, + scores: List[float], + lower: int = 5, + upper: int = 95 + ) -> Tuple[float, float]: + """ + 计算分位点 + + Args: + scores: 分数列表 + lower: 下分位点百分比（默认5） + upper: 上分位点百分比（默认95） + + Returns: + (下分位值, 上分位值) 元组 + """ + if not scores: + return 0.0, 0.0 + + sorted_scores = sorted(scores) + n = len(sorted_scores) + + # 计算分位点索引 + lower_idx = max(0, int(n * lower / 100) - 1) + upper_idx = min(n - 1, int(n * upper / 100)) + + return sorted_scores[lower_idx], sorted_scores[upper_idx] + + def winsorize(self, value: float, lower: float, upper: float) -> float: + """ + 分位截断（Winsorize） + + 将值限制在 [lower, upper] 范围内 + + Args: + value: 原始值 + lower: 下限（P5分位） + upper: 上限（P95分位） + + Returns: + 截断后的值 + """ + return min(max(value, lower), upper) + + # ========================================================================== + # 0-10映射 + # ========================================================================== + + def normalize_to_display( + self, + value: float, + min_val: float, + max_val: float, + use_log: bool = False, + epsilon: float = 1e-6 + ) -> float: + """ + 归一化到0-10分 + + 映射流程： + 1. [可选] 对数压缩：y = ln(1 + x) + 2. MinMax映射：score = 10 * (y - min) / (max - min) + + Args: + value: 原始值（已Winsorize） + min_val: 最小值（通常为P5） + max_val: 最大值（通常为P95） + use_log: 是否使用对数压缩（亲密指数建议启用） + epsilon: 防除零小量 + + Returns: + 0-10范围的分数 + """ + if use_log: + value = math.log1p(value) + min_val = math.log1p(min_val) + max_val = math.log1p(max_val) + + # 防止分母为0 + range_val = max_val - min_val + if range_val < epsilon: + return 5.0 # 几乎全员相同时返回中间值 + + score = 10.0 * (value - min_val) / range_val + + # 确保在0-10范围内 + return max(0.0, min(10.0, score)) + + def batch_normalize_to_display( + self, + raw_scores: List[Tuple[Any, float]], # [(entity_id, raw_score), ...] + use_log: bool = False, + percentile_lower: int = 5, + percentile_upper: int = 95, + use_smoothing: bool = False, + site_id: Optional[int] = None + ) -> List[Tuple[Any, float, float]]: + """ + 批量归一化Raw Score到Display Score + + 流程： + 1. 提取所有raw_score + 2. 计算分位点（可选EWMA平滑） + 3. Winsorize截断 + 4. MinMax映射到0-10 + + Args: + raw_scores: (entity_id, raw_score) 元组列表 + use_log: 是否使用对数压缩 + percentile_lower: 下分位百分比 + percentile_upper: 上分位百分比 + use_smoothing: 是否使用EWMA平滑分位点 + site_id: 门店ID（平滑时需要） + + Returns: + (entity_id, raw_score, display_score) 元组列表 + """ + if not raw_scores: + return [] + + # 提取raw_score + scores = [s for _, s in raw_scores] + + # 计算分位点 + q_l, q_u = self.calculate_percentiles(scores, percentile_lower, percentile_upper) + + # EWMA平滑 + if use_smoothing and site_id is not None: + q_l, q_u = self._apply_ewma_smoothing(site_id, q_l, q_u) + + # 映射 + results = [] + for entity_id, raw_score in raw_scores: + clipped = self.winsorize(raw_score, q_l, q_u) + display = self.normalize_to_display(clipped, q_l, q_u, use_log) + results.append((entity_id, raw_score, round(display, 2))) + + return results + + # ========================================================================== + # 算法参数加载 + # ========================================================================== + + def load_index_parameters( + self, + index_type: Optional[str] = None, + force_reload: bool = False + ) -> Dict[str, float]: + """ + 加载指数算法参数 + + Args: + index_type: 指数类型（默认使用子类定义的INDEX_TYPE） + force_reload: 是否强制重新加载 + + Returns: + 参数名到参数值的字典 + """ + if index_type is None: + index_type = self.get_index_type() + + now = datetime.now(self.tz) + + # 检查缓存 + if ( + not force_reload + and self._index_params_cache is not None + and (now - self._index_params_cache.loaded_at).total_seconds() < self._index_params_ttl + ): + return self._index_params_cache.params + + self.logger.debug("加载指数算法参数: %s", index_type) + + sql = """ + SELECT param_name, param_value + FROM billiards_dws.cfg_index_parameters + WHERE index_type = %s + AND effective_from <= CURRENT_DATE + AND (effective_to IS NULL OR effective_to >= CURRENT_DATE) + ORDER BY effective_from DESC + """ + + rows = self.db.query(sql, (index_type,)) + + params = {} + seen = set() + for row in (rows or []): + row_dict = dict(row) + name = row_dict['param_name'] + if name not in seen: + params[name] = float(row_dict['param_value']) + seen.add(name) + + self._index_params_cache = IndexParameters( + params=params, + loaded_at=now + ) + + return params + + def get_param(self, name: str, default: float = 0.0) -> float: + """ + 获取单个参数值 + + Args: + name: 参数名 + default: 默认值 + + Returns: + 参数值 + """ + params = self.load_index_parameters() + return params.get(name, default) + + # ========================================================================== + # 分位点历史管理（EWMA平滑） + # ========================================================================== + + def get_last_percentile_history( + self, + site_id: int, + index_type: Optional[str] = None + ) -> Optional[PercentileHistory]: + """ + 获取最近一次分位点历史 + + Args: + site_id: 门店ID + index_type: 指数类型 + + Returns: + PercentileHistory 或 None + """ + if index_type is None: + index_type = self.get_index_type() + + sql = """ + SELECT + percentile_5, percentile_95, + percentile_5_smoothed, percentile_95_smoothed, + record_count, calc_time + FROM billiards_dws.dws_index_percentile_history + WHERE site_id = %s AND index_type = %s + ORDER BY calc_time DESC + LIMIT 1 + """ + + rows = self.db.query(sql, (site_id, index_type)) + + if not rows: + return None + + row = dict(rows[0]) + return PercentileHistory( + percentile_5=float(row['percentile_5'] or 0), + percentile_95=float(row['percentile_95'] or 0), + percentile_5_smoothed=float(row['percentile_5_smoothed'] or 0), + percentile_95_smoothed=float(row['percentile_95_smoothed'] or 0), + record_count=int(row['record_count'] or 0), + calc_time=row['calc_time'] + ) + + def save_percentile_history( + self, + site_id: int, + percentile_5: float, + percentile_95: float, + percentile_5_smoothed: float, + percentile_95_smoothed: float, + record_count: int, + min_raw: float, + max_raw: float, + avg_raw: float, + index_type: Optional[str] = None + ) -> None: + """ + 保存分位点历史 + + Args: + site_id: 门店ID + percentile_5: 原始5分位 + percentile_95: 原始95分位 + percentile_5_smoothed: 平滑后5分位 + percentile_95_smoothed: 平滑后95分位 + record_count: 记录数 + min_raw: 最小Raw Score + max_raw: 最大Raw Score + avg_raw: 平均Raw Score + index_type: 指数类型 + """ + if index_type is None: + index_type = self.get_index_type() + + sql = """ + INSERT INTO billiards_dws.dws_index_percentile_history ( + site_id, index_type, calc_time, + percentile_5, percentile_95, + percentile_5_smoothed, percentile_95_smoothed, + record_count, min_raw_score, max_raw_score, avg_raw_score + ) VALUES (%s, %s, NOW(), %s, %s, %s, %s, %s, %s, %s, %s) + """ + + with self.db.conn.cursor() as cur: + cur.execute(sql, ( + site_id, index_type, + percentile_5, percentile_95, + percentile_5_smoothed, percentile_95_smoothed, + record_count, min_raw, max_raw, avg_raw + )) + self.db.conn.commit() + + def _apply_ewma_smoothing( + self, + site_id: int, + current_p5: float, + current_p95: float, + alpha: Optional[float] = None + ) -> Tuple[float, float]: + """ + 应用EWMA平滑到分位点 + + 公式: Q_t = (1 - α) * Q_{t-1} + α * Q_now + + Args: + site_id: 门店ID + current_p5: 当前5分位 + current_p95: 当前95分位 + alpha: 平滑系数（默认0.2） + + Returns: + (平滑后的P5, 平滑后的P95) + """ + if alpha is None: + alpha = self.get_param('ewma_alpha', self.DEFAULT_EWMA_ALPHA) + + history = self.get_last_percentile_history(site_id) + + if history is None: + # 首次计算，不平滑 + return current_p5, current_p95 + + smoothed_p5 = (1 - alpha) * history.percentile_5_smoothed + alpha * current_p5 + smoothed_p95 = (1 - alpha) * history.percentile_95_smoothed + alpha * current_p95 + + return smoothed_p5, smoothed_p95 + + # ========================================================================== + # 统计工具方法 + # ========================================================================== + + def calculate_median(self, values: List[float]) -> float: + """计算中位数""" + if not values: + return 0.0 + sorted_vals = sorted(values) + n = len(sorted_vals) + mid = n // 2 + if n % 2 == 0: + return (sorted_vals[mid - 1] + sorted_vals[mid]) / 2 + return sorted_vals[mid] + + def calculate_mad(self, values: List[float]) -> float: + """ + 计算MAD（中位绝对偏差） + + MAD = median(|x - median(x)|) + + MAD是比标准差更稳健的离散度度量，不受极端值影响 + """ + if not values: + return 0.0 + median_val = self.calculate_median(values) + deviations = [abs(v - median_val) for v in values] + return self.calculate_median(deviations) + + def safe_log(self, value: float, default: float = 0.0) -> float: + """安全的对数运算""" + if value <= 0: + return default + return math.log(value) + + def safe_ln1p(self, value: float) -> float: + """安全的ln(1+x)运算""" + if value < -1: + return 0.0 + return math.log1p(value) diff --git a/etl_billiards/tasks/dws/index/intimacy_index_task.py b/etl_billiards/tasks/dws/index/intimacy_index_task.py new file mode 100644 index 0000000..db2dfa5 --- /dev/null +++ b/etl_billiards/tasks/dws/index/intimacy_index_task.py @@ -0,0 +1,688 @@ +# -*- coding: utf-8 -*- +""" +客户-助教亲密指数计算任务 + +功能说明： + - 衡量客户与助教的关系强度和近期温度 + - 用于助教约课精力分配和约课成功率预估 + - 附加课权重 = 基础课的1.5倍 + - 检测频率激增并放大权重 + +算法公式： + Raw Score = (w_F × F + w_R × R + w_M × M + w_D × D) × mult + + 其中： + - F = Σ(τ_i × decay(d_i, h_sess)) # 频次强度 + - R = decay(d_last, h_last) # 最近温度 + - M = Σ(ln(1+amt/A0) × decay(d_r, h_pay)) # 归因充值强度 + - D = Σ(sqrt(dur/60) × τ × decay(d, h)) # 时长贡献 + - mult = 1 + γ × burst # 激增放大 + - burst = max(0, ln(1 + (F_short/F_long - 1))) + +特殊逻辑： + - 会话合并：同一客人对同一助教，间隔<4小时算同次服务 + - 充值归因：服务结束后1小时内的充值算做该助教贡献 + +数据来源： + - dwd_assistant_service_log: 服务记录 + - dwd_recharge_order: 充值记录 + +更新频率：每4小时 + +作者：ETL团队 +创建日期：2026-02-03 +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_index_task import BaseIndexTask, PercentileHistory +from ..base_dws_task import TaskContext + + +# ============================================================================= +# 数据类定义 +# ============================================================================= + +@dataclass +class ServiceSession: + """合并后的服务会话""" + session_start: datetime + session_end: datetime + total_duration_minutes: int = 0 + course_weight: float = 1.0 # 1.0=基础课, 1.5=附加课 + is_incentive: bool = False # 是否为附加课 + + +@dataclass +class AttributedRecharge: + """归因充值""" + pay_time: datetime + pay_amount: float + days_ago: float + + +@dataclass +class MemberAssistantIntimacyData: + """客户-助教亲密数据""" + member_id: int + assistant_no: str # 助教工号（字符串，如 "1", "2", "15"） + assistant_nickname: str # 助教昵称 + site_id: int + tenant_id: int + + # 计算输入特征 + session_count: int = 0 + total_duration_minutes: int = 0 + basic_session_count: int = 0 + incentive_session_count: int = 0 + days_since_last_session: Optional[int] = None + attributed_recharge_count: int = 0 + attributed_recharge_amount: float = 0.0 + + # 分项得分 + score_frequency: float = 0.0 + score_recency: float = 0.0 + score_recharge: float = 0.0 + score_duration: float = 0.0 + burst_multiplier: float = 1.0 + + # 最终分数 + raw_score: float = 0.0 + display_score: float = 0.0 + + # 中间数据 + sessions: List[ServiceSession] = field(default_factory=list) + recharges: List[AttributedRecharge] = field(default_factory=list) + + +# ============================================================================= +# 亲密指数任务 +# ============================================================================= + +class IntimacyIndexTask(BaseIndexTask): + """ + 客户-助教亲密指数计算任务 + + 计算流程： + 1. 提取近60天的助教服务记录 + 2. 按(member_id, assistant_id)分组，合并4小时内的服务 + 3. 提取归因充值（服务结束后1小时内） + 4. 计算5项分数（频次、最近、充值、时长、激增） + 5. 汇总Raw Score + 6. 分位截断 + Log压缩 + MinMax映射到0-10 + 7. 写入DWS表 + """ + + INDEX_TYPE = "INTIMACY" + + # 技能ID映射 + SKILL_ID_BASIC = 2790683529513797 # 基础课 + SKILL_ID_INCENTIVE = 2790683529513798 # 附加课/激励课 + SKILL_ID_BOX = 3039912271463941 # 包厢课 + + # 默认参数 + DEFAULT_PARAMS = { + 'lookback_days': 60, + 'halflife_session': 14.0, + 'halflife_last': 10.0, + 'halflife_recharge': 21.0, + 'halflife_short': 7.0, + 'halflife_long': 30.0, + 'amount_base': 500.0, + 'incentive_weight': 1.5, + 'session_merge_hours': 4, + 'recharge_attribute_hours': 1, + 'weight_frequency': 2.0, + 'weight_recency': 1.5, + 'weight_recharge': 2.0, + 'weight_duration': 0.5, + 'burst_gamma': 0.6, + 'percentile_lower': 5, + 'percentile_upper': 95, + } + + # ========================================================================== + # 抽象方法实现 + # ========================================================================== + + def get_task_code(self) -> str: + return "DWS_INTIMACY_INDEX" + + def get_target_table(self) -> str: + return "dws_member_assistant_intimacy" + + def get_primary_keys(self) -> List[str]: + return ['site_id', 'member_id', 'assistant_id'] + + def get_index_type(self) -> str: + return self.INDEX_TYPE + + # ========================================================================== + # 任务执行 + # ========================================================================== + + def execute(self, context: Optional[TaskContext]) -> Dict[str, Any]: + """执行亲密指数计算""" + self.logger.info("开始计算客户-助教亲密指数") + + # 获取门店ID + site_id = self._get_site_id(context) + tenant_id = self._get_tenant_id() + + # 加载参数 + params = self._load_params() + lookback_days = int(params['lookback_days']) + + # 计算基准日期和时间 + now = datetime.now(self.tz) + base_date = now.date() + start_datetime = now - timedelta(days=lookback_days) + + self.logger.info( + "参数: lookback=%d天, h_sess=%.1f, h_last=%.1f, h_pay=%.1f, γ=%.2f", + lookback_days, params['halflife_session'], params['halflife_last'], + params['halflife_recharge'], params['burst_gamma'] + ) + + # 1. 提取服务记录 + raw_services = self._extract_service_records(site_id, start_datetime, now) + self.logger.info("提取到 %d 条原始服务记录", len(raw_services)) + + if not raw_services: + self.logger.warning("没有服务记录，跳过计算") + return {'status': 'skipped', 'reason': 'no_data'} + + # 2. 按(member_id, assistant_id)分组并合并会话 + pair_data = self._group_and_merge_sessions(raw_services, params, now) + self.logger.info("合并为 %d 个客户-助教对", len(pair_data)) + + # 3. 提取归因充值 + self._extract_attributed_recharges(site_id, pair_data, params, now) + + # 4. 计算每个pair的特征和分数 + intimacy_data_list: List[MemberAssistantIntimacyData] = [] + + for key, data in pair_data.items(): + data.site_id = site_id + data.tenant_id = tenant_id + + # 计算分项得分 + self._calculate_component_scores(data, params, now) + + # 汇总Raw Score + base_score = ( + params['weight_frequency'] * data.score_frequency + + params['weight_recency'] * data.score_recency + + params['weight_recharge'] * data.score_recharge + + params['weight_duration'] * data.score_duration + ) + data.raw_score = base_score * data.burst_multiplier + + intimacy_data_list.append(data) + + self.logger.info("计算完成 %d 个pair的Raw Score", len(intimacy_data_list)) + + # 5. 归一化到Display Score（使用对数压缩） + raw_scores = [((d.member_id, d.assistant_no), d.raw_score) for d in intimacy_data_list] + normalized = self.batch_normalize_to_display( + raw_scores, + use_log=True, # 亲密指数建议使用对数压缩 + percentile_lower=int(params['percentile_lower']), + percentile_upper=int(params['percentile_upper']), + use_smoothing=True, + site_id=site_id + ) + + # 更新display_score + score_map = {key: (raw, display) for key, raw, display in normalized} + for data in intimacy_data_list: + key = (data.member_id, data.assistant_no) + if key in score_map: + _, data.display_score = score_map[key] + + # 6. 保存分位点历史 + if intimacy_data_list: + all_raw = [d.raw_score for d in intimacy_data_list] + q_l, q_u = self.calculate_percentiles( + all_raw, + int(params['percentile_lower']), + int(params['percentile_upper']) + ) + smoothed_l, smoothed_u = self._apply_ewma_smoothing(site_id, q_l, q_u) + + self.save_percentile_history( + site_id=site_id, + percentile_5=q_l, + percentile_95=q_u, + percentile_5_smoothed=smoothed_l, + percentile_95_smoothed=smoothed_u, + record_count=len(all_raw), + min_raw=min(all_raw), + max_raw=max(all_raw), + avg_raw=sum(all_raw) / len(all_raw) + ) + + # 7. 写入DWS表 + inserted = self._save_intimacy_data(intimacy_data_list) + + self.logger.info("亲密指数计算完成，写入 %d 条记录", inserted) + + return { + 'status': 'success', + 'pair_count': len(intimacy_data_list), + 'records_inserted': inserted + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_service_records( + self, + site_id: int, + start_datetime: datetime, + end_datetime: datetime + ) -> List[Dict[str, Any]]: + """ + 提取服务记录 + + 注意: 使用 assistant_no (助教工号) 作为助教标识，而不是 site_assistant_id + 因为 site_assistant_id 在数据中是每次服务的唯一ID，不是助教的唯一标识 + + Returns: + [{'member_id', 'assistant_no', 'assistant_nickname', 'start_time', 'end_time', 'duration_minutes', 'skill_id'}, ...] + """ + sql = """ + SELECT + tenant_member_id AS member_id, + assistant_no, + nickname AS assistant_nickname, + start_use_time, + last_use_time, + COALESCE(income_seconds, 0) / 60 AS duration_minutes, + skill_id + FROM billiards_dwd.dwd_assistant_service_log + WHERE site_id = %s + AND tenant_member_id > 0 -- 排除散客 + AND is_delete = 0 + AND assistant_no IS NOT NULL -- 确保有助教工号 + AND last_use_time >= %s + AND last_use_time < %s + ORDER BY tenant_member_id, assistant_no, start_use_time + """ + + rows = self.db.query(sql, (site_id, start_datetime, end_datetime)) + + result = [] + for row in (rows or []): + row_dict = dict(row) + # 使用 assistant_no 作为助教标识 + assistant_no = row_dict['assistant_no'] + if assistant_no: + result.append({ + 'member_id': int(row_dict['member_id']), + 'assistant_no': str(assistant_no), # 助教工号（字符串） + 'assistant_nickname': row_dict['assistant_nickname'] or '', + 'start_time': row_dict['start_use_time'], + 'end_time': row_dict['last_use_time'], + 'duration_minutes': int(row_dict['duration_minutes'] or 0), + 'skill_id': int(row_dict['skill_id'] or 0) + }) + + return result + + def _group_and_merge_sessions( + self, + raw_services: List[Dict[str, Any]], + params: Dict[str, float], + now: datetime + ) -> Dict[Tuple[int, str], MemberAssistantIntimacyData]: + """ + 按(member_id, assistant_no)分组并合并会话 + + 合并逻辑：同一客人对同一助教，间隔<4小时算同次服务 + """ + merge_threshold_hours = int(params['session_merge_hours']) + merge_threshold = timedelta(hours=merge_threshold_hours) + incentive_weight = params['incentive_weight'] + + pair_data: Dict[Tuple[int, str], MemberAssistantIntimacyData] = {} + + # 按pair分组（使用assistant_no） + pair_services: Dict[Tuple[int, str], List[Dict[str, Any]]] = {} + for svc in raw_services: + key = (svc['member_id'], svc['assistant_no']) + if key not in pair_services: + pair_services[key] = [] + pair_services[key].append(svc) + + # 对每个pair合并会话 + for key, services in pair_services.items(): + member_id, assistant_no = key + # 取第一个服务记录的昵称 + assistant_nickname = services[0]['assistant_nickname'] if services else '' + + data = MemberAssistantIntimacyData( + member_id=member_id, + assistant_no=assistant_no, + assistant_nickname=assistant_nickname, + site_id=0, # 稍后填充 + tenant_id=0 + ) + + # 按开始时间排序 + sorted_services = sorted(services, key=lambda x: x['start_time']) + + # 合并会话 + current_session: Optional[ServiceSession] = None + + for svc in sorted_services: + start_time = svc['start_time'] + end_time = svc['end_time'] + duration = svc['duration_minutes'] + skill_id = svc['skill_id'] + + # 判断课型 + is_incentive = (skill_id == self.SKILL_ID_INCENTIVE) + weight = incentive_weight if is_incentive else 1.0 + + if current_session is None: + # 开始新会话 + current_session = ServiceSession( + session_start=start_time, + session_end=end_time, + total_duration_minutes=duration, + course_weight=weight, + is_incentive=is_incentive + ) + elif start_time - current_session.session_end <= merge_threshold: + # 合并到当前会话 + current_session.session_end = max(current_session.session_end, end_time) + current_session.total_duration_minutes += duration + # 同次服务取最高权重 + current_session.course_weight = max(current_session.course_weight, weight) + current_session.is_incentive = current_session.is_incentive or is_incentive + else: + # 保存当前会话，开始新会话 + data.sessions.append(current_session) + current_session = ServiceSession( + session_start=start_time, + session_end=end_time, + total_duration_minutes=duration, + course_weight=weight, + is_incentive=is_incentive + ) + + # 保存最后一个会话 + if current_session is not None: + data.sessions.append(current_session) + + # 统计特征 + data.session_count = len(data.sessions) + data.total_duration_minutes = sum(s.total_duration_minutes for s in data.sessions) + data.basic_session_count = sum(1 for s in data.sessions if not s.is_incentive) + data.incentive_session_count = sum(1 for s in data.sessions if s.is_incentive) + + # 最近一次服务 + if data.sessions: + last_session = max(data.sessions, key=lambda s: s.session_end) + data.days_since_last_session = (now - last_session.session_end).days + + pair_data[key] = data + + return pair_data + + def _extract_attributed_recharges( + self, + site_id: int, + pair_data: Dict[Tuple[int, int], MemberAssistantIntimacyData], + params: Dict[str, float], + now: datetime + ) -> None: + """ + 提取归因充值 + + 归因逻辑：服务结束后1小时内的充值算做该助教贡献 + """ + attribution_hours = int(params['recharge_attribute_hours']) + attribution_window = timedelta(hours=attribution_hours) + + # 获取所有相关会员ID + member_ids = set(key[0] for key in pair_data.keys()) + if not member_ids: + return + + member_ids_str = ','.join(str(m) for m in member_ids) + + # 查询充值记录 + sql = f""" + SELECT + member_id, + pay_time, + pay_amount + FROM billiards_dwd.dwd_recharge_order + WHERE site_id = %s + AND member_id IN ({member_ids_str}) + AND settle_type = 5 -- 充值订单 + AND pay_time >= %s + """ + + lookback_days = int(params['lookback_days']) + start_datetime = now - timedelta(days=lookback_days) + + rows = self.db.query(sql, (site_id, start_datetime)) + + # 为每个充值找到归因助教 + for row in (rows or []): + row_dict = dict(row) + member_id = int(row_dict['member_id']) + pay_time = row_dict['pay_time'] + pay_amount = float(row_dict['pay_amount'] or 0) + + if pay_amount <= 0: + continue + + # 查找该会员在pay_time前1小时内结束服务的助教 + for key, data in pair_data.items(): + if key[0] != member_id: + continue + + for session in data.sessions: + # 服务结束后1小时内的充值 + if (session.session_end <= pay_time and + pay_time - session.session_end <= attribution_window): + # 归因给这个助教 + data.attributed_recharge_count += 1 + data.attributed_recharge_amount += pay_amount + data.recharges.append(AttributedRecharge( + pay_time=pay_time, + pay_amount=pay_amount, + days_ago=(now - pay_time).total_seconds() / 86400 + )) + break # 一笔充值只归因给一个助教 + + # ========================================================================== + # 分数计算方法 + # ========================================================================== + + def _calculate_component_scores( + self, + data: MemberAssistantIntimacyData, + params: Dict[str, float], + now: datetime + ) -> None: + """计算5项分数""" + epsilon = 1e-6 + + h_sess = params['halflife_session'] + h_last = params['halflife_last'] + h_pay = params['halflife_recharge'] + h_short = params['halflife_short'] + h_long = params['halflife_long'] + A0 = params['amount_base'] + gamma = params['burst_gamma'] + + # 1. 频次强度 F = Σ(τ_i × decay(d_i, h_sess)) + F = 0.0 + for session in data.sessions: + days_ago = (now - session.session_end).total_seconds() / 86400 + F += session.course_weight * self.decay(days_ago, h_sess) + data.score_frequency = F + + # 2. 最近温度 R = decay(d_last, h_last) + if data.days_since_last_session is not None: + data.score_recency = self.decay(data.days_since_last_session, h_last) + else: + data.score_recency = 0.0 + + # 3. 归因充值强度 M = Σ(ln(1+amt/A0) × decay(d_r, h_pay)) + M = 0.0 + for recharge in data.recharges: + m_amt = math.log1p(recharge.pay_amount / A0) + M += m_amt * self.decay(recharge.days_ago, h_pay) + data.score_recharge = M + + # 4. 时长贡献 D = Σ(sqrt(dur/60) × τ × decay(d, h_sess)) + D = 0.0 + for session in data.sessions: + days_ago = (now - session.session_end).total_seconds() / 86400 + dur_hours = session.total_duration_minutes / 60.0 + D += math.sqrt(dur_hours) * session.course_weight * self.decay(days_ago, h_sess) + data.score_duration = D + + # 5. 频率激增放大 mult = 1 + γ × burst + # F_short = Σ(τ × decay(d, h_short)) + # F_long = Σ(τ × decay(d, h_long)) + F_short = 0.0 + F_long = 0.0 + for session in data.sessions: + days_ago = (now - session.session_end).total_seconds() / 86400 + F_short += session.course_weight * self.decay(days_ago, h_short) + F_long += session.course_weight * self.decay(days_ago, h_long) + + # burst = max(0, ln(1 + (F_short/F_long - 1))) + ratio = F_short / (F_long + epsilon) + if ratio > 1: + burst = self.safe_ln1p(ratio - 1) + else: + burst = 0.0 + + data.burst_multiplier = 1 + gamma * burst + + # ========================================================================== + # 数据保存方法 + # ========================================================================== + + def _save_intimacy_data(self, data_list: List[MemberAssistantIntimacyData]) -> int: + """保存亲密数据到DWS表""" + if not data_list: + return 0 + + # 先删除已存在的记录 + site_id = data_list[0].site_id + + # 构建删除条件（使用assistant_no） + # 注意：assistant_id字段在数据库中存储assistant_no的整数形式 + keys = [(d.member_id, d.assistant_no) for d in data_list] + conditions = " OR ".join( + f"(member_id = {m} AND assistant_id = {int(a)})" for m, a in keys + ) + + delete_sql = f""" + DELETE FROM billiards_dws.dws_member_assistant_intimacy + WHERE site_id = %s AND ({conditions}) + """ + + with self.db.conn.cursor() as cur: + cur.execute(delete_sql, (site_id,)) + + # 插入新记录 + # 使用assistant_no的整数值作为assistant_id + insert_sql = """ + INSERT INTO billiards_dws.dws_member_assistant_intimacy ( + site_id, tenant_id, member_id, assistant_id, + session_count, total_duration_minutes, + basic_session_count, incentive_session_count, + days_since_last_session, + attributed_recharge_count, attributed_recharge_amount, + score_frequency, score_recency, score_recharge, score_duration, + burst_multiplier, raw_score, display_score, + calc_time, created_at, updated_at + ) VALUES ( + %s, %s, %s, %s, + %s, %s, + %s, %s, + %s, + %s, %s, + %s, %s, %s, %s, + %s, %s, %s, + NOW(), NOW(), NOW() + ) + """ + + inserted = 0 + with self.db.conn.cursor() as cur: + for data in data_list: + # 将assistant_no转为整数作为assistant_id + assistant_id = int(data.assistant_no) if data.assistant_no.isdigit() else 0 + cur.execute(insert_sql, ( + data.site_id, data.tenant_id, data.member_id, assistant_id, + data.session_count, data.total_duration_minutes, + data.basic_session_count, data.incentive_session_count, + data.days_since_last_session, + data.attributed_recharge_count, data.attributed_recharge_amount, + data.score_frequency, data.score_recency, data.score_recharge, data.score_duration, + data.burst_multiplier, data.raw_score, data.display_score + )) + inserted += cur.rowcount + + # 提交事务 + self.db.conn.commit() + + return inserted + + # ========================================================================== + # 辅助方法 + # ========================================================================== + + def _load_params(self) -> Dict[str, float]: + """加载参数，缺失时使用默认值""" + params = self.load_index_parameters() + result = dict(self.DEFAULT_PARAMS) + result.update(params) + return result + + def _get_site_id(self, context: Optional[TaskContext]) -> int: + """获取门店ID""" + if context and hasattr(context, 'store_id') and context.store_id: + return context.store_id + + site_id = self.config.get('app.default_site_id') + if site_id: + return int(site_id) + + sql = "SELECT DISTINCT site_id FROM billiards_dwd.dwd_assistant_service_log LIMIT 1" + rows = self.db.query(sql) + if rows: + return int(dict(rows[0])['site_id']) + + raise ValueError("无法确定门店ID") + + def _get_tenant_id(self) -> int: + """获取租户ID""" + tenant_id = self.config.get('app.tenant_id') + if tenant_id: + return int(tenant_id) + + sql = "SELECT DISTINCT tenant_id FROM billiards_dwd.dwd_assistant_service_log LIMIT 1" + rows = self.db.query(sql) + if rows: + return int(dict(rows[0])['tenant_id']) + + return 0 diff --git a/etl_billiards/tasks/dws/index/recall_index_task.py b/etl_billiards/tasks/dws/index/recall_index_task.py new file mode 100644 index 0000000..a54f2ec --- /dev/null +++ b/etl_billiards/tasks/dws/index/recall_index_task.py @@ -0,0 +1,564 @@ +# -*- coding: utf-8 -*- +""" +客户召回指数计算任务 + +功能说明： + - 衡量客户召回的必要性和紧急程度 + - 尊重客户个人到店周期（μ=中位数, σ=MAD） + - 对新客户、刚充值客户增加召回倾向 + - 检测"热了又断"的情况 + +算法公式： + Raw Score = w_over × overdue + w_new × new_bonus + w_re × re_bonus + w_hot × hot_drop + + 其中： + - overdue = 1 - exp(-max(0, (t-μ)/σ)) # 超期紧急性 + - new_bonus = decay(d_first, h_new) # 新客户加分 + - re_bonus = decay(d_recharge, h_re) # 刚充值加分 + - hot_drop = max(0, ln(1 + (r14/r60 - 1))) # 热度断档加分 + +数据来源： + - dwd_settlement_head: 会员到店记录 + - dwd_recharge_order: 充值记录 + - dim_member: 首访时间 + +更新频率：每2小时 + +作者：ETL团队 +创建日期：2026-02-03 +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from .base_index_task import BaseIndexTask, PercentileHistory +from ..base_dws_task import TaskContext + + +# ============================================================================= +# 数据类定义 +# ============================================================================= + +@dataclass +class MemberRecallData: + """会员召回数据""" + member_id: int + site_id: int + tenant_id: int + + # 计算输入特征 + days_since_last_visit: Optional[int] = None + visit_interval_median: Optional[float] = None + visit_interval_mad: Optional[float] = None + days_since_first_visit: Optional[int] = None + days_since_last_recharge: Optional[int] = None + visits_last_14_days: int = 0 + visits_last_60_days: int = 0 + + # 分项得分 + score_overdue: float = 0.0 + score_new_bonus: float = 0.0 + score_recharge_bonus: float = 0.0 + score_hot_drop: float = 0.0 + + # 最终分数 + raw_score: float = 0.0 + display_score: float = 0.0 + + +# ============================================================================= +# 召回指数任务 +# ============================================================================= + +class RecallIndexTask(BaseIndexTask): + """ + 客户召回指数计算任务 + + 计算流程： + 1. 提取近60天有到店记录的会员 + 2. 计算每个会员的到店间隔特征（中位数、MAD） + 3. 计算4项分数（超期、新客、充值、热度断档） + 4. 汇总Raw Score + 5. 分位截断 + MinMax映射到0-10 + 6. 写入DWS表 + """ + + INDEX_TYPE = "RECALL" + + # 默认参数 + DEFAULT_PARAMS = { + 'lookback_days': 60, + 'sigma_min': 2.0, + 'halflife_new': 7.0, + 'halflife_recharge': 10.0, + 'weight_overdue': 3.0, + 'weight_new': 1.0, + 'weight_recharge': 1.0, + 'weight_hot': 1.0, + 'percentile_lower': 5, + 'percentile_upper': 95, + } + + # ========================================================================== + # 抽象方法实现 + # ========================================================================== + + def get_task_code(self) -> str: + return "DWS_RECALL_INDEX" + + def get_target_table(self) -> str: + return "dws_member_recall_index" + + def get_primary_keys(self) -> List[str]: + return ['site_id', 'member_id'] + + def get_index_type(self) -> str: + return self.INDEX_TYPE + + # ========================================================================== + # 任务执行 + # ========================================================================== + + def execute(self, context: Optional[TaskContext]) -> Dict[str, Any]: + """执行召回指数计算""" + self.logger.info("开始计算客户召回指数") + + # 获取门店ID + site_id = self._get_site_id(context) + tenant_id = self._get_tenant_id() + + # 加载参数 + params = self._load_params() + lookback_days = int(params['lookback_days']) + + # 计算基准日期 + base_date = date.today() + start_date = base_date - timedelta(days=lookback_days) + + self.logger.info( + "参数: lookback=%d天, sigma_min=%.1f, h_new=%.1f, h_re=%.1f", + lookback_days, params['sigma_min'], params['halflife_new'], params['halflife_recharge'] + ) + + # 1. 提取会员到店数据 + member_visits = self._extract_member_visits(site_id, start_date, base_date) + self.logger.info("提取到 %d 个会员的到店记录", len(member_visits)) + + if not member_visits: + self.logger.warning("没有会员到店记录，跳过计算") + return {'status': 'skipped', 'reason': 'no_data'} + + # 2. 提取充值记录 + recharge_data = self._extract_recharge_data(site_id, start_date, base_date) + self.logger.info("提取到 %d 个会员的充值记录", len(recharge_data)) + + # 3. 提取首访时间 + first_visit_data = self._extract_first_visit_data(site_id, list(member_visits.keys())) + self.logger.info("提取到 %d 个会员的首访时间", len(first_visit_data)) + + # 4. 计算每个会员的召回数据 + recall_data_list: List[MemberRecallData] = [] + + for member_id, visit_dates in member_visits.items(): + data = MemberRecallData( + member_id=member_id, + site_id=site_id, + tenant_id=tenant_id + ) + + # 计算特征 + self._calculate_visit_features(data, visit_dates, base_date, params) + + # 补充充值特征 + if member_id in recharge_data: + last_recharge_date = recharge_data[member_id] + data.days_since_last_recharge = (base_date - last_recharge_date).days + + # 补充首访特征 + if member_id in first_visit_data: + first_visit_date = first_visit_data[member_id] + data.days_since_first_visit = (base_date - first_visit_date).days + + # 计算分项得分 + self._calculate_component_scores(data, params) + + # 汇总Raw Score + data.raw_score = ( + params['weight_overdue'] * data.score_overdue + + params['weight_new'] * data.score_new_bonus + + params['weight_recharge'] * data.score_recharge_bonus + + params['weight_hot'] * data.score_hot_drop + ) + + recall_data_list.append(data) + + self.logger.info("计算完成 %d 个会员的Raw Score", len(recall_data_list)) + + # 5. 归一化到Display Score + raw_scores = [(d.member_id, d.raw_score) for d in recall_data_list] + normalized = self.batch_normalize_to_display( + raw_scores, + use_log=False, + percentile_lower=int(params['percentile_lower']), + percentile_upper=int(params['percentile_upper']), + use_smoothing=True, + site_id=site_id + ) + + # 更新display_score + score_map = {member_id: (raw, display) for member_id, raw, display in normalized} + for data in recall_data_list: + if data.member_id in score_map: + _, data.display_score = score_map[data.member_id] + + # 6. 保存分位点历史 + if recall_data_list: + all_raw = [d.raw_score for d in recall_data_list] + q_l, q_u = self.calculate_percentiles( + all_raw, + int(params['percentile_lower']), + int(params['percentile_upper']) + ) + smoothed_l, smoothed_u = self._apply_ewma_smoothing(site_id, q_l, q_u) + + self.save_percentile_history( + site_id=site_id, + percentile_5=q_l, + percentile_95=q_u, + percentile_5_smoothed=smoothed_l, + percentile_95_smoothed=smoothed_u, + record_count=len(all_raw), + min_raw=min(all_raw), + max_raw=max(all_raw), + avg_raw=sum(all_raw) / len(all_raw) + ) + + # 7. 写入DWS表 + inserted = self._save_recall_data(recall_data_list) + + self.logger.info("召回指数计算完成，写入 %d 条记录", inserted) + + return { + 'status': 'success', + 'member_count': len(recall_data_list), + 'records_inserted': inserted + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_member_visits( + self, + site_id: int, + start_date: date, + end_date: date + ) -> Dict[int, List[date]]: + """ + 提取会员到店记录 + + Returns: + {member_id: [visit_date1, visit_date2, ...]} + """ + sql = """ + SELECT + member_id, + DATE(pay_time) AS visit_date + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %s + AND member_id > 0 -- 排除散客 + AND settle_type = 1 -- 台桌结账 + AND pay_time >= %s + AND pay_time < %s + INTERVAL '1 day' + GROUP BY member_id, DATE(pay_time) + ORDER BY member_id, visit_date + """ + + rows = self.db.query(sql, (site_id, start_date, end_date)) + + result: Dict[int, List[date]] = {} + for row in (rows or []): + row_dict = dict(row) + member_id = int(row_dict['member_id']) + visit_date = row_dict['visit_date'] + + if member_id not in result: + result[member_id] = [] + result[member_id].append(visit_date) + + return result + + def _extract_recharge_data( + self, + site_id: int, + start_date: date, + end_date: date + ) -> Dict[int, date]: + """ + 提取最近充值记录 + + Returns: + {member_id: last_recharge_date} + """ + sql = """ + SELECT + member_id, + MAX(DATE(pay_time)) AS last_recharge_date + FROM billiards_dwd.dwd_recharge_order + WHERE site_id = %s + AND member_id > 0 + AND settle_type = 5 -- 充值订单 + AND pay_time >= %s + AND pay_time < %s + INTERVAL '1 day' + GROUP BY member_id + """ + + rows = self.db.query(sql, (site_id, start_date, end_date)) + + result: Dict[int, date] = {} + for row in (rows or []): + row_dict = dict(row) + result[int(row_dict['member_id'])] = row_dict['last_recharge_date'] + + return result + + def _extract_first_visit_data( + self, + site_id: int, + member_ids: List[int] + ) -> Dict[int, date]: + """ + 提取首访时间 + + 优先使用dim_member.create_time，如果没有则使用dwd_settlement_head中的首次消费时间 + + Returns: + {member_id: first_visit_date} + """ + if not member_ids: + return {} + + # 使用dim_member的create_time作为首访时间 + member_ids_str = ','.join(str(m) for m in member_ids) + sql = f""" + SELECT + member_id, + DATE(create_time) AS first_visit_date + FROM billiards_dwd.dim_member + WHERE member_id IN ({member_ids_str}) + AND scd2_is_current = 1 + """ + + rows = self.db.query(sql) + + result: Dict[int, date] = {} + for row in (rows or []): + row_dict = dict(row) + member_id = int(row_dict['member_id']) + first_date = row_dict['first_visit_date'] + if first_date: + result[member_id] = first_date + + return result + + # ========================================================================== + # 特征计算方法 + # ========================================================================== + + def _calculate_visit_features( + self, + data: MemberRecallData, + visit_dates: List[date], + base_date: date, + params: Dict[str, float] + ) -> None: + """计算到店特征""" + if not visit_dates: + return + + # 最近一次到店 + last_visit = max(visit_dates) + data.days_since_last_visit = (base_date - last_visit).days + + # 到店间隔 + sorted_dates = sorted(visit_dates) + intervals = [] + for i in range(1, len(sorted_dates)): + interval = (sorted_dates[i] - sorted_dates[i-1]).days + intervals.append(float(interval)) + + if intervals: + # 中位数（μ） + data.visit_interval_median = self.calculate_median(intervals) + + # MAD（σ），下限为sigma_min + mad = self.calculate_mad(intervals) + data.visit_interval_mad = max(mad, params['sigma_min']) + else: + # 只有一次到店，使用默认值 + data.visit_interval_median = 7.0 # 默认周期7天 + data.visit_interval_mad = params['sigma_min'] + + # 近14天/60天到店次数 + days_14_ago = base_date - timedelta(days=14) + days_60_ago = base_date - timedelta(days=60) + + data.visits_last_14_days = sum(1 for d in visit_dates if d >= days_14_ago) + data.visits_last_60_days = sum(1 for d in visit_dates if d >= days_60_ago) + + def _calculate_component_scores( + self, + data: MemberRecallData, + params: Dict[str, float] + ) -> None: + """计算4项分数""" + + # 1. 超期紧急性 + if data.days_since_last_visit is not None and data.visit_interval_median is not None: + t = data.days_since_last_visit + mu = data.visit_interval_median + sigma = data.visit_interval_mad or params['sigma_min'] + + # z = max(0, (t - μ) / σ) + z = max(0.0, (t - mu) / sigma) + # overdue = 1 - exp(-z) + data.score_overdue = 1.0 - math.exp(-z) + + # 2. 新客户加分 + lookback_days = int(params['lookback_days']) + if data.days_since_first_visit is not None and data.days_since_first_visit <= lookback_days: + data.score_new_bonus = self.decay( + data.days_since_first_visit, + params['halflife_new'] + ) + + # 3. 刚充值加分 + if data.days_since_last_recharge is not None and data.days_since_last_recharge <= lookback_days: + data.score_recharge_bonus = self.decay( + data.days_since_last_recharge, + params['halflife_recharge'] + ) + + # 4. 热度断档加分 + epsilon = 1e-6 + n14 = data.visits_last_14_days + n60 = data.visits_last_60_days + + r14 = n14 / 14.0 + r60 = (n60 + 1) / 60.0 # +1 平滑 + + hot_ratio = r14 / (r60 + epsilon) + + # hot_drop = max(0, ln(1 + (hot_ratio - 1))) + if hot_ratio > 1: + data.score_hot_drop = self.safe_ln1p(hot_ratio - 1) + else: + data.score_hot_drop = 0.0 + + # ========================================================================== + # 数据保存方法 + # ========================================================================== + + def _save_recall_data(self, data_list: List[MemberRecallData]) -> int: + """保存召回数据到DWS表""" + if not data_list: + return 0 + + # 先删除已存在的记录 + site_id = data_list[0].site_id + member_ids = [d.member_id for d in data_list] + + member_ids_str = ','.join(str(m) for m in member_ids) + delete_sql = f""" + DELETE FROM billiards_dws.dws_member_recall_index + WHERE site_id = %s AND member_id IN ({member_ids_str}) + """ + + with self.db.conn.cursor() as cur: + cur.execute(delete_sql, (site_id,)) + + # 插入新记录 + insert_sql = """ + INSERT INTO billiards_dws.dws_member_recall_index ( + site_id, tenant_id, member_id, + days_since_last_visit, visit_interval_median, visit_interval_mad, + days_since_first_visit, days_since_last_recharge, + visits_last_14_days, visits_last_60_days, + score_overdue, score_new_bonus, score_recharge_bonus, score_hot_drop, + raw_score, display_score, + calc_time, created_at, updated_at + ) VALUES ( + %s, %s, %s, + %s, %s, %s, + %s, %s, + %s, %s, + %s, %s, %s, %s, + %s, %s, + NOW(), NOW(), NOW() + ) + """ + + inserted = 0 + with self.db.conn.cursor() as cur: + for data in data_list: + cur.execute(insert_sql, ( + data.site_id, data.tenant_id, data.member_id, + data.days_since_last_visit, data.visit_interval_median, data.visit_interval_mad, + data.days_since_first_visit, data.days_since_last_recharge, + data.visits_last_14_days, data.visits_last_60_days, + data.score_overdue, data.score_new_bonus, data.score_recharge_bonus, data.score_hot_drop, + data.raw_score, data.display_score + )) + inserted += cur.rowcount + + # 提交事务 + self.db.conn.commit() + + return inserted + + # ========================================================================== + # 辅助方法 + # ========================================================================== + + def _load_params(self) -> Dict[str, float]: + """加载参数，缺失时使用默认值""" + params = self.load_index_parameters() + result = dict(self.DEFAULT_PARAMS) + result.update(params) + return result + + def _get_site_id(self, context: Optional[TaskContext]) -> int: + """获取门店ID""" + if context and hasattr(context, 'store_id') and context.store_id: + return context.store_id + + # 从配置获取默认门店ID + site_id = self.config.get('app.default_site_id') + if site_id: + return int(site_id) + + # 查询数据库获取第一个门店 + sql = "SELECT DISTINCT site_id FROM billiards_dwd.dwd_settlement_head LIMIT 1" + rows = self.db.query(sql) + if rows: + return int(dict(rows[0])['site_id']) + + raise ValueError("无法确定门店ID") + + def _get_tenant_id(self) -> int: + """获取租户ID""" + tenant_id = self.config.get('app.tenant_id') + if tenant_id: + return int(tenant_id) + + sql = "SELECT DISTINCT tenant_id FROM billiards_dwd.dwd_settlement_head LIMIT 1" + rows = self.db.query(sql) + if rows: + return int(dict(rows[0])['tenant_id']) + + return 0 diff --git a/etl_billiards/tasks/dws/member_consumption_task.py b/etl_billiards/tasks/dws/member_consumption_task.py new file mode 100644 index 0000000..63d70b7 --- /dev/null +++ b/etl_billiards/tasks/dws/member_consumption_task.py @@ -0,0 +1,368 @@ +# -*- coding: utf-8 -*- +""" +会员消费汇总任务 + +功能说明： + 以"会员"为粒度，统计消费行为和滚动窗口指标 + +数据来源： + - dwd_settlement_head: 结账单头表 + - dim_member: 会员维度 + - dim_member_card_account: 会员卡账户 + +目标表： + billiards_dws.dws_member_consumption_summary + +更新策略： + - 更新频率：每日更新 + - 幂等方式：delete-before-insert（按统计日期） + +业务规则： + - 散客处理：member_id=0 不进入此表 + - 滚动窗口：7/10/15/30/60/90天 + - 卡余额：区分储值卡（现金卡）和赠送卡 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Set, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class MemberConsumptionTask(BaseDwsTask): + """ + 会员消费汇总任务 + + 统计每个会员的： + - 首次/最近消费日期 + - 累计消费统计 + - 滚动窗口统计（7/10/15/30/60/90天） + - 卡余额快照 + - 活跃度指标和客户分层 + """ + + def get_task_code(self) -> str: + return "DWS_MEMBER_CONSUMPTION" + + def get_target_table(self) -> str: + return "dws_member_consumption_summary" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "member_id", "stat_date"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据 + """ + stat_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + self.logger.info( + "%s: 提取数据，统计日期 %s", + self.get_task_code(), stat_date + ) + + # 1. 获取会员消费统计（含滚动窗口） + consumption_stats = self._extract_consumption_stats(site_id, stat_date) + + # 2. 获取会员信息 + member_info = self._extract_member_info(site_id) + + # 3. 获取会员卡余额 + card_balances = self._extract_card_balances(site_id) + + return { + 'consumption_stats': consumption_stats, + 'member_info': member_info, + 'card_balances': card_balances, + 'stat_date': stat_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据 + """ + consumption_stats = extracted['consumption_stats'] + member_info = extracted['member_info'] + card_balances = extracted['card_balances'] + stat_date = extracted['stat_date'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，%d 条会员消费记录", + self.get_task_code(), len(consumption_stats) + ) + + results = [] + + for stats in consumption_stats: + member_id = stats.get('member_id') + + # 跳过散客 + if self.is_guest(member_id): + continue + + memb_info = member_info.get(member_id, {}) + balance = card_balances.get(member_id, {}) + + # 计算活跃度和客户分层 + days_since_last = self._calc_days_since(stat_date, stats.get('last_consume_date')) + customer_tier = self._calculate_customer_tier(stats, days_since_last) + + record = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'member_id': member_id, + 'stat_date': stat_date, + # 会员基本信息 + 'member_nickname': memb_info.get('nickname'), + 'member_mobile': self._mask_mobile(memb_info.get('mobile')), + 'card_grade_name': memb_info.get('member_card_grade_name'), + 'register_date': memb_info.get('register_date'), + # 全量累计统计 + 'first_consume_date': stats.get('first_consume_date'), + 'last_consume_date': stats.get('last_consume_date'), + 'total_visit_count': self.safe_int(stats.get('total_visit_count', 0)), + 'total_consume_amount': self.safe_decimal(stats.get('total_consume_amount', 0)), + 'total_recharge_amount': self.safe_decimal(memb_info.get('recharge_money_sum', 0)), + 'total_table_fee': self.safe_decimal(stats.get('total_table_fee', 0)), + 'total_goods_amount': self.safe_decimal(stats.get('total_goods_amount', 0)), + 'total_assistant_amount': self.safe_decimal(stats.get('total_assistant_amount', 0)), + # 滚动窗口统计 + 'visit_count_7d': self.safe_int(stats.get('visit_count_7d', 0)), + 'visit_count_10d': self.safe_int(stats.get('visit_count_10d', 0)), + 'visit_count_15d': self.safe_int(stats.get('visit_count_15d', 0)), + 'visit_count_30d': self.safe_int(stats.get('visit_count_30d', 0)), + 'visit_count_60d': self.safe_int(stats.get('visit_count_60d', 0)), + 'visit_count_90d': self.safe_int(stats.get('visit_count_90d', 0)), + 'consume_amount_7d': self.safe_decimal(stats.get('consume_amount_7d', 0)), + 'consume_amount_10d': self.safe_decimal(stats.get('consume_amount_10d', 0)), + 'consume_amount_15d': self.safe_decimal(stats.get('consume_amount_15d', 0)), + 'consume_amount_30d': self.safe_decimal(stats.get('consume_amount_30d', 0)), + 'consume_amount_60d': self.safe_decimal(stats.get('consume_amount_60d', 0)), + 'consume_amount_90d': self.safe_decimal(stats.get('consume_amount_90d', 0)), + # 卡余额 + 'cash_card_balance': self.safe_decimal(balance.get('cash_balance', 0)), + 'gift_card_balance': self.safe_decimal(balance.get('gift_balance', 0)), + 'total_card_balance': self.safe_decimal(balance.get('total_balance', 0)), + # 活跃度指标 + 'days_since_last': days_since_last, + 'is_active_7d': self.safe_int(stats.get('visit_count_7d', 0)) > 0, + 'is_active_30d': self.safe_int(stats.get('visit_count_30d', 0)) > 0, + 'is_active_90d': self.safe_int(stats.get('visit_count_90d', 0)) > 0, + # 客户分层 + 'customer_tier': customer_tier, + } + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + deleted = self.delete_existing_data(context, date_col="stat_date") + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_consumption_stats( + self, + site_id: int, + stat_date: date + ) -> List[Dict[str, Any]]: + """ + 提取会员消费统计（含滚动窗口） + """ + sql = """ + WITH consume_base AS ( + SELECT + member_id, + DATE(create_time) AS consume_date, + consume_money, + table_charge_money, + goods_money, + assistant_pd_money + assistant_cx_money AS assistant_amount + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %s + AND member_id IS NOT NULL + AND member_id != 0 + ) + SELECT + member_id, + MIN(consume_date) AS first_consume_date, + MAX(consume_date) AS last_consume_date, + -- 全量累计 + COUNT(*) AS total_visit_count, + SUM(consume_money) AS total_consume_amount, + SUM(table_charge_money) AS total_table_fee, + SUM(goods_money) AS total_goods_amount, + SUM(assistant_amount) AS total_assistant_amount, + -- 滚动窗口 + COUNT(CASE WHEN consume_date >= %s - INTERVAL '6 days' THEN 1 END) AS visit_count_7d, + COUNT(CASE WHEN consume_date >= %s - INTERVAL '9 days' THEN 1 END) AS visit_count_10d, + COUNT(CASE WHEN consume_date >= %s - INTERVAL '14 days' THEN 1 END) AS visit_count_15d, + COUNT(CASE WHEN consume_date >= %s - INTERVAL '29 days' THEN 1 END) AS visit_count_30d, + COUNT(CASE WHEN consume_date >= %s - INTERVAL '59 days' THEN 1 END) AS visit_count_60d, + COUNT(CASE WHEN consume_date >= %s - INTERVAL '89 days' THEN 1 END) AS visit_count_90d, + SUM(CASE WHEN consume_date >= %s - INTERVAL '6 days' THEN consume_money ELSE 0 END) AS consume_amount_7d, + SUM(CASE WHEN consume_date >= %s - INTERVAL '9 days' THEN consume_money ELSE 0 END) AS consume_amount_10d, + SUM(CASE WHEN consume_date >= %s - INTERVAL '14 days' THEN consume_money ELSE 0 END) AS consume_amount_15d, + SUM(CASE WHEN consume_date >= %s - INTERVAL '29 days' THEN consume_money ELSE 0 END) AS consume_amount_30d, + SUM(CASE WHEN consume_date >= %s - INTERVAL '59 days' THEN consume_money ELSE 0 END) AS consume_amount_60d, + SUM(CASE WHEN consume_date >= %s - INTERVAL '89 days' THEN consume_money ELSE 0 END) AS consume_amount_90d + FROM consume_base + GROUP BY member_id + """ + params = [site_id] + [stat_date] * 12 + rows = self.db.query(sql, tuple(params)) + return [dict(row) for row in rows] if rows else [] + + def _extract_member_info(self, site_id: int) -> Dict[int, Dict[str, Any]]: + """ + 提取会员信息 + """ + sql = """ + SELECT + member_id, + nickname, + mobile, + member_card_grade_name, + DATE(create_time) AS register_date, + recharge_money_sum + FROM billiards_dwd.dim_member + WHERE site_id = %s + """ + rows = self.db.query(sql, (site_id,)) + + result = {} + for row in (rows or []): + row_dict = dict(row) + result[row_dict['member_id']] = row_dict + return result + + def _extract_card_balances(self, site_id: int) -> Dict[int, Dict[str, Decimal]]: + """ + 提取会员卡余额 + """ + # 卡类型ID + CASH_CARD_TYPE_ID = 2793249295533893 + GIFT_CARD_TYPE_IDS = [2791990152417157, 2793266846533445, 2794699703437125] + + sql = """ + SELECT + tenant_member_id AS member_id, + card_type_id, + balance + FROM billiards_dwd.dim_member_card_account + WHERE site_id = %s + AND valid_to IS NULL + """ + rows = self.db.query(sql, (site_id,)) + + result: Dict[int, Dict[str, Decimal]] = {} + for row in (rows or []): + row_dict = dict(row) + member_id = row_dict.get('member_id') + card_type_id = row_dict.get('card_type_id') + balance = self.safe_decimal(row_dict.get('balance', 0)) + + if member_id not in result: + result[member_id] = { + 'cash_balance': Decimal('0'), + 'gift_balance': Decimal('0'), + 'total_balance': Decimal('0') + } + + if card_type_id == CASH_CARD_TYPE_ID: + result[member_id]['cash_balance'] += balance + elif card_type_id in GIFT_CARD_TYPE_IDS: + result[member_id]['gift_balance'] += balance + + result[member_id]['total_balance'] = ( + result[member_id]['cash_balance'] + result[member_id]['gift_balance'] + ) + + return result + + # ========================================================================== + # 工具方法 + # ========================================================================== + + def _mask_mobile(self, mobile: Optional[str]) -> Optional[str]: + """手机号脱敏""" + if not mobile or len(mobile) < 7: + return mobile + return mobile[:3] + "****" + mobile[-4:] + + def _calc_days_since(self, stat_date: date, last_date: Optional[date]) -> Optional[int]: + """计算距离最近消费的天数""" + if not last_date: + return None + if isinstance(last_date, datetime): + last_date = last_date.date() + return (stat_date - last_date).days + + def _calculate_customer_tier( + self, + stats: Dict[str, Any], + days_since_last: Optional[int] + ) -> str: + """ + 计算客户分层 + + 分层规则： + - 高价值：90天内消费>=3次 且 消费金额>=1000 + - 中等：30天内有消费 + - 低活跃：90天内有消费但30天内无消费 + - 流失：90天内无消费 + """ + visit_90d = self.safe_int(stats.get('visit_count_90d', 0)) + visit_30d = self.safe_int(stats.get('visit_count_30d', 0)) + amount_90d = self.safe_decimal(stats.get('consume_amount_90d', 0)) + + if visit_90d >= 3 and amount_90d >= 1000: + return "高价值" + elif visit_30d > 0: + return "中等" + elif visit_90d > 0: + return "低活跃" + else: + return "流失" + + +# 便于外部导入 +__all__ = ['MemberConsumptionTask'] diff --git a/etl_billiards/tasks/dws/member_visit_task.py b/etl_billiards/tasks/dws/member_visit_task.py new file mode 100644 index 0000000..0fee63d --- /dev/null +++ b/etl_billiards/tasks/dws/member_visit_task.py @@ -0,0 +1,386 @@ +# -*- coding: utf-8 -*- +""" +会员来店明细任务 + +功能说明： + 以"会员+订单"为粒度，记录每次来店消费明细 + +数据来源： + - dwd_settlement_head: 结账单头表 + - dwd_assistant_service_log: 助教服务流水 + - dim_member: 会员维度 + - dim_table: 台桌维度 + - cfg_area_category: 区域分类映射 + +目标表： + billiards_dws.dws_member_visit_detail + +更新策略： + - 更新频率：每日增量更新 + - 幂等方式：delete-before-insert（按日期窗口） + +业务规则： + - 散客处理：member_id=0 不进入此表 + - 区域分类：使用cfg_area_category映射 + - 助教服务：以JSON格式存储多个助教的服务明细 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +import json +from datetime import date, datetime, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Set, Tuple + +from .base_dws_task import BaseDwsTask, TaskContext + + +class MemberVisitTask(BaseDwsTask): + """ + 会员来店明细任务 + + 记录每个会员每次来店的： + - 台桌信息和区域分类 + - 消费金额明细 + - 支付方式明细 + - 助教服务明细（JSON格式） + """ + + def get_task_code(self) -> str: + return "DWS_MEMBER_VISIT" + + def get_target_table(self) -> str: + return "dws_member_visit_detail" + + def get_primary_keys(self) -> List[str]: + return ["site_id", "member_id", "order_settle_id"] + + # ========================================================================== + # ETL主流程 + # ========================================================================== + + def extract(self, context: TaskContext) -> Dict[str, Any]: + """ + 提取数据 + """ + start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start + end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end + site_id = context.store_id + + self.logger.info( + "%s: 提取数据，日期范围 %s ~ %s", + self.get_task_code(), start_date, end_date + ) + + # 1. 获取结账单 + settlements = self._extract_settlements(site_id, start_date, end_date) + + # 2. 获取助教服务明细 + assistant_services = self._extract_assistant_services(site_id, start_date, end_date) + + # 3. 获取会员信息 + member_info = self._extract_member_info(site_id) + + # 4. 获取台桌信息 + table_info = self._extract_table_info(site_id) + + # 5. 加载配置 + self.load_config_cache() + + return { + 'settlements': settlements, + 'assistant_services': assistant_services, + 'member_info': member_info, + 'table_info': table_info, + 'start_date': start_date, + 'end_date': end_date, + 'site_id': site_id + } + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]: + """ + 转换数据 + """ + settlements = extracted['settlements'] + assistant_services = extracted['assistant_services'] + member_info = extracted['member_info'] + table_info = extracted['table_info'] + site_id = extracted['site_id'] + + self.logger.info( + "%s: 转换数据，%d 条结账单", + self.get_task_code(), len(settlements) + ) + + # 构建助教服务索引：order_settle_id -> [services] + service_index = self._build_service_index(assistant_services) + + results = [] + + for settle in settlements: + member_id = settle.get('member_id') + + # 跳过散客 + if self.is_guest(member_id): + continue + + order_settle_id = settle.get('order_settle_id') + table_id = settle.get('table_id') + + memb_info = member_info.get(member_id, {}) + tbl_info = table_info.get(table_id, {}) + services = service_index.get(order_settle_id, []) + + # 获取区域分类 + area_name = tbl_info.get('area_name') + area_cat = self.get_area_category(area_name) + + # 构建助教服务JSON + assistant_services_json = self._build_assistant_services_json(services) + + # 计算时长 + table_duration = self._calc_table_duration(settle) + assistant_duration = sum( + self.safe_int(s.get('income_seconds', 0)) + for s in services + ) // 60 # 转为分钟 + + record = { + 'site_id': site_id, + 'tenant_id': self.config.get("app.tenant_id", site_id), + 'member_id': member_id, + 'order_settle_id': order_settle_id, + 'visit_date': settle.get('visit_date'), + 'visit_time': settle.get('create_time'), + # 会员信息 + 'member_nickname': memb_info.get('nickname'), + 'member_mobile': self._mask_mobile(memb_info.get('mobile')), + 'member_birthday': memb_info.get('birthday'), + # 台桌信息 + 'table_id': table_id, + 'table_name': tbl_info.get('table_name'), + 'area_name': area_name, + 'area_category': area_cat.get('category_name'), + # 消费金额 + 'table_fee': self.safe_decimal(settle.get('table_charge_money', 0)), + 'goods_amount': self.safe_decimal(settle.get('goods_money', 0)), + 'assistant_amount': self.safe_decimal(settle.get('assistant_pd_money', 0)) + \ + self.safe_decimal(settle.get('assistant_cx_money', 0)), + 'total_consume': self.safe_decimal(settle.get('consume_money', 0)), + 'total_discount': self._calc_total_discount(settle), + 'actual_pay': self.safe_decimal(settle.get('pay_amount', 0)), + # 支付方式 + 'cash_pay': self.safe_decimal(settle.get('pay_amount', 0)), + 'cash_card_pay': self.safe_decimal(settle.get('balance_amount', 0)), + 'gift_card_pay': self.safe_decimal(settle.get('gift_card_amount', 0)), + 'groupbuy_pay': self.safe_decimal(settle.get('coupon_amount', 0)), + # 时长 + 'table_duration_min': table_duration, + 'assistant_duration_min': assistant_duration, + # 助教服务明细 + 'assistant_services': assistant_services_json, + } + results.append(record) + + return results + + def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict: + """ + 加载数据 + """ + if not transformed: + self.logger.info("%s: 无数据需要写入", self.get_task_code()) + return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}} + + deleted = self.delete_existing_data(context, date_col="visit_date") + inserted = self.bulk_insert(transformed) + + self.logger.info( + "%s: 加载完成，删除 %d 行，插入 %d 行", + self.get_task_code(), deleted, inserted + ) + + return { + "counts": { + "fetched": len(transformed), + "inserted": inserted, + "updated": 0, + "skipped": 0, + "errors": 0 + }, + "extra": {"deleted": deleted} + } + + # ========================================================================== + # 数据提取方法 + # ========================================================================== + + def _extract_settlements( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取结账单 + """ + sql = """ + SELECT + order_settle_id, + order_trade_no, + table_id, + member_id, + create_time, + DATE(create_time) AS visit_date, + consume_money, + pay_amount, + table_charge_money, + goods_money, + assistant_pd_money, + assistant_cx_money, + coupon_amount, + adjust_amount, + member_discount_amount, + rounding_amount, + gift_card_amount, + balance_amount, + recharge_card_amount + FROM billiards_dwd.dwd_settlement_head + WHERE site_id = %s + AND DATE(create_time) >= %s + AND DATE(create_time) <= %s + AND member_id IS NOT NULL + AND member_id != 0 + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_assistant_services( + self, + site_id: int, + start_date: date, + end_date: date + ) -> List[Dict[str, Any]]: + """ + 提取助教服务明细 + """ + sql = """ + SELECT + order_settle_id, + site_assistant_id AS assistant_id, + nickname AS assistant_nickname, + income_seconds, + ledger_amount + FROM billiards_dwd.dwd_assistant_service_log + WHERE site_id = %s + AND DATE(start_use_time) >= %s + AND DATE(start_use_time) <= %s + """ + rows = self.db.query(sql, (site_id, start_date, end_date)) + return [dict(row) for row in rows] if rows else [] + + def _extract_member_info(self, site_id: int) -> Dict[int, Dict[str, Any]]: + """ + 提取会员信息 + """ + sql = """ + SELECT + member_id, + nickname, + mobile, + birthday + FROM billiards_dwd.dim_member + WHERE site_id = %s + """ + rows = self.db.query(sql, (site_id,)) + return {r['member_id']: dict(r) for r in (rows or [])} + + def _extract_table_info(self, site_id: int) -> Dict[int, Dict[str, Any]]: + """ + 提取台桌信息 + """ + sql = """ + SELECT + site_table_id AS table_id, + site_table_name AS table_name, + site_table_area_name AS area_name + FROM billiards_dwd.dim_table + WHERE site_id = %s + AND valid_to IS NULL + """ + rows = self.db.query(sql, (site_id,)) + return {r['table_id']: dict(r) for r in (rows or [])} + + # ========================================================================== + # 工具方法 + # ========================================================================== + + def _build_service_index( + self, + services: List[Dict[str, Any]] + ) -> Dict[int, List[Dict[str, Any]]]: + """ + 构建助教服务索引 + """ + index: Dict[int, List[Dict[str, Any]]] = {} + for service in services: + order_id = service.get('order_settle_id') + if order_id: + if order_id not in index: + index[order_id] = [] + index[order_id].append(service) + return index + + def _build_assistant_services_json( + self, + services: List[Dict[str, Any]] + ) -> Optional[str]: + """ + 构建助教服务JSON + """ + if not services: + return None + + json_data = [] + for s in services: + json_data.append({ + 'assistant_id': s.get('assistant_id'), + 'nickname': s.get('assistant_nickname'), + 'duration_min': self.safe_int(s.get('income_seconds', 0)) // 60, + 'amount': float(self.safe_decimal(s.get('ledger_amount', 0))) + }) + + return json.dumps(json_data, ensure_ascii=False) + + def _calc_table_duration(self, settle: Dict[str, Any]) -> int: + """ + 计算台桌使用时长（分钟） + 简化处理：根据台费和假设单价估算 + """ + table_fee = self.safe_decimal(settle.get('table_charge_money', 0)) + if table_fee <= 0: + return 0 + # 假设平均台费单价为0.5元/分钟 + return int(table_fee / Decimal('0.5')) + + def _calc_total_discount(self, settle: Dict[str, Any]) -> Decimal: + """ + 计算总优惠 + """ + adjust = self.safe_decimal(settle.get('adjust_amount', 0)) + member_discount = self.safe_decimal(settle.get('member_discount_amount', 0)) + rounding = self.safe_decimal(settle.get('rounding_amount', 0)) + return adjust + member_discount + rounding + + def _mask_mobile(self, mobile: Optional[str]) -> Optional[str]: + """手机号脱敏""" + if not mobile or len(mobile) < 7: + return mobile + return mobile[:3] + "****" + mobile[-4:] + + +# 便于外部导入 +__all__ = ['MemberVisitTask'] diff --git a/etl_billiards/tasks/dws/retention_cleanup_task.py b/etl_billiards/tasks/dws/retention_cleanup_task.py new file mode 100644 index 0000000..680afc9 --- /dev/null +++ b/etl_billiards/tasks/dws/retention_cleanup_task.py @@ -0,0 +1,161 @@ +# -*- coding: utf-8 -*- +""" +DWS 时间分层清理任务 + +功能说明： + 按配置的时间分层范围，对 DWS 表执行历史数据清理。 + 该任务默认不启用，需通过配置显式开启。 + +配置示例（.env / settings）： + DWS_RETENTION_ENABLED=true + DWS_RETENTION_LAYER=LAST_3_MONTHS + DWS_RETENTION_TABLES=dws_finance_daily_summary,dws_assistant_daily_detail + DWS_RETENTION_TABLE_LAYERS={"dws_finance_expense_summary":"ALL"} + +作者：ETL团队 +创建日期：2026-02-03 +""" +from __future__ import annotations + +import json +from datetime import date +from typing import Any, Dict, List, Optional + +from .base_dws_task import BaseDwsTask, TaskContext, TimeLayer + + +class DwsRetentionCleanupTask(BaseDwsTask): + """ + DWS 时间分层清理任务 + """ + + DEFAULT_TABLES = [ + {"table": "dws_assistant_daily_detail", "date_col": "stat_date"}, + {"table": "dws_assistant_monthly_summary", "date_col": "stat_month"}, + {"table": "dws_assistant_customer_stats", "date_col": "stat_date"}, + {"table": "dws_assistant_salary_calc", "date_col": "salary_month"}, + {"table": "dws_assistant_recharge_commission", "date_col": "commission_month"}, + {"table": "dws_assistant_finance_analysis", "date_col": "stat_date"}, + {"table": "dws_member_consumption_summary", "date_col": "stat_date"}, + {"table": "dws_member_visit_detail", "date_col": "visit_date"}, + {"table": "dws_finance_daily_summary", "date_col": "stat_date"}, + {"table": "dws_finance_income_structure", "date_col": "stat_date"}, + {"table": "dws_finance_discount_detail", "date_col": "stat_date"}, + {"table": "dws_finance_recharge_summary", "date_col": "stat_date"}, + {"table": "dws_finance_expense_summary", "date_col": "expense_month"}, + {"table": "dws_platform_settlement", "date_col": "settlement_date"}, + ] + + def get_task_code(self) -> str: + return "DWS_RETENTION_CLEANUP" + + def get_target_table(self) -> str: + return "dws_finance_daily_summary" + + def get_primary_keys(self) -> List[str]: + return [] + + def extract(self, context: TaskContext) -> Dict[str, Any]: + return {"site_id": context.store_id} + + def transform(self, extracted: Dict[str, Any], context: TaskContext) -> Dict[str, Any]: + return extracted + + def load(self, transformed: Dict[str, Any], context: TaskContext) -> Dict: + """ + 执行清理逻辑 + """ + if not self._is_retention_enabled(): + self.logger.info("%s: 未启用清理配置，跳过", self.get_task_code()) + return {"counts": {"cleaned": 0}} + + base_date = context.window_end.date() if hasattr(context.window_end, "date") else context.window_end + default_layer = self._get_retention_layer(self.config.get("dws.retention.layer", "ALL")) + if default_layer is None: + self.logger.warning("%s: 未识别的清理层级，跳过", self.get_task_code()) + return {"counts": {"cleaned": 0}} + + target_tables = self._resolve_target_tables() + if not target_tables: + self.logger.info("%s: 未配置需要清理的表，跳过", self.get_task_code()) + return {"counts": {"cleaned": 0}} + + table_layers = self._resolve_table_layers() + + total_deleted = 0 + details = [] + for item in target_tables: + table = item["table"] + date_col = item["date_col"] + layer_name = table_layers.get(table, default_layer.value) + layer = self._get_retention_layer(layer_name) + if layer is None or layer == TimeLayer.ALL: + continue + + time_range = self.get_time_layer_range(layer, base_date) + cutoff = self._normalize_cutoff(date_col, time_range.start) + deleted = self._cleanup_table(table, date_col, cutoff, context.store_id) + total_deleted += deleted + details.append({"table": table, "deleted": deleted, "cutoff": str(cutoff)}) + + self.logger.info("%s: 清理完成，总删除 %d 行", self.get_task_code(), total_deleted) + return {"counts": {"cleaned": total_deleted}, "extra": {"details": details}} + + def _is_retention_enabled(self) -> bool: + return bool(self.config.get("dws.retention.enabled", False)) + + def _get_retention_layer(self, layer_name: Optional[str]) -> Optional[TimeLayer]: + if not layer_name: + return None + name = str(layer_name).upper() + try: + return TimeLayer[name] + except KeyError: + return None + + def _resolve_target_tables(self) -> List[Dict[str, str]]: + table_list = self.config.get("dws.retention.tables") + if not table_list: + return self.DEFAULT_TABLES + + if isinstance(table_list, str): + names = [t.strip() for t in table_list.split(",") if t.strip()] + else: + names = list(table_list) + + selected = [] + for item in self.DEFAULT_TABLES: + if item["table"] in names: + selected.append(item) + return selected + + def _resolve_table_layers(self) -> Dict[str, str]: + raw = self.config.get("dws.retention.table_layers") + if not raw: + return {} + if isinstance(raw, dict): + return {str(k): str(v) for k, v in raw.items()} + if isinstance(raw, str): + try: + parsed = json.loads(raw) + if isinstance(parsed, dict): + return {str(k): str(v) for k, v in parsed.items()} + except json.JSONDecodeError: + return {} + return {} + + def _normalize_cutoff(self, date_col: str, cutoff: date) -> date: + monthly_cols = {"stat_month", "salary_month", "commission_month", "expense_month"} + if date_col in monthly_cols: + return cutoff.replace(day=1) + return cutoff + + def _cleanup_table(self, table: str, date_col: str, cutoff: date, site_id: int) -> int: + full_table = f"{self.DWS_SCHEMA}.{table}" + sql = f"DELETE FROM {full_table} WHERE site_id = %s AND {date_col} < %s" + with self.db.conn.cursor() as cur: + cur.execute(sql, (site_id, cutoff)) + return cur.rowcount + + +__all__ = ["DwsRetentionCleanupTask"] diff --git a/etl_billiards/tasks/ods_tasks.py b/etl_billiards/tasks/ods_tasks.py index 46e695d..ad8d26b 100644 --- a/etl_billiards/tasks/ods_tasks.py +++ b/etl_billiards/tasks/ods_tasks.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """ODS ingestion tasks.""" from __future__ import annotations @@ -305,7 +305,9 @@ class BaseOdsTask(BaseTask): source_endpoint: str | None, ) -> tuple[int, int]: """ - 按 DB 表结构动态写入 ODS（只插新数据：ON CONFLICT DO NOTHING）。 + 按 DB 表结构动态写入 ODS。 + - 新记录：插入 + - 已存在的记录：回填 NULL 列（用新值填充数据库中为 NULL 的字段） 返回 (inserted, skipped)。 """ if not records: @@ -324,9 +326,54 @@ class BaseOdsTask(BaseTask): col_names = [c[0] for c in cols_info] quoted_cols = ", ".join(f'\"{c}\"' for c in col_names) sql = f"INSERT INTO {table} ({quoted_cols}) VALUES %s" + + # 冲突处理模式: + # "nothing" - 跳过已存在记录 (DO NOTHING) + # "backfill" - 只回填 NULL 列 (COALESCE) + # "update" - 全字段对比更新 (覆盖所有变化的字段) + conflict_mode = str(self.config.get("run.ods_conflict_mode", "update")).lower() + + # 兼容旧配置 + if self.config.get("run.ods_backfill_null_columns") is False: + conflict_mode = "nothing" + if pk_cols: pk_clause = ", ".join(f'\"{c}\"' for c in pk_cols) - sql += f" ON CONFLICT ({pk_clause}) DO NOTHING" + + if conflict_mode in ("backfill", "update"): + # 排除主键列和元数据列 + meta_cols = {"payload", "source_file", "source_endpoint", "fetched_at", "content_hash"} + pk_cols_lower = {c.lower() for c in pk_cols} + update_cols = [ + c for c in col_names + if c.lower() not in pk_cols_lower and c.lower() not in meta_cols + ] + + if update_cols: + if conflict_mode == "backfill": + # 回填模式：只填充 NULL 列 + set_clause = ", ".join( + f'"{c}" = COALESCE({table}."{c}", EXCLUDED."{c}")' + for c in update_cols + ) + where_clause = " OR ".join(f'{table}."{c}" IS NULL' for c in update_cols) + sql += f" ON CONFLICT ({pk_clause}) DO UPDATE SET {set_clause} WHERE {where_clause}" + else: + # update 模式：全字段对比更新 + set_clause = ", ".join( + f'"{c}" = EXCLUDED."{c}"' + for c in update_cols + ) + # 只在有字段变化时才更新 + where_clause = " OR ".join( + f'{table}."{c}" IS DISTINCT FROM EXCLUDED."{c}"' + for c in update_cols + ) + sql += f" ON CONFLICT ({pk_clause}) DO UPDATE SET {set_clause} WHERE {where_clause}" + else: + sql += f" ON CONFLICT ({pk_clause}) DO NOTHING" + else: + sql += f" ON CONFLICT ({pk_clause}) DO NOTHING" now = datetime.now(self.tz) json_dump = lambda v: json.dumps(v, ensure_ascii=False) # noqa: E731 @@ -499,6 +546,14 @@ class BaseOdsTask(BaseTask): if value is None: return None dt = (data_type or "").lower() + if dt == "boolean": + if isinstance(value, bool): + return value + if isinstance(value, (int, float)): + return bool(value) + if isinstance(value, str): + return value.lower() in ("true", "1", "yes", "t") + return bool(value) if dt in ("integer", "bigint", "smallint"): if isinstance(value, bool): return int(value) diff --git a/etl_billiards/tasks/seed_dws_config_task.py b/etl_billiards/tasks/seed_dws_config_task.py new file mode 100644 index 0000000..1dfa7f2 --- /dev/null +++ b/etl_billiards/tasks/seed_dws_config_task.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +""" +DWS配置数据初始化任务 + +功能说明： + 执行 seed_dws_config.sql，向配置表插入初始数据 + +执行前提： + - billiards_dws schema 已创建（INIT_DWS_SCHEMA） + - 配置表已存在 + +作者：ETL团队 +创建日期：2026-02-01 +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from .base_task import BaseTask, TaskContext + + +class SeedDwsConfigTask(BaseTask): + """ + DWS配置数据初始化任务 + + 执行 seed_dws_config.sql 文件，向以下配置表插入初始数据： + - cfg_performance_tier: 绩效档位配置 + - cfg_assistant_level_price: 助教等级定价 + - cfg_bonus_rules: 奖金规则配置 + - cfg_area_category: 台区分类映射 + - cfg_skill_type: 技能课程类型映射 + """ + + def get_task_code(self) -> str: + return "SEED_DWS_CONFIG" + + def extract(self, context: TaskContext) -> dict[str, Any]: + """ + 读取配置数据SQL文件 + """ + base_dir = Path(__file__).resolve().parents[1] / "database" + seed_path = Path(self.config.get("schema.seed_dws_file", base_dir / "seed_dws_config.sql")) + + if not seed_path.exists(): + raise FileNotFoundError(f"未找到 DWS 配置数据文件: {seed_path}") + + return { + "seed_sql": seed_path.read_text(encoding="utf-8"), + "seed_file": str(seed_path) + } + + def load(self, extracted: dict[str, Any], context: TaskContext) -> dict: + """ + 执行配置数据SQL + """ + with self.db.conn.cursor() as cur: + self.logger.info("执行 DWS 配置数据文件: %s", extracted["seed_file"]) + cur.execute(extracted["seed_sql"]) + + self.logger.info("DWS 配置数据初始化完成") + return {"executed": 1, "files": [extracted["seed_file"]]} diff --git a/etl_billiards/tests/test_dws_tasks.py b/etl_billiards/tests/test_dws_tasks.py new file mode 100644 index 0000000..5c27023 --- /dev/null +++ b/etl_billiards/tests/test_dws_tasks.py @@ -0,0 +1,472 @@ +# -*- coding: utf-8 -*- +""" +DWS任务单元测试 + +测试内容： +- BaseDwsTask基类方法 +- 时间计算方法 +- 配置应用方法 +- 排名计算方法 +""" + +import pytest +from datetime import date, datetime, timedelta +from decimal import Decimal +from unittest.mock import MagicMock, patch + +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from etl_billiards.tasks.dws.base_dws_task import ( + BaseDwsTask, + TimeLayer, + TimeWindow, + CourseType, + TimeRange, + ConfigCache +) +from etl_billiards.tasks.dws.finance_daily_task import FinanceDailyTask +from etl_billiards.tasks.dws.assistant_monthly_task import AssistantMonthlyTask + + +class TestTimeLayerRange: + """测试时间分层范围计算""" + + def test_last_2_days(self): + """测试近2天""" + base_date = date(2026, 2, 1) + # 创建一个模拟的BaseDwsTask实例 + task = create_mock_task() + + result = task.get_time_layer_range(TimeLayer.LAST_2_DAYS, base_date) + + assert result.start == date(2026, 1, 31) + assert result.end == date(2026, 2, 1) + + def test_last_1_month(self): + """测试近1月""" + base_date = date(2026, 2, 1) + task = create_mock_task() + + result = task.get_time_layer_range(TimeLayer.LAST_1_MONTH, base_date) + + assert result.start == date(2026, 1, 2) + assert result.end == date(2026, 2, 1) + + def test_last_3_months(self): + """测试近3月""" + base_date = date(2026, 2, 1) + task = create_mock_task() + + result = task.get_time_layer_range(TimeLayer.LAST_3_MONTHS, base_date) + + assert result.start == date(2025, 11, 3) + assert result.end == date(2026, 2, 1) + + +class TestTimeWindowRange: + """测试时间窗口范围计算""" + + def test_this_week_monday_start(self): + """测试本周（周一起始）""" + # 2026-02-01 是周日 + base_date = date(2026, 2, 1) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.THIS_WEEK, base_date) + + # 本周一是 2026-01-26 + assert result.start == date(2026, 1, 26) + assert result.end == date(2026, 2, 1) + + def test_last_week(self): + """测试上周""" + base_date = date(2026, 2, 1) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.LAST_WEEK, base_date) + + # 上周一是 2026-01-19，上周日是 2026-01-25 + assert result.start == date(2026, 1, 19) + assert result.end == date(2026, 1, 25) + + def test_this_month(self): + """测试本月""" + base_date = date(2026, 2, 15) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.THIS_MONTH, base_date) + + assert result.start == date(2026, 2, 1) + assert result.end == date(2026, 2, 15) + + def test_last_month(self): + """测试上月""" + base_date = date(2026, 2, 15) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.LAST_MONTH, base_date) + + assert result.start == date(2026, 1, 1) + assert result.end == date(2026, 1, 31) + + def test_last_3_months_excl_current(self): + """测试前3个月（不含本月）""" + base_date = date(2026, 2, 15) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.LAST_3_MONTHS_EXCL_CURRENT, base_date) + + assert result.start == date(2025, 11, 1) + assert result.end == date(2026, 1, 31) + + def test_last_3_months_incl_current(self): + """测试前3个月（含本月）""" + base_date = date(2026, 2, 15) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.LAST_3_MONTHS_INCL_CURRENT, base_date) + + assert result.start == date(2025, 12, 1) + assert result.end == date(2026, 2, 15) + + def test_this_quarter(self): + """测试本季度""" + base_date = date(2026, 2, 15) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.THIS_QUARTER, base_date) + + assert result.start == date(2026, 1, 1) + assert result.end == date(2026, 2, 15) + + def test_last_6_months(self): + """测试最近半年（不含本月）""" + base_date = date(2026, 2, 15) + task = create_mock_task() + + result = task.get_time_window_range(TimeWindow.LAST_6_MONTHS, base_date) + + # 不含本月，从上月末往前6个月 + assert result.end == date(2026, 1, 31) + assert result.start == date(2025, 8, 1) + + +class TestComparisonRange: + """测试环比区间计算""" + + def test_comparison_7_days(self): + """测试7天环比""" + task = create_mock_task() + current = TimeRange(start=date(2026, 2, 1), end=date(2026, 2, 7)) + + result = task.get_comparison_range(current) + + # 上一个7天：1月25日-1月31日 + assert result.start == date(2026, 1, 25) + assert result.end == date(2026, 1, 31) + + def test_comparison_30_days(self): + """测试30天环比""" + task = create_mock_task() + current = TimeRange(start=date(2026, 2, 1), end=date(2026, 3, 2)) + + result = task.get_comparison_range(current) + + # 上一个30天区间 + assert (result.end - result.start).days == (current.end - current.start).days + + +class TestFinanceDailyRecord: + """测试财务日度记录计算""" + + def test_groupbuy_and_cashflow(self): + """测试团购优惠与现金流口径""" + task = create_finance_daily_task() + stat_date = date(2026, 2, 1) + + settle = { + 'gross_amount': Decimal('1000'), + 'table_fee_amount': Decimal('1000'), + 'goods_amount': Decimal('0'), + 'assistant_pd_amount': Decimal('0'), + 'assistant_cx_amount': Decimal('0'), + 'cash_pay_amount': Decimal('300'), + 'card_pay_amount': Decimal('0'), + 'balance_pay_amount': Decimal('0'), + 'gift_card_pay_amount': Decimal('0'), + 'coupon_amount': Decimal('200'), + 'pl_coupon_sale_amount': Decimal('0'), + 'adjust_amount': Decimal('50'), + 'member_discount_amount': Decimal('10'), + 'rounding_amount': Decimal('0'), + 'order_count': 1, + 'member_order_count': 1, + 'guest_order_count': 0, + } + groupbuy = {'groupbuy_pay_total': Decimal('80')} + recharge = {'recharge_cash': Decimal('20')} + expense = {'expense_amount': Decimal('40')} + platform = { + 'settlement_amount': Decimal('60'), + 'commission_amount': Decimal('5'), + 'service_fee': Decimal('5'), + } + big_customer = {'big_customer_amount': Decimal('20')} + + record = task._build_daily_record( + stat_date, settle, groupbuy, recharge, expense, platform, big_customer, 1 + ) + + assert record['discount_groupbuy'] == Decimal('120') + assert record['discount_other'] == Decimal('30') + assert record['platform_settlement_amount'] == Decimal('60') + assert record['platform_fee_amount'] == Decimal('10') + assert record['cash_inflow_total'] == Decimal('380') + assert record['cash_outflow_total'] == Decimal('50') + assert record['cash_balance_change'] == Decimal('330') + + +class TestNewHireTier: + """测试新入职定档规则""" + + def test_new_hire_tier_hours(self): + """测试日均*30折算""" + task = create_assistant_monthly_task() + effective_hours = Decimal('15') + work_days = 5 + result = task._calc_new_hire_tier_hours(effective_hours, work_days) + assert result == Decimal('90') + + def test_max_tier_level_cap(self): + """测试新入职定档上限""" + task = create_mock_task() + now = datetime.now() + task._config_cache = ConfigCache( + performance_tiers=[ + {'tier_id': 1, 'tier_level': 1, 'min_hours': 0, 'max_hours': 100, 'is_new_hire_tier': False, 'effective_from': date(2020, 1, 1), 'effective_to': date(2099, 1, 1)}, + {'tier_id': 2, 'tier_level': 2, 'min_hours': 100, 'max_hours': 200, 'is_new_hire_tier': False, 'effective_from': date(2020, 1, 1), 'effective_to': date(2099, 1, 1)}, + {'tier_id': 3, 'tier_level': 3, 'min_hours': 200, 'max_hours': 300, 'is_new_hire_tier': False, 'effective_from': date(2020, 1, 1), 'effective_to': date(2099, 1, 1)}, + {'tier_id': 4, 'tier_level': 4, 'min_hours': 300, 'max_hours': None, 'is_new_hire_tier': False, 'effective_from': date(2020, 1, 1), 'effective_to': date(2099, 1, 1)}, + ], + level_prices=[], + bonus_rules=[], + area_categories={}, + skill_types={}, + loaded_at=now + ) + + tier = task.get_performance_tier( + Decimal('350'), + is_new_hire=True, + effective_date=date(2026, 2, 1), + max_tier_level=3 + ) + assert tier['tier_level'] == 3 + + +class TestNewHireCheck: + """测试新入职判断""" + + def test_new_hire_in_month(self): + """测试月内入职为新入职""" + task = create_mock_task() + hire_date = date(2026, 2, 5) + stat_month = date(2026, 2, 1) + + assert task.is_new_hire_in_month(hire_date, stat_month) == True + + def test_not_new_hire(self): + """测试月前入职不是新入职""" + task = create_mock_task() + hire_date = date(2026, 1, 15) + stat_month = date(2026, 2, 1) + + assert task.is_new_hire_in_month(hire_date, stat_month) == False + + def test_hire_on_first_day(self): + """测试月1日入职为新入职""" + task = create_mock_task() + hire_date = date(2026, 2, 1) + stat_month = date(2026, 2, 1) + + assert task.is_new_hire_in_month(hire_date, stat_month) == True + + +class TestRankWithTies: + """测试考虑并列的排名计算""" + + def test_no_ties(self): + """测试无并列情况""" + task = create_mock_task() + values = [ + (1, Decimal('100')), + (2, Decimal('90')), + (3, Decimal('80')), + ] + + result = task.calculate_rank_with_ties(values) + + assert result[0] == (1, 1, 1) # 第1名 + assert result[1] == (2, 2, 2) # 第2名 + assert result[2] == (3, 3, 3) # 第3名 + + def test_with_ties(self): + """测试有并列情况""" + task = create_mock_task() + values = [ + (1, Decimal('100')), + (2, Decimal('100')), # 并列第1 + (3, Decimal('80')), + ] + + result = task.calculate_rank_with_ties(values) + + # 两个第1，下一个是第3 + assert result[0][1] == 1 # 第1名 + assert result[1][1] == 1 # 并列第1名 + assert result[2][1] == 3 # 第3名（跳过2） + + def test_all_ties(self): + """测试全部并列""" + task = create_mock_task() + values = [ + (1, Decimal('100')), + (2, Decimal('100')), + (3, Decimal('100')), + ] + + result = task.calculate_rank_with_ties(values) + + # 全部第1 + assert all(r[1] == 1 for r in result) + + +class TestGuestCheck: + """测试散客判断""" + + def test_guest_zero(self): + """测试member_id=0为散客""" + task = create_mock_task() + assert task.is_guest(0) == True + + def test_guest_none(self): + """测试member_id=None为散客""" + task = create_mock_task() + assert task.is_guest(None) == True + + def test_not_guest(self): + """测试正常会员不是散客""" + task = create_mock_task() + assert task.is_guest(12345) == False + + +class TestUtilityMethods: + """测试工具方法""" + + def test_safe_decimal(self): + """测试安全Decimal转换""" + task = create_mock_task() + + assert task.safe_decimal(100) == Decimal('100') + assert task.safe_decimal('123.45') == Decimal('123.45') + assert task.safe_decimal(None) == Decimal('0') + assert task.safe_decimal('invalid') == Decimal('0') + + def test_safe_int(self): + """测试安全int转换""" + task = create_mock_task() + + assert task.safe_int(100) == 100 + assert task.safe_int('123') == 123 + assert task.safe_int(None) == 0 + assert task.safe_int('invalid') == 0 + + def test_seconds_to_hours(self): + """测试秒转小时""" + task = create_mock_task() + + assert task.seconds_to_hours(3600) == Decimal('1') + assert task.seconds_to_hours(5400) == Decimal('1.5') + assert task.seconds_to_hours(0) == Decimal('0') + + def test_hours_to_seconds(self): + """测试小时转秒""" + task = create_mock_task() + + assert task.hours_to_seconds(Decimal('1')) == 3600 + assert task.hours_to_seconds(Decimal('1.5')) == 5400 + + +class TestCourseType: + """测试课程类型""" + + def test_base_course(self): + """测试基础课""" + assert CourseType.BASE.value == 'BASE' + + def test_bonus_course(self): + """测试附加课""" + assert CourseType.BONUS.value == 'BONUS' + + +# ============================================================================= +# 辅助函数 +# ============================================================================= + +def create_mock_task(): + """ + 创建一个模拟的BaseDwsTask实例用于测试 + """ + # 创建一个具体的子类用于测试 + class TestDwsTask(BaseDwsTask): + def get_task_code(self): + return "TEST_DWS_TASK" + + def get_target_table(self): + return "test_table" + + def get_primary_keys(self): + return ["id"] + + def extract(self, context): + return {} + + def load(self, transformed, context): + return {} + + # 创建模拟的依赖 + mock_config = MagicMock() + mock_config.get.return_value = None + + mock_db = MagicMock() + mock_api = MagicMock() + mock_logger = MagicMock() + + task = TestDwsTask(mock_config, mock_db, mock_api, mock_logger) + return task + + +def create_finance_daily_task(): + """创建 FinanceDailyTask 实例用于测试""" + mock_config = MagicMock() + mock_config.get.side_effect = lambda key, default=None: 1 if key == "app.tenant_id" else default + mock_db = MagicMock() + mock_api = MagicMock() + mock_logger = MagicMock() + return FinanceDailyTask(mock_config, mock_db, mock_api, mock_logger) + + +def create_assistant_monthly_task(): + """创建 AssistantMonthlyTask 实例用于测试""" + mock_config = MagicMock() + mock_config.get.side_effect = lambda key, default=None: default + mock_db = MagicMock() + mock_api = MagicMock() + mock_logger = MagicMock() + return AssistantMonthlyTask(mock_config, mock_db, mock_api, mock_logger) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/scripts/backfill_202507_to_now.bat b/scripts/backfill_202507_to_now.bat new file mode 100644 index 0000000..2449165 --- /dev/null +++ b/scripts/backfill_202507_to_now.bat @@ -0,0 +1,45 @@ +@echo off +REM 数据补全脚本：从 2025-07-01 到当前日期 +REM 用法：直接运行此脚本 + +cd /d %~dp0.. +echo ==================================== +echo 数据补全：2025-07-01 到现在 +echo ==================================== +echo. +echo 开始时间：%date% %time% +echo. + +REM 设置时间窗口 +set WINDOW_START=2025-07-01 00:00:00 +set WINDOW_END=2026-02-03 00:00:00 + +echo 时间窗口：%WINDOW_START% ~ %WINDOW_END% +echo. + +REM 执行 ODS 抓取（分月执行以避免超时） +echo [1/2] 开始 ODS 数据抓取... +python -m etl_billiards.cli.main --pipeline-flow FULL --window-start "%WINDOW_START%" --window-end "%WINDOW_END%" --window-split-unit month --force-window-override + +if %ERRORLEVEL% NEQ 0 ( + echo [错误] ODS 抓取失败！ + pause + exit /b 1 +) + +echo. +echo [2/2] 开始 DWD 装载... +python -m etl_billiards.cli.main --pipeline-flow INGEST_ONLY --tasks DWD_LOAD_FROM_ODS --window-start "%WINDOW_START%" --window-end "%WINDOW_END%" --force-window-override + +if %ERRORLEVEL% NEQ 0 ( + echo [错误] DWD 装载失败！ + pause + exit /b 1 +) + +echo. +echo ==================================== +echo 数据补全完成！ +echo 结束时间：%date% %time% +echo ==================================== +pause diff --git a/tmp/Untitled b/tmp/Untitled new file mode 100644 index 0000000..a0bc186 --- /dev/null +++ b/tmp/Untitled @@ -0,0 +1,94 @@ +# DWS 数据层需求 +## 简介 +项目路径：C:\dev\LLTQ\ETL\feiqiu-ETL + +本文档描述在ETL已完成的DWD层数据基础上对DWS层的数据处理： +- 完成对DWS层数据库的处理，即数据库设计，成果为DDL的SQL语句。 +- 数据读取处理到落库，即DWD读取，Python处理，SQL写入。 + +文档更多聚焦业务描述，你需要使用专业技能，使用面向对象编程OOP思想，完成程序设计直至代码完成： +- 参考.\README.md 了解现在项目现状。 +- 参考.\etl_billiards\docs 了解 DWD的schema的表和字段。 +- SQL和Python代码需要详尽的，高密度的中文注释。 +- 完成内容，需要详尽高密度的补充至.\README.md，以方便后续维护。 +- DWS的表与表的字段 参考.\etl_billiards\docs\dwd_main_tables_dictionary.md 完成类似的数据库文档，方便后续维护。 +- 注意中文编码需求。 + +## 通用需求 +### 数据分层 +我希望使用互联网软件的业内通用方法，将数据按照更新时间分为4层，以符合业务层面的查询效率速度。 +- 第一层：回溯两天前到当前数据。 +- 第二层：回溯1个月前到当前数据。 +- 第三层：回溯3个月前到当前数据。 +- 第四层：全量数据。 +- 需要有配套的机制及时添加删除整理数据。 + +### 统计注意 +当统计一些数据时，注意口径，数据有效性标识。举例： +- 计算助教业绩/工资时，需要参考助教废除表，相关业务数据的影响。 +- 计算助教业绩/工资时，注意辨别 助教课 附加课影响。 + +## 业务需求 +### 系统设置 +- 助教新的绩效考核和工资结算方式更新为以下算法，影响工资结算和财务账务方面的统计核算，相关内容需要落库，以方便后续调整。还要标记执行时间（如哪个月执行哪个标准等），执行相关结算和计算逻辑。： +档位原因考虑 总业绩小时数阈值 专业课抽成（元/小时） 打赏课抽成 次月休假（天） +0档 淘汰压力 H <100 28 50% 3 +1档 及格档（重点激励） 100≤ H <130 18 40% 4 +2档 良好档（重点激励） 130≤ H <160 15 38% 4 +3档 优秀档 160≤ H <190 13 35% 5 +4档 卓越加速档（高端人才倾斜） 190≤ H <220 10 33% 6 +5档 冠军加速档（高端人才倾斜） H ≥220 8 30% 休假自由 + +*课程分为2种（dwd_assistant_service_log表的skill_name）： +基础课：又名 专业课 上桌 上钟，是为客户提供台球助教陪练的课程，按时长统计。精确到分钟。 +附加课：又名 超休 激励 打赏，是客户支付较为高昂的价格，买断整小时与助教外出。 +总业绩小时数阈值指基础课和附加课总和。 + +各级别助教（dim_assistant表的level）基础课，对客户收费：初级 98元/小时；中级 108元/小时；高级 118元/小时；星级 138元/小时； +附加课对客户收费统一为190元/小时。 + +充值提成： + + + + +冲刺奖 达成奖金 +当月 H ≥ 190：300 元 +当月 H ≥ 220：800 元（与上条不叠加，取高） + +额外奖金： +冲刺奖 达成奖金 +当月 H ≥ 190：300 元 +当月 H ≥ 220：800 元（与上条不叠加，取高） + +Top3 奖金： +第1名：1000 元 +第2名：600 元 +第3名：400 元 + +规则： +1、过档后，所有时长按新档位进行计算。 +举例，当前某中级助教已完成185小时，基础课占170小时，附加课15小时。则该月工资计算方法： +170*（108-13）+15*（1-0.35） + +2、本月新入职助教，定档方案： + 按照日均*30的总业绩小时数定档。 + 在该25日之后入职的新助教，最高定档至3档。 + 该折算仅用于定档，不适用于“冲刺奖”和“Top3奖”的计算口径。 + +### 助教维度 +以每个助教个体的视角 +- 我要知道我的业绩档位，历史月份与本月档位进度，档位影响的收入单价。及相邻月份的变化。 +- 我要知道我的有效业绩：历史月份与本月的 基础课课时，激励课课时，全部课课时。相邻月份的变化。 +- 我要知道我的收入：历史月份与本月的收入（注意助教等级，业绩档位，课程种类等因素的总和计算）。相邻月份的变化。 +- 我要知道我的客户情况：过去7天、10天、15天、30天、60天、90天 的跨度进行统计，我服务过（基础课+附加课）的客户数据，并关联每次服务的 时间 时长 台桌 分类 等详细信息。 + +### 客户维度 +统计每个客户的信息 +- 我要知道每个客户：过去7天、10天、15天、30天、60天、90天 的跨度进行统计，来店消费情况，并关联每次服务的 时间 食品饮品 时长 台桌 分类 助教服务 等详细信息。 + + +### 财务维度 +财务维度的需求（已经落到原型图需求级别了），见财务页面需求.md + + diff --git a/tmp/add_missing_dwd_columns.py b/tmp/add_missing_dwd_columns.py new file mode 100644 index 0000000..553ddbe --- /dev/null +++ b/tmp/add_missing_dwd_columns.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +""" +添加缺失的 DWD 列到数据库 +根据 ODS 新增字段，在对应的 DWD 表中添加相关列 +""" +import psycopg2 + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +# DWD 表缺失字段定义：表名 -> [(列名, 类型, 注释)] +# 根据计划，核心业务字段放主表，扩展字段放 _ex 表 +MISSING_COLUMNS = { + # 结算表 - 核心金额字段放主表 + 'billiards_dwd.dwd_settlement_head': [ + ('electricity_money', 'NUMERIC(18,2)', '电费金额'), + ('real_electricity_money', 'NUMERIC(18,2)', '实际电费金额'), + ('electricity_adjust_money', 'NUMERIC(18,2)', '电费调整金额'), + ('pl_coupon_sale_amount', 'NUMERIC(18,2)', '平台券销售额'), + ('mervou_sales_amount', 'NUMERIC(18,2)', '商户券销售额'), + ], + 'billiards_dwd.dwd_settlement_head_ex': [ + ('settle_list', 'JSONB', '结算明细列表'), + ], + + # 台费流水表 + 'billiards_dwd.dwd_table_fee_log': [ + ('activity_discount_amount', 'NUMERIC(18,2)', '活动折扣金额'), + ('real_service_money', 'NUMERIC(18,2)', '实际服务费金额'), + ], + 'billiards_dwd.dwd_table_fee_log_ex': [ + ('order_consumption_type', 'INT', '订单消费类型'), + ], + + # 助教服务流水表 + 'billiards_dwd.dwd_assistant_service_log': [ + ('real_service_money', 'NUMERIC(18,2)', '实际服务费金额'), + ], + 'billiards_dwd.dwd_assistant_service_log_ex': [ + ('assistant_team_name', 'TEXT', '助教团队名称'), + ], + + # 团购核销记录表 + 'billiards_dwd.dwd_groupbuy_redemption': [ + ('member_discount_money', 'NUMERIC(18,2)', '会员折扣金额'), + ('coupon_sale_id', 'BIGINT', '优惠券销售ID'), + ], + 'billiards_dwd.dwd_groupbuy_redemption_ex': [ + ('table_share_money', 'NUMERIC(18,2)', '台费分摊金额'), + ('table_service_share_money', 'NUMERIC(18,2)', '台费服务分摊金额'), + ('goods_share_money', 'NUMERIC(18,2)', '商品分摊金额'), + ('good_service_share_money', 'NUMERIC(18,2)', '商品服务分摊金额'), + ('assistant_share_money', 'NUMERIC(18,2)', '助教分摊金额'), + ('assistant_service_share_money', 'NUMERIC(18,2)', '助教服务分摊金额'), + ('recharge_share_money', 'NUMERIC(18,2)', '充值分摊金额'), + ], + + # 台费调整记录表 + 'billiards_dwd.dwd_table_fee_adjust': [ + ('table_name', 'TEXT', '台桌名称'), + ('table_price', 'NUMERIC(18,2)', '台桌价格'), + ('charge_free', 'BOOLEAN', '是否免费'), + ], + 'billiards_dwd.dwd_table_fee_adjust_ex': [ + ('area_type_id', 'BIGINT', '区域类型ID'), + ('site_table_area_id', 'BIGINT', '门店台区ID'), + ('site_table_area_name', 'TEXT', '门店台区名称'), + ('site_name', 'TEXT', '门店名称'), + ('tenant_name', 'TEXT', '租户名称'), + ], + + # 会员储值卡维度表 + 'billiards_dwd.dim_member_card_account': [ + ('principal_balance', 'NUMERIC(18,2)', '本金余额'), + ('member_grade', 'INT', '会员等级'), + ], + 'billiards_dwd.dim_member_card_account_ex': [ + ('able_share_member_discount', 'BOOLEAN', '是否可共享会员折扣'), + ('electricity_deduct_radio', 'NUMERIC(18,4)', '电费扣减比例'), + ('electricity_discount', 'NUMERIC(18,4)', '电费折扣'), + ('electricity_card_deduct', 'BOOLEAN', '电费卡扣'), + ('recharge_freeze_balance', 'NUMERIC(18,2)', '充值冻结余额'), + ], + + # 会员维度表 + 'billiards_dwd.dim_member': [ + ('pay_money_sum', 'NUMERIC(18,2)', '累计支付金额'), + ('recharge_money_sum', 'NUMERIC(18,2)', '累计充值金额'), + ], + 'billiards_dwd.dim_member_ex': [ + ('person_tenant_org_id', 'BIGINT', '人员租户组织ID'), + ('person_tenant_org_name', 'TEXT', '人员租户组织名称'), + ('register_source', 'TEXT', '注册来源'), + ], + + # 会员余额变更表 + 'billiards_dwd.dwd_member_balance_change': [ + ('principal_before', 'NUMERIC(18,2)', '变动前本金'), + ('principal_after', 'NUMERIC(18,2)', '变动后本金'), + ], + 'billiards_dwd.dwd_member_balance_change_ex': [ + ('principal_data', 'TEXT', '本金变动数据'), + ], + + # 团购套餐维度表 + 'billiards_dwd.dim_groupbuy_package': [ + ('sort', 'INT', '排序'), + ('is_first_limit', 'BOOLEAN', '是否首单限制'), + ], + 'billiards_dwd.dim_groupbuy_package_ex': [ + ('tenant_coupon_sale_order_item_id', 'BIGINT', '租户券销售订单项ID'), + ], + + # 门店商品维度表 + 'billiards_dwd.dim_store_goods': [ + ('commodity_code', 'TEXT', '商品编码'), + ('not_sale', 'BOOLEAN', '是否停售'), + ], + + # 台桌维度表 + 'billiards_dwd.dim_table': [ + ('order_id', 'BIGINT', '订单ID'), + ], + + # 租户商品维度表 + 'billiards_dwd.dim_tenant_goods': [ + ('not_sale', 'BOOLEAN', '是否停售'), + ], + + # 助教作废记录表 + 'billiards_dwd.dwd_assistant_cancel_log': [ + ('tenant_id', 'BIGINT', '租户ID'), + ], + + # 商品销售流水表 + 'billiards_dwd.dwd_goods_sale_log': [ + ('coupon_share_money', 'NUMERIC(18,2)', '优惠券分摊金额'), + ], + + # 支付流水表 + 'billiards_dwd.dwd_payment': [ + ('tenant_id', 'BIGINT', '租户ID'), + ], +} + + +def get_existing_columns(conn, schema, table): + """获取表已有的列""" + sql = """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + """ + with conn.cursor() as cur: + cur.execute(sql, (schema, table)) + return {row[0].lower() for row in cur.fetchall()} + + +def table_exists(conn, schema, table): + """检查表是否存在""" + sql = """ + SELECT EXISTS ( + SELECT 1 FROM information_schema.tables + WHERE table_schema = %s AND table_name = %s + ) + """ + with conn.cursor() as cur: + cur.execute(sql, (schema, table)) + return cur.fetchone()[0] + + +def add_column(conn, full_table, col_name, col_type, comment): + """添加列""" + sql = f'ALTER TABLE {full_table} ADD COLUMN IF NOT EXISTS "{col_name}" {col_type}' + comment_sql = f"COMMENT ON COLUMN {full_table}.\"{col_name}\" IS '{comment}'" + + with conn.cursor() as cur: + cur.execute(sql) + cur.execute(comment_sql) + conn.commit() + print(f" [OK] 添加列: {col_name} ({col_type})") + + +def main(): + conn = psycopg2.connect(DSN) + + print("=" * 80) + print("添加缺失的 DWD 列") + print("=" * 80) + + total_added = 0 + total_skipped = 0 + tables_not_found = [] + + for full_table, columns in MISSING_COLUMNS.items(): + schema, table = full_table.split('.') + + if not table_exists(conn, schema, table): + print(f"\n[跳过] 表不存在: {full_table}") + tables_not_found.append(full_table) + continue + + print(f"\n处理表: {full_table}") + + existing = get_existing_columns(conn, schema, table) + + for col_name, col_type, comment in columns: + if col_name.lower() in existing: + print(f" [跳过] 列已存在: {col_name}") + total_skipped += 1 + else: + add_column(conn, full_table, col_name, col_type, comment) + total_added += 1 + + conn.close() + + print("\n" + "=" * 80) + print(f"完成: 添加 {total_added} 列, 跳过 {total_skipped} 列") + if tables_not_found: + print(f"未找到的表: {len(tables_not_found)}") + for t in tables_not_found: + print(f" - {t}") + print("=" * 80) + + +if __name__ == '__main__': + main() diff --git a/tmp/add_missing_ods_columns.py b/tmp/add_missing_ods_columns.py new file mode 100644 index 0000000..39a4871 --- /dev/null +++ b/tmp/add_missing_ods_columns.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +""" +添加缺失的 ODS 列到数据库 +""" +import psycopg2 + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +# 缺失字段定义：表名 -> [(列名, 类型, 注释)] +MISSING_COLUMNS = { + 'billiards_ods.settlement_records': [ + ('electricityadjustmoney', 'NUMERIC(18,2)', '电费调整金额'), + ('electricitymoney', 'NUMERIC(18,2)', '电费金额'), + ('mervousalesamount', 'NUMERIC(18,2)', '商户券销售额'), + ('plcouponsaleamount', 'NUMERIC(18,2)', '平台券销售额'), + ('realelectricitymoney', 'NUMERIC(18,2)', '实际电费金额'), + ('settlelist', 'JSONB', '结算明细列表'), + ], + 'billiards_ods.recharge_settlements': [ + ('electricityadjustmoney', 'NUMERIC(18,2)', '电费调整金额'), + ('electricitymoney', 'NUMERIC(18,2)', '电费金额'), + ('mervousalesamount', 'NUMERIC(18,2)', '商户券销售额'), + ('plcouponsaleamount', 'NUMERIC(18,2)', '平台券销售额'), + ('realelectricitymoney', 'NUMERIC(18,2)', '实际电费金额'), + ('settlelist', 'JSONB', '结算明细列表'), + ], + 'billiards_ods.table_fee_transactions': [ + ('activity_discount_amount', 'NUMERIC(18,2)', '活动折扣金额'), + ('order_consumption_type', 'INT', '订单消费类型'), + ('real_service_money', 'NUMERIC(18,2)', '实际服务费金额'), + ], + 'billiards_ods.assistant_service_records': [ + ('assistantteamname', 'TEXT', '助教团队名称'), + ('real_service_money', 'NUMERIC(18,2)', '实际服务费金额'), + ], + 'billiards_ods.group_buy_redemption_records': [ + ('assistant_service_share_money', 'NUMERIC(18,2)', '助教服务分摊金额'), + ('assistant_share_money', 'NUMERIC(18,2)', '助教分摊金额'), + ('coupon_sale_id', 'BIGINT', '优惠券销售ID'), + ('good_service_share_money', 'NUMERIC(18,2)', '商品服务分摊金额'), + ('goods_share_money', 'NUMERIC(18,2)', '商品分摊金额'), + ('member_discount_money', 'NUMERIC(18,2)', '会员折扣金额'), + ('recharge_share_money', 'NUMERIC(18,2)', '充值分摊金额'), + ('table_service_share_money', 'NUMERIC(18,2)', '台费服务分摊金额'), + ('table_share_money', 'NUMERIC(18,2)', '台费分摊金额'), + ], + 'billiards_ods.table_fee_discount_records': [ + ('area_type_id', 'BIGINT', '区域类型ID'), + ('charge_free', 'BOOLEAN', '是否免费'), + ('site_table_area_id', 'BIGINT', '门店台区ID'), + ('site_table_area_name', 'TEXT', '门店台区名称'), + ('sitename', 'TEXT', '门店名称'), + ('table_name', 'TEXT', '台桌名称'), + ('table_price', 'NUMERIC(18,2)', '台桌价格'), + ('tenant_name', 'TEXT', '租户名称'), + ], + 'billiards_ods.member_stored_value_cards': [ + ('able_share_member_discount', 'BOOLEAN', '是否可共享会员折扣'), + ('electricity_deduct_radio', 'NUMERIC(18,4)', '电费扣减比例'), + ('electricity_discount', 'NUMERIC(18,4)', '电费折扣'), + ('electricitycarddeduct', 'BOOLEAN', '电费卡扣'), + ('member_grade', 'INT', '会员等级'), + ('principal_balance', 'NUMERIC(18,2)', '本金余额'), + ('rechargefreezebalance', 'NUMERIC(18,2)', '充值冻结余额'), + ], + 'billiards_ods.member_profiles': [ + ('pay_money_sum', 'NUMERIC(18,2)', '累计支付金额'), + ('person_tenant_org_id', 'BIGINT', '人员租户组织ID'), + ('person_tenant_org_name', 'TEXT', '人员租户组织名称'), + ('recharge_money_sum', 'NUMERIC(18,2)', '累计充值金额'), + ('register_source', 'TEXT', '注册来源'), + ], + 'billiards_ods.member_balance_changes': [ + ('principal_after', 'NUMERIC(18,2)', '变动后本金'), + ('principal_before', 'NUMERIC(18,2)', '变动前本金'), + ('principal_data', 'TEXT', '本金变动数据'), + ], + 'billiards_ods.group_buy_packages': [ + ('is_first_limit', 'BOOLEAN', '是否首单限制'), + ('sort', 'INT', '排序'), + ('tenantcouponsaleorderitemid', 'BIGINT', '租户券销售订单项ID'), + ], + 'billiards_ods.store_goods_master': [ + ('commodity_code', 'TEXT', '商品编码'), + ('not_sale', 'BOOLEAN', '是否停售'), + ], + 'billiards_ods.assistant_cancellation_records': [ + ('tenant_id', 'BIGINT', '租户ID'), + ], + 'billiards_ods.store_goods_sales_records': [ + ('coupon_share_money', 'NUMERIC(18,2)', '优惠券分摊金额'), + ], + 'billiards_ods.payment_transactions': [ + ('tenant_id', 'BIGINT', '租户ID'), + ], + 'billiards_ods.site_tables_master': [ + ('order_id', 'BIGINT', '订单ID'), + ], + 'billiards_ods.tenant_goods_master': [ + ('not_sale', 'BOOLEAN', '是否停售'), + ], +} + + +def get_existing_columns(conn, schema, table): + """获取表已有的列""" + sql = """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + """ + with conn.cursor() as cur: + cur.execute(sql, (schema, table)) + return {row[0].lower() for row in cur.fetchall()} + + +def add_column(conn, full_table, col_name, col_type, comment): + """添加列""" + schema, table = full_table.split('.') + sql = f'ALTER TABLE {full_table} ADD COLUMN IF NOT EXISTS "{col_name}" {col_type}' + comment_sql = f"COMMENT ON COLUMN {full_table}.\"{col_name}\" IS '{comment}'" + + with conn.cursor() as cur: + cur.execute(sql) + cur.execute(comment_sql) + conn.commit() + print(f" [OK] 添加列: {col_name} ({col_type})") + + +def main(): + conn = psycopg2.connect(DSN) + + print("=" * 80) + print("添加缺失的 ODS 列") + print("=" * 80) + + total_added = 0 + total_skipped = 0 + + for full_table, columns in MISSING_COLUMNS.items(): + schema, table = full_table.split('.') + print(f"\n处理表: {full_table}") + + existing = get_existing_columns(conn, schema, table) + + for col_name, col_type, comment in columns: + if col_name.lower() in existing: + print(f" [跳过] 列已存在: {col_name}") + total_skipped += 1 + else: + add_column(conn, full_table, col_name, col_type, comment) + total_added += 1 + + conn.close() + + print("\n" + "=" * 80) + print(f"完成: 添加 {total_added} 列, 跳过 {total_skipped} 列") + print("=" * 80) + + +if __name__ == '__main__': + main() diff --git a/tmp/add_remaining_dwd_columns.py b/tmp/add_remaining_dwd_columns.py new file mode 100644 index 0000000..a264ce2 --- /dev/null +++ b/tmp/add_remaining_dwd_columns.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +""" +添加剩余的 DWD 列 +""" +import psycopg2 + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +# 修正后的表名 +MISSING_COLUMNS = { + 'billiards_dwd.dwd_assistant_trash_event': [ + ('tenant_id', 'BIGINT', '租户ID'), + ], + 'billiards_dwd.dwd_store_goods_sale': [ + ('coupon_share_money', 'NUMERIC(18,2)', '优惠券分摊金额'), + ], +} + +def add_column(conn, full_table, col_name, col_type, comment): + sql = f'ALTER TABLE {full_table} ADD COLUMN IF NOT EXISTS "{col_name}" {col_type}' + comment_sql = f"COMMENT ON COLUMN {full_table}.\"{col_name}\" IS '{comment}'" + with conn.cursor() as cur: + cur.execute(sql) + cur.execute(comment_sql) + conn.commit() + print(f" [OK] {full_table}.{col_name} ({col_type})") + +def main(): + conn = psycopg2.connect(DSN) + for full_table, columns in MISSING_COLUMNS.items(): + for col_name, col_type, comment in columns: + add_column(conn, full_table, col_name, col_type, comment) + conn.close() + print("Done!") + +if __name__ == '__main__': + main() diff --git a/tmp/api_ods_comparison.json b/tmp/api_ods_comparison.json new file mode 100644 index 0000000..41f7008 --- /dev/null +++ b/tmp/api_ods_comparison.json @@ -0,0 +1,2305 @@ +{ + "ODS_ASSISTANT_ACCOUNT": { + "table_name": "billiards_ods.assistant_accounts_master", + "endpoint": "/PersonnelManagement/SearchAssistantInfo", + "api_records": 50, + "api_fields_count": 62, + "ods_columns_count": 67, + "missing_in_ods": [], + "ods_only": [], + "api_fields": [ + "allow_cx", + "assistant_grade", + "assistant_no", + "assistant_status", + "avatar", + "birth_date", + "charge_way", + "create_time", + "criticism_status", + "cx_unit_price", + "ding_talk_synced", + "end_time", + "entry_sign_status", + "entry_time", + "entry_type", + "gender", + "get_grade_times", + "group_id", + "group_name", + "height", + "id", + "introduce", + "is_delete", + "is_guaranteed", + "is_team_leader", + "job_num", + "last_table_id", + "last_table_name", + "last_update_name", + "leave_status", + "level", + "light_equipment_id", + "light_status", + "mobile", + "nickname", + "online_status", + "order_trade_no", + "pd_unit_price", + "person_org_id", + "real_name", + "resign_sign_status", + "resign_time", + "salary_grant_enabled", + "serial_number", + "shop_name", + "show_sort", + "show_status", + "site_id", + "site_light_cfg_id", + "staff_id", + "staff_profile_id", + "start_time", + "sum_grade", + "system_role_id", + "team_id", + "team_name", + "tenant_id", + "update_time", + "user_id", + "video_introduction_url", + "weight", + "work_status" + ], + "ods_columns": [ + "allow_cx", + "assistant_grade", + "assistant_no", + "assistant_status", + "avatar", + "birth_date", + "charge_way", + "content_hash", + "create_time", + "criticism_status", + "cx_unit_price", + "ding_talk_synced", + "end_time", + "entry_sign_status", + "entry_time", + "entry_type", + "fetched_at", + "gender", + "get_grade_times", + "group_id", + "group_name", + "height", + "id", + "introduce", + "is_delete", + "is_guaranteed", + "is_team_leader", + "job_num", + "last_table_id", + "last_table_name", + "last_update_name", + "leave_status", + "level", + "light_equipment_id", + "light_status", + "mobile", + "nickname", + "online_status", + "order_trade_no", + "payload", + "pd_unit_price", + "person_org_id", + "real_name", + "resign_sign_status", + "resign_time", + "salary_grant_enabled", + "serial_number", + "shop_name", + "show_sort", + "show_status", + "site_id", + "site_light_cfg_id", + "source_endpoint", + "source_file", + "staff_id", + "staff_profile_id", + "start_time", + "sum_grade", + "system_role_id", + "team_id", + "team_name", + "tenant_id", + "update_time", + "user_id", + "video_introduction_url", + "weight", + "work_status" + ] + }, + "ODS_SETTLEMENT_RECORDS": { + "table_name": "billiards_ods.settlement_records", + "endpoint": "/Site/GetAllOrderSettleList", + "api_records": 50, + "api_fields_count": 93, + "ods_columns_count": 66, + "missing_in_ods": [ + "electricityadjustmoney", + "electricitymoney", + "mervousalesamount", + "plcouponsaleamount", + "realelectricitymoney", + "settlelist", + "tenant_id" + ], + "ods_only": [], + "api_fields": [ + "activitydiscount", + "address", + "adjustamount", + "allcoupondiscount", + "assistantcxmoney", + "assistantmanualdiscount", + "assistantpdmoney", + "assistantpromotionmoney", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "balanceamount", + "business_tel", + "canberevoked", + "cardamount", + "cashamount", + "consumemoney", + "couponamount", + "couponsaleamount", + "createtime", + "customer_service_qrcode", + "customer_service_wechat", + "electricityadjustmoney", + "electricitymoney", + "fixed_pay_qrcode", + "full_address", + "giftcardamount", + "goodsmoney", + "goodspromotionmoney", + "id", + "isactivity", + "isbindmember", + "isfirst", + "isusecoupon", + "isusediscount", + "latitude", + "light_status", + "light_token", + "light_type", + "longitude", + "membercardtypename", + "memberdiscountamount", + "memberid", + "membername", + "memberphone", + "mervousalesamount", + "onlineamount", + "operatorid", + "operatorname", + "orderremark", + "org_id", + "payamount", + "paymentmethod", + "paytime", + "plcouponsaleamount", + "pointamount", + "pointdiscountcost", + "pointdiscountprice", + "prepaymoney", + "prod_env", + "realelectricitymoney", + "realgoodsmoney", + "rechargecardamount", + "refundamount", + "revokeorderid", + "revokeordername", + "revoketime", + "roundingamount", + "salesmanname", + "salesmanuserid", + "serialnumber", + "servicemoney", + "settlelist", + "settlename", + "settlerelateid", + "settlestatus", + "settletype", + "shop_name", + "shop_status", + "site_label", + "site_type", + "siteid", + "sitename", + "siteprofile", + "tablechargemoney", + "tableid", + "tenant_id", + "tenant_site_region_id", + "tenantid", + "tenantmembercardid", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "activitydiscount", + "adjustamount", + "allcoupondiscount", + "assistantcxmoney", + "assistantmanualdiscount", + "assistantpdmoney", + "assistantpromotionmoney", + "balanceamount", + "canberevoked", + "cardamount", + "cashamount", + "consumemoney", + "content_hash", + "couponamount", + "couponsaleamount", + "createtime", + "fetched_at", + "giftcardamount", + "goodsmoney", + "goodspromotionmoney", + "id", + "isactivity", + "isbindmember", + "isfirst", + "isusecoupon", + "isusediscount", + "membercardtypename", + "memberdiscountamount", + "memberid", + "membername", + "memberphone", + "onlineamount", + "operatorid", + "operatorname", + "orderremark", + "payamount", + "payload", + "paymentmethod", + "paytime", + "pointamount", + "pointdiscountcost", + "pointdiscountprice", + "prepaymoney", + "realgoodsmoney", + "rechargecardamount", + "refundamount", + "revokeorderid", + "revokeordername", + "revoketime", + "roundingamount", + "salesmanname", + "salesmanuserid", + "serialnumber", + "servicemoney", + "settlename", + "settlerelateid", + "settlestatus", + "settletype", + "siteid", + "sitename", + "source_endpoint", + "source_file", + "tablechargemoney", + "tableid", + "tenantid", + "tenantmembercardid" + ] + }, + "ODS_TABLE_USE": { + "table_name": "billiards_ods.table_fee_transactions", + "endpoint": "/Site/GetSiteTableOrderDetails", + "api_records": 50, + "api_fields_count": 66, + "ods_columns_count": 44, + "missing_in_ods": [ + "activity_discount_amount", + "order_consumption_type", + "real_service_money" + ], + "ods_only": [], + "api_fields": [ + "activity_discount_amount", + "add_clock_seconds", + "address", + "adjust_amount", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "business_tel", + "coupon_promotion_amount", + "create_time", + "customer_service_qrcode", + "customer_service_wechat", + "fee_total", + "fixed_pay_qrcode", + "full_address", + "id", + "is_delete", + "is_single_order", + "last_use_time", + "latitude", + "ledger_amount", + "ledger_count", + "ledger_end_time", + "ledger_name", + "ledger_start_time", + "ledger_status", + "ledger_unit_price", + "light_status", + "light_token", + "light_type", + "longitude", + "member_discount_amount", + "member_id", + "mgmt_fee", + "operator_id", + "operator_name", + "order_consumption_type", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "org_id", + "prod_env", + "real_service_money", + "real_table_charge_money", + "real_table_use_seconds", + "salesman_name", + "salesman_org_id", + "salesman_user_id", + "service_money", + "shop_name", + "shop_status", + "site_id", + "site_label", + "site_table_area_id", + "site_table_area_name", + "site_table_id", + "site_type", + "siteprofile", + "start_use_time", + "tenant_id", + "tenant_site_region_id", + "tenant_table_area_id", + "used_card_amount", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "add_clock_seconds", + "adjust_amount", + "content_hash", + "coupon_promotion_amount", + "create_time", + "fee_total", + "fetched_at", + "id", + "is_delete", + "is_single_order", + "last_use_time", + "ledger_amount", + "ledger_count", + "ledger_end_time", + "ledger_name", + "ledger_start_time", + "ledger_status", + "ledger_unit_price", + "member_discount_amount", + "member_id", + "mgmt_fee", + "operator_id", + "operator_name", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "payload", + "real_table_charge_money", + "real_table_use_seconds", + "salesman_name", + "salesman_org_id", + "salesman_user_id", + "service_money", + "site_id", + "site_table_area_id", + "site_table_area_name", + "site_table_id", + "siteprofile", + "source_endpoint", + "source_file", + "start_use_time", + "tenant_id", + "tenant_table_area_id", + "used_card_amount" + ] + }, + "ODS_ASSISTANT_LEDGER": { + "table_name": "billiards_ods.assistant_service_records", + "endpoint": "/AssistantPerformance/GetOrderAssistantDetails", + "api_records": 50, + "api_fields_count": 90, + "ods_columns_count": 69, + "missing_in_ods": [ + "assistantteamname", + "real_service_money" + ], + "ods_only": [], + "api_fields": [ + "add_clock", + "address", + "assistant_level", + "assistant_team_id", + "assistantname", + "assistantno", + "assistantteamname", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "business_tel", + "composite_grade", + "composite_grade_time", + "coupon_deduct_money", + "create_time", + "customer_service_qrcode", + "customer_service_wechat", + "fixed_pay_qrcode", + "full_address", + "get_grade_times", + "grade_status", + "id", + "income_seconds", + "is_confirm", + "is_delete", + "is_not_responding", + "is_single_order", + "is_trash", + "last_use_time", + "latitude", + "ledger_amount", + "ledger_count", + "ledger_end_time", + "ledger_group_name", + "ledger_name", + "ledger_start_time", + "ledger_status", + "ledger_unit_price", + "levelname", + "light_status", + "light_token", + "light_type", + "longitude", + "manual_discount_amount", + "member_discount_amount", + "nickname", + "operator_id", + "operator_name", + "order_assistant_id", + "order_assistant_type", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "org_id", + "person_org_id", + "prod_env", + "projected_income", + "real_service_money", + "real_use_seconds", + "returns_clock", + "salesman_name", + "salesman_org_id", + "salesman_user_id", + "service_grade", + "service_money", + "shop_name", + "shop_status", + "site_assistant_id", + "site_id", + "site_label", + "site_table_id", + "site_type", + "siteprofile", + "skill_grade", + "skill_id", + "skillname", + "start_use_time", + "sum_grade", + "system_member_id", + "tablename", + "tenant_id", + "tenant_member_id", + "tenant_site_region_id", + "trash_applicant_id", + "trash_applicant_name", + "trash_reason", + "user_id", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "add_clock", + "assistant_level", + "assistant_team_id", + "assistantname", + "assistantno", + "composite_grade", + "composite_grade_time", + "content_hash", + "coupon_deduct_money", + "create_time", + "fetched_at", + "get_grade_times", + "grade_status", + "id", + "income_seconds", + "is_confirm", + "is_delete", + "is_not_responding", + "is_single_order", + "is_trash", + "last_use_time", + "ledger_amount", + "ledger_count", + "ledger_end_time", + "ledger_group_name", + "ledger_name", + "ledger_start_time", + "ledger_status", + "ledger_unit_price", + "levelname", + "manual_discount_amount", + "member_discount_amount", + "nickname", + "operator_id", + "operator_name", + "order_assistant_id", + "order_assistant_type", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "payload", + "person_org_id", + "projected_income", + "real_use_seconds", + "returns_clock", + "salesman_name", + "salesman_org_id", + "salesman_user_id", + "service_grade", + "service_money", + "site_assistant_id", + "site_id", + "site_table_id", + "siteprofile", + "skill_grade", + "skill_id", + "skillname", + "source_endpoint", + "source_file", + "start_use_time", + "sum_grade", + "system_member_id", + "tablename", + "tenant_id", + "tenant_member_id", + "trash_applicant_id", + "trash_applicant_name", + "trash_reason", + "user_id" + ] + }, + "ODS_ASSISTANT_ABOLISH": { + "table_name": "billiards_ods.assistant_cancellation_records", + "endpoint": "/AssistantPerformance/GetAbolitionAssistant", + "api_records": 50, + "api_fields_count": 38, + "ods_columns_count": 18, + "missing_in_ods": [ + "tenant_id" + ], + "ods_only": [], + "api_fields": [ + "address", + "assistantabolishamount", + "assistantname", + "assistanton", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "business_tel", + "createtime", + "customer_service_qrcode", + "customer_service_wechat", + "fixed_pay_qrcode", + "full_address", + "id", + "latitude", + "light_status", + "light_token", + "light_type", + "longitude", + "org_id", + "pdchargeminutes", + "prod_env", + "shop_name", + "shop_status", + "site_label", + "site_type", + "siteid", + "siteprofile", + "tablearea", + "tableareaid", + "tableid", + "tablename", + "tenant_id", + "tenant_site_region_id", + "trashreason", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "assistantabolishamount", + "assistantname", + "assistanton", + "content_hash", + "createtime", + "fetched_at", + "id", + "payload", + "pdchargeminutes", + "siteid", + "siteprofile", + "source_endpoint", + "source_file", + "tablearea", + "tableareaid", + "tableid", + "tablename", + "trashreason" + ] + }, + "ODS_STORE_GOODS_SALES": { + "table_name": "billiards_ods.store_goods_sales_records", + "endpoint": "/TenantGoods/GetGoodsSalesList", + "api_records": 50, + "api_fields_count": 51, + "ods_columns_count": 56, + "missing_in_ods": [ + "coupon_share_money" + ], + "ods_only": [ + "option_name" + ], + "api_fields": [ + "cost_money", + "coupon_deduct_money", + "coupon_share_money", + "create_time", + "discount_money", + "discount_price", + "goods_remark", + "id", + "is_delete", + "is_single_order", + "ledger_amount", + "ledger_count", + "ledger_group_name", + "ledger_name", + "ledger_status", + "ledger_unit_price", + "member_coupon_id", + "member_discount_amount", + "opensalesman", + "operator_id", + "operator_name", + "option_coupon_deduct_money", + "option_member_discount_money", + "option_price", + "option_value_name", + "order_coupon_id", + "order_goods_id", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "ordergoodsid", + "package_coupon_id", + "point_discount_money", + "point_discount_money_cost", + "push_money", + "real_goods_money", + "returns_number", + "sales_man_org_id", + "sales_type", + "salesman_name", + "salesman_role_id", + "salesman_user_id", + "site_goods_id", + "site_id", + "site_table_id", + "siteid", + "sitename", + "tenant_goods_business_id", + "tenant_goods_category_id", + "tenant_goods_id", + "tenant_id" + ], + "ods_columns": [ + "content_hash", + "cost_money", + "coupon_deduct_money", + "create_time", + "discount_money", + "discount_price", + "fetched_at", + "goods_remark", + "id", + "is_delete", + "is_single_order", + "ledger_amount", + "ledger_count", + "ledger_group_name", + "ledger_name", + "ledger_status", + "ledger_unit_price", + "member_coupon_id", + "member_discount_amount", + "opensalesman", + "operator_id", + "operator_name", + "option_coupon_deduct_money", + "option_member_discount_money", + "option_name", + "option_price", + "option_value_name", + "order_coupon_id", + "order_goods_id", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "ordergoodsid", + "package_coupon_id", + "payload", + "point_discount_money", + "point_discount_money_cost", + "push_money", + "real_goods_money", + "returns_number", + "sales_man_org_id", + "sales_type", + "salesman_name", + "salesman_role_id", + "salesman_user_id", + "site_goods_id", + "site_id", + "site_table_id", + "siteid", + "sitename", + "source_endpoint", + "source_file", + "tenant_goods_business_id", + "tenant_goods_category_id", + "tenant_goods_id", + "tenant_id" + ] + }, + "ODS_PAYMENT": { + "table_name": "billiards_ods.payment_transactions", + "endpoint": "/PayLog/GetPayLogListPage", + "api_records": 50, + "api_fields_count": 36, + "ods_columns_count": 16, + "missing_in_ods": [ + "tenant_id" + ], + "ods_only": [], + "api_fields": [ + "address", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "business_tel", + "create_time", + "customer_service_qrcode", + "customer_service_wechat", + "fixed_pay_qrcode", + "full_address", + "id", + "latitude", + "light_status", + "light_token", + "light_type", + "longitude", + "online_pay_channel", + "org_id", + "pay_amount", + "pay_status", + "pay_time", + "payment_method", + "prod_env", + "relate_id", + "relate_type", + "shop_name", + "shop_status", + "site_id", + "site_label", + "site_type", + "siteprofile", + "tenant_id", + "tenant_site_region_id", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "content_hash", + "create_time", + "fetched_at", + "id", + "online_pay_channel", + "pay_amount", + "pay_status", + "pay_time", + "payload", + "payment_method", + "relate_id", + "relate_type", + "site_id", + "siteprofile", + "source_endpoint", + "source_file" + ] + }, + "ODS_REFUND": { + "table_name": "billiards_ods.refund_transactions", + "endpoint": "/Order/GetRefundPayLogList", + "api_records": 23, + "api_fields_count": 56, + "ods_columns_count": 37, + "missing_in_ods": [], + "ods_only": [], + "api_fields": [ + "action_type", + "address", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "balance_frozen_amount", + "business_tel", + "card_frozen_amount", + "cashier_point_id", + "channel_fee", + "channel_pay_no", + "channel_payer_id", + "check_status", + "create_time", + "customer_service_qrcode", + "customer_service_wechat", + "fixed_pay_qrcode", + "full_address", + "id", + "is_delete", + "is_revoke", + "latitude", + "light_status", + "light_token", + "light_type", + "longitude", + "member_card_id", + "member_id", + "online_pay_channel", + "online_pay_type", + "operator_id", + "org_id", + "pay_amount", + "pay_config_id", + "pay_sn", + "pay_status", + "pay_terminal", + "pay_time", + "payment_method", + "prod_env", + "refund_amount", + "relate_id", + "relate_type", + "round_amount", + "shop_name", + "shop_status", + "site_id", + "site_label", + "site_type", + "siteprofile", + "tenant_id", + "tenant_site_region_id", + "tenantname", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "action_type", + "balance_frozen_amount", + "card_frozen_amount", + "cashier_point_id", + "channel_fee", + "channel_pay_no", + "channel_payer_id", + "check_status", + "content_hash", + "create_time", + "fetched_at", + "id", + "is_delete", + "is_revoke", + "member_card_id", + "member_id", + "online_pay_channel", + "online_pay_type", + "operator_id", + "pay_amount", + "pay_config_id", + "pay_sn", + "pay_status", + "pay_terminal", + "pay_time", + "payload", + "payment_method", + "refund_amount", + "relate_id", + "relate_type", + "round_amount", + "site_id", + "siteprofile", + "source_endpoint", + "source_file", + "tenant_id", + "tenantname" + ] + }, + "ODS_PLATFORM_COUPON": { + "table_name": "billiards_ods.platform_coupon_redemption_records", + "endpoint": "/Promotion/GetOfflineCouponConsumePageList", + "api_records": 50, + "api_fields_count": 50, + "ods_columns_count": 31, + "missing_in_ods": [], + "ods_only": [], + "api_fields": [ + "address", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "business_tel", + "certificate_id", + "channel_deal_id", + "consume_time", + "coupon_channel", + "coupon_code", + "coupon_cover", + "coupon_free_time", + "coupon_money", + "coupon_name", + "coupon_remark", + "create_time", + "customer_service_qrcode", + "customer_service_wechat", + "deal_id", + "fixed_pay_qrcode", + "full_address", + "group_package_id", + "groupon_type", + "id", + "is_delete", + "latitude", + "light_status", + "light_token", + "light_type", + "longitude", + "operator_id", + "operator_name", + "org_id", + "prod_env", + "sale_price", + "shop_name", + "shop_status", + "site_id", + "site_label", + "site_order_id", + "site_type", + "siteprofile", + "table_id", + "tenant_id", + "tenant_site_region_id", + "use_status", + "verify_id", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "certificate_id", + "channel_deal_id", + "consume_time", + "content_hash", + "coupon_channel", + "coupon_code", + "coupon_cover", + "coupon_free_time", + "coupon_money", + "coupon_name", + "coupon_remark", + "create_time", + "deal_id", + "fetched_at", + "group_package_id", + "groupon_type", + "id", + "is_delete", + "operator_id", + "operator_name", + "payload", + "sale_price", + "site_id", + "site_order_id", + "siteprofile", + "source_endpoint", + "source_file", + "table_id", + "tenant_id", + "use_status", + "verify_id" + ] + }, + "ODS_MEMBER": { + "table_name": "billiards_ods.member_profiles", + "endpoint": "/MemberProfile/GetTenantMemberList", + "api_records": 16, + "api_fields_count": 20, + "ods_columns_count": 20, + "missing_in_ods": [ + "pay_money_sum", + "person_tenant_org_id", + "person_tenant_org_name", + "recharge_money_sum", + "register_source" + ], + "ods_only": [], + "api_fields": [ + "create_time", + "growth_value", + "id", + "member_card_grade_code", + "member_card_grade_name", + "mobile", + "nickname", + "pay_money_sum", + "person_tenant_org_id", + "person_tenant_org_name", + "point", + "recharge_money_sum", + "referrer_member_id", + "register_site_id", + "register_source", + "site_name", + "status", + "system_member_id", + "tenant_id", + "user_status" + ], + "ods_columns": [ + "content_hash", + "create_time", + "fetched_at", + "growth_value", + "id", + "member_card_grade_code", + "member_card_grade_name", + "mobile", + "nickname", + "payload", + "point", + "referrer_member_id", + "register_site_id", + "site_name", + "source_endpoint", + "source_file", + "status", + "system_member_id", + "tenant_id", + "user_status" + ] + }, + "ODS_MEMBER_CARD": { + "table_name": "billiards_ods.member_stored_value_cards", + "endpoint": "/MemberProfile/GetTenantMemberCardList", + "api_records": 32, + "api_fields_count": 75, + "ods_columns_count": 74, + "missing_in_ods": [ + "able_share_member_discount", + "electricity_deduct_radio", + "electricity_discount", + "electricitycarddeduct", + "member_grade", + "principal_balance", + "rechargefreezebalance" + ], + "ods_only": [ + "able_site_transfer" + ], + "api_fields": [ + "able_cross_site", + "able_share_member_discount", + "assistant_deduct_radio", + "assistant_discount", + "assistant_discount_sub_switch", + "assistant_reward_deduct_radio", + "assistant_reward_discount", + "assistant_reward_discount_sub_switch", + "assistant_service_deduct_radio", + "assistant_service_discount", + "assistantcarddeduct", + "assistantrewardcarddeduct", + "assistantservicecarddeduct", + "balance", + "bind_password", + "card_no", + "card_physics_type", + "card_type_id", + "cardsettlededuct", + "coupon_deduct_radio", + "coupon_discount", + "couponcarddeduct", + "create_time", + "cxassisnatlevel", + "deliveryfeededuct", + "denomination", + "disable_end_time", + "disable_start_time", + "effect_site_id", + "electricity_deduct_radio", + "electricity_discount", + "electricitycarddeduct", + "end_time", + "goods_deduct_radio", + "goods_discount", + "goods_discount_range_type", + "goods_discount_sub_switch", + "goods_service_deduct_radio", + "goods_service_discount", + "goodscardeduct", + "goodscategoryid", + "goodsservicecarddeduct", + "id", + "is_allow_give", + "is_allow_order_deduct", + "is_delete", + "last_consume_time", + "member_card_grade_code", + "member_card_grade_code_name", + "member_card_type_name", + "member_grade", + "member_mobile", + "member_name", + "pdassisnatlevel", + "principal_balance", + "rechargefreezebalance", + "register_site_id", + "site_name", + "sort", + "start_time", + "status", + "system_member_id", + "table_deduct_radio", + "table_discount", + "table_discount_sub_switch", + "table_service_deduct_radio", + "table_service_discount", + "tableareaid", + "tablecarddeduct", + "tableservicecarddeduct", + "tenant_id", + "tenant_member_id", + "tenantavatar", + "tenantname", + "use_scene" + ], + "ods_columns": [ + "able_cross_site", + "able_site_transfer", + "assistant_deduct_radio", + "assistant_discount", + "assistant_discount_sub_switch", + "assistant_reward_deduct_radio", + "assistant_reward_discount", + "assistant_reward_discount_sub_switch", + "assistant_service_deduct_radio", + "assistant_service_discount", + "assistantcarddeduct", + "assistantrewardcarddeduct", + "assistantservicecarddeduct", + "balance", + "bind_password", + "card_no", + "card_physics_type", + "card_type_id", + "cardsettlededuct", + "content_hash", + "coupon_deduct_radio", + "coupon_discount", + "couponcarddeduct", + "create_time", + "cxassisnatlevel", + "deliveryfeededuct", + "denomination", + "disable_end_time", + "disable_start_time", + "effect_site_id", + "end_time", + "fetched_at", + "goods_deduct_radio", + "goods_discount", + "goods_discount_range_type", + "goods_discount_sub_switch", + "goods_service_deduct_radio", + "goods_service_discount", + "goodscardeduct", + "goodscategoryid", + "goodsservicecarddeduct", + "id", + "is_allow_give", + "is_allow_order_deduct", + "is_delete", + "last_consume_time", + "member_card_grade_code", + "member_card_grade_code_name", + "member_card_type_name", + "member_mobile", + "member_name", + "payload", + "pdassisnatlevel", + "register_site_id", + "site_name", + "sort", + "source_endpoint", + "source_file", + "start_time", + "status", + "system_member_id", + "table_deduct_radio", + "table_discount", + "table_discount_sub_switch", + "table_service_deduct_radio", + "table_service_discount", + "tableareaid", + "tablecarddeduct", + "tableservicecarddeduct", + "tenant_id", + "tenant_member_id", + "tenantavatar", + "tenantname", + "use_scene" + ] + }, + "ODS_MEMBER_BALANCE": { + "table_name": "billiards_ods.member_balance_changes", + "endpoint": "/MemberProfile/GetMemberCardBalanceChange", + "api_records": 50, + "api_fields_count": 28, + "ods_columns_count": 30, + "missing_in_ods": [ + "principal_after", + "principal_before", + "principal_data" + ], + "ods_only": [], + "api_fields": [ + "account_data", + "after", + "before", + "card_type_id", + "create_time", + "from_type", + "id", + "is_delete", + "membercardtypename", + "membermobile", + "membername", + "operator_id", + "operator_name", + "payment_method", + "paysitename", + "principal_after", + "principal_before", + "principal_data", + "refund_amount", + "register_site_id", + "registersitename", + "relate_id", + "remark", + "site_id", + "system_member_id", + "tenant_id", + "tenant_member_card_id", + "tenant_member_id" + ], + "ods_columns": [ + "account_data", + "after", + "before", + "card_type_id", + "content_hash", + "create_time", + "fetched_at", + "from_type", + "id", + "is_delete", + "membercardtypename", + "membermobile", + "membername", + "operator_id", + "operator_name", + "payload", + "payment_method", + "paysitename", + "refund_amount", + "register_site_id", + "registersitename", + "relate_id", + "remark", + "site_id", + "source_endpoint", + "source_file", + "system_member_id", + "tenant_id", + "tenant_member_card_id", + "tenant_member_id" + ] + }, + "ODS_RECHARGE_SETTLE": { + "table_name": "billiards_ods.recharge_settlements", + "endpoint": "/Site/GetRechargeSettleList", + "api_records": 50, + "api_fields_count": 93, + "ods_columns_count": 66, + "missing_in_ods": [ + "electricityadjustmoney", + "electricitymoney", + "mervousalesamount", + "plcouponsaleamount", + "realelectricitymoney", + "settlelist", + "tenant_id" + ], + "ods_only": [], + "api_fields": [ + "activitydiscount", + "address", + "adjustamount", + "allcoupondiscount", + "assistantcxmoney", + "assistantmanualdiscount", + "assistantpdmoney", + "assistantpromotionmoney", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "balanceamount", + "business_tel", + "canberevoked", + "cardamount", + "cashamount", + "consumemoney", + "couponamount", + "couponsaleamount", + "createtime", + "customer_service_qrcode", + "customer_service_wechat", + "electricityadjustmoney", + "electricitymoney", + "fixed_pay_qrcode", + "full_address", + "giftcardamount", + "goodsmoney", + "goodspromotionmoney", + "id", + "isactivity", + "isbindmember", + "isfirst", + "isusecoupon", + "isusediscount", + "latitude", + "light_status", + "light_token", + "light_type", + "longitude", + "membercardtypename", + "memberdiscountamount", + "memberid", + "membername", + "memberphone", + "mervousalesamount", + "onlineamount", + "operatorid", + "operatorname", + "orderremark", + "org_id", + "payamount", + "paymentmethod", + "paytime", + "plcouponsaleamount", + "pointamount", + "pointdiscountcost", + "pointdiscountprice", + "prepaymoney", + "prod_env", + "realelectricitymoney", + "realgoodsmoney", + "rechargecardamount", + "refundamount", + "revokeorderid", + "revokeordername", + "revoketime", + "roundingamount", + "salesmanname", + "salesmanuserid", + "serialnumber", + "servicemoney", + "settlelist", + "settlename", + "settlerelateid", + "settlestatus", + "settletype", + "shop_name", + "shop_status", + "site_label", + "site_type", + "siteid", + "sitename", + "siteprofile", + "tablechargemoney", + "tableid", + "tenant_id", + "tenant_site_region_id", + "tenantid", + "tenantmembercardid", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "activitydiscount", + "adjustamount", + "allcoupondiscount", + "assistantcxmoney", + "assistantmanualdiscount", + "assistantpdmoney", + "assistantpromotionmoney", + "balanceamount", + "canberevoked", + "cardamount", + "cashamount", + "consumemoney", + "content_hash", + "couponamount", + "couponsaleamount", + "createtime", + "fetched_at", + "giftcardamount", + "goodsmoney", + "goodspromotionmoney", + "id", + "isactivity", + "isbindmember", + "isfirst", + "isusecoupon", + "isusediscount", + "membercardtypename", + "memberdiscountamount", + "memberid", + "membername", + "memberphone", + "onlineamount", + "operatorid", + "operatorname", + "orderremark", + "payamount", + "payload", + "paymentmethod", + "paytime", + "pointamount", + "pointdiscountcost", + "pointdiscountprice", + "prepaymoney", + "realgoodsmoney", + "rechargecardamount", + "refundamount", + "revokeorderid", + "revokeordername", + "revoketime", + "roundingamount", + "salesmanname", + "salesmanuserid", + "serialnumber", + "servicemoney", + "settlename", + "settlerelateid", + "settlestatus", + "settletype", + "siteid", + "sitename", + "source_endpoint", + "source_file", + "tablechargemoney", + "tableid", + "tenantid", + "tenantmembercardid" + ] + }, + "ODS_GROUP_PACKAGE": { + "table_name": "billiards_ods.group_buy_packages", + "endpoint": "/PackageCoupon/QueryPackageCouponList", + "api_records": 18, + "api_fields_count": 40, + "ods_columns_count": 40, + "missing_in_ods": [ + "is_first_limit", + "sort", + "tableareanamelist", + "tenantcouponsaleorderitemid", + "tenanttableareaidlist" + ], + "ods_only": [], + "api_fields": [ + "add_end_clock", + "add_start_clock", + "area_tag_type", + "card_type_ids", + "coupon_money", + "create_time", + "creator_name", + "date_info", + "date_type", + "duration", + "effective_status", + "end_clock", + "end_time", + "group_type", + "id", + "is_delete", + "is_enabled", + "is_first_limit", + "max_selectable_categories", + "package_id", + "package_name", + "selling_price", + "site_id", + "site_name", + "sort", + "start_clock", + "start_time", + "system_group_type", + "table_area_id", + "table_area_id_list", + "table_area_name", + "tableareanamelist", + "tenant_id", + "tenant_table_area_id", + "tenant_table_area_id_list", + "tenantcouponsaleorderitemid", + "tenanttableareaidlist", + "type", + "usable_count", + "usable_range" + ], + "ods_columns": [ + "add_end_clock", + "add_start_clock", + "area_tag_type", + "card_type_ids", + "content_hash", + "coupon_money", + "create_time", + "creator_name", + "date_info", + "date_type", + "duration", + "effective_status", + "end_clock", + "end_time", + "fetched_at", + "group_type", + "id", + "is_delete", + "is_enabled", + "max_selectable_categories", + "package_id", + "package_name", + "payload", + "selling_price", + "site_id", + "site_name", + "source_endpoint", + "source_file", + "start_clock", + "start_time", + "system_group_type", + "table_area_id", + "table_area_id_list", + "table_area_name", + "tenant_id", + "tenant_table_area_id", + "tenant_table_area_id_list", + "type", + "usable_count", + "usable_range" + ] + }, + "ODS_GROUP_BUY_REDEMPTION": { + "table_name": "billiards_ods.group_buy_redemption_records", + "endpoint": "/Site/GetSiteTableUseDetails", + "api_records": 50, + "api_fields_count": 52, + "ods_columns_count": 48, + "missing_in_ods": [ + "assistant_service_share_money", + "assistant_share_money", + "coupon_sale_id", + "good_service_share_money", + "goods_share_money", + "member_discount_money", + "recharge_share_money", + "table_service_share_money", + "table_share_money" + ], + "ods_only": [], + "api_fields": [ + "assistant_promotion_money", + "assistant_service_promotion_money", + "assistant_service_share_money", + "assistant_share_money", + "coupon_code", + "coupon_money", + "coupon_origin_id", + "coupon_sale_id", + "create_time", + "good_service_share_money", + "goods_promotion_money", + "goods_share_money", + "goodsoptionprice", + "id", + "is_delete", + "is_single_order", + "ledger_amount", + "ledger_count", + "ledger_group_name", + "ledger_name", + "ledger_status", + "ledger_unit_price", + "member_discount_money", + "offer_type", + "operator_id", + "operator_name", + "order_coupon_channel", + "order_coupon_id", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "promotion_activity_id", + "promotion_coupon_id", + "promotion_seconds", + "recharge_promotion_money", + "recharge_share_money", + "reward_promotion_money", + "sales_man_org_id", + "salesman_name", + "salesman_role_id", + "salesman_user_id", + "site_id", + "sitename", + "table_charge_seconds", + "table_id", + "table_service_promotion_money", + "table_service_share_money", + "table_share_money", + "tableareaname", + "tablename", + "tenant_id", + "tenant_table_area_id" + ], + "ods_columns": [ + "assistant_promotion_money", + "assistant_service_promotion_money", + "content_hash", + "coupon_code", + "coupon_money", + "coupon_origin_id", + "create_time", + "fetched_at", + "goods_promotion_money", + "goodsoptionprice", + "id", + "is_delete", + "is_single_order", + "ledger_amount", + "ledger_count", + "ledger_group_name", + "ledger_name", + "ledger_status", + "ledger_unit_price", + "offer_type", + "operator_id", + "operator_name", + "order_coupon_channel", + "order_coupon_id", + "order_pay_id", + "order_settle_id", + "order_trade_no", + "payload", + "promotion_activity_id", + "promotion_coupon_id", + "promotion_seconds", + "recharge_promotion_money", + "reward_promotion_money", + "sales_man_org_id", + "salesman_name", + "salesman_role_id", + "salesman_user_id", + "site_id", + "sitename", + "source_endpoint", + "source_file", + "table_charge_seconds", + "table_id", + "table_service_promotion_money", + "tableareaname", + "tablename", + "tenant_id", + "tenant_table_area_id" + ] + }, + "ODS_INVENTORY_STOCK": { + "table_name": "billiards_ods.goods_stock_summary", + "endpoint": "/TenantGoods/GetGoodsStockReport", + "api_records": 50, + "api_fields_count": 14, + "ods_columns_count": 19, + "missing_in_ods": [], + "ods_only": [], + "api_fields": [ + "categoryname", + "currentstock", + "goodscategoryid", + "goodscategorysecondid", + "goodsname", + "goodsunit", + "rangeendstock", + "rangein", + "rangeinventory", + "rangeout", + "rangesale", + "rangesalemoney", + "rangestartstock", + "sitegoodsid" + ], + "ods_columns": [ + "categoryname", + "content_hash", + "currentstock", + "fetched_at", + "goodscategoryid", + "goodscategorysecondid", + "goodsname", + "goodsunit", + "payload", + "rangeendstock", + "rangein", + "rangeinventory", + "rangeout", + "rangesale", + "rangesalemoney", + "rangestartstock", + "sitegoodsid", + "source_endpoint", + "source_file" + ] + }, + "ODS_INVENTORY_CHANGE": { + "table_name": "billiards_ods.goods_stock_movements", + "endpoint": "/GoodsStockManage/QueryGoodsOutboundReceipt", + "api_records": 50, + "api_fields_count": 19, + "ods_columns_count": 24, + "missing_in_ods": [], + "ods_only": [], + "api_fields": [ + "changenum", + "changenuma", + "createtime", + "endnum", + "endnuma", + "goodscategoryid", + "goodsname", + "goodssecondcategoryid", + "operatorname", + "price", + "remark", + "sitegoodsid", + "sitegoodsstockid", + "siteid", + "startnum", + "startnuma", + "stocktype", + "tenantid", + "unit" + ], + "ods_columns": [ + "changenum", + "changenuma", + "content_hash", + "createtime", + "endnum", + "endnuma", + "fetched_at", + "goodscategoryid", + "goodsname", + "goodssecondcategoryid", + "operatorname", + "payload", + "price", + "remark", + "sitegoodsid", + "sitegoodsstockid", + "siteid", + "source_endpoint", + "source_file", + "startnum", + "startnuma", + "stocktype", + "tenantid", + "unit" + ] + }, + "ODS_TABLES": { + "table_name": "billiards_ods.site_tables_master", + "endpoint": "/Table/GetSiteTables", + "api_records": 50, + "api_fields_count": 26, + "ods_columns_count": 30, + "missing_in_ods": [ + "order_id" + ], + "ods_only": [], + "api_fields": [ + "appletqrcodeurl", + "areaname", + "audit_status", + "charge_free", + "create_time", + "delay_lights_time", + "id", + "is_online_reservation", + "is_rest_area", + "light_status", + "only_allow_groupon", + "order_delay_time", + "order_id", + "self_table", + "show_status", + "site_id", + "site_table_area_id", + "sitename", + "table_cloth_use_cycle", + "table_cloth_use_time", + "table_name", + "table_price", + "table_status", + "tablestatusname", + "temporary_light_second", + "virtual_table" + ], + "ods_columns": [ + "appletqrcodeurl", + "areaname", + "audit_status", + "charge_free", + "content_hash", + "create_time", + "delay_lights_time", + "fetched_at", + "id", + "is_online_reservation", + "is_rest_area", + "light_status", + "only_allow_groupon", + "order_delay_time", + "payload", + "self_table", + "show_status", + "site_id", + "site_table_area_id", + "sitename", + "source_endpoint", + "source_file", + "table_cloth_use_cycle", + "table_cloth_use_time", + "table_name", + "table_price", + "table_status", + "tablestatusname", + "temporary_light_second", + "virtual_table" + ] + }, + "ODS_GOODS_CATEGORY": { + "table_name": "billiards_ods.stock_goods_category_tree", + "endpoint": "/TenantGoodsCategory/QueryPrimarySecondaryCategory", + "api_records": 9, + "api_fields_count": 11, + "ods_columns_count": 16, + "missing_in_ods": [], + "ods_only": [], + "api_fields": [ + "alias_name", + "business_name", + "category_name", + "categoryboxes", + "id", + "is_warehousing", + "open_salesman", + "pid", + "sort", + "tenant_goods_business_id", + "tenant_id" + ], + "ods_columns": [ + "alias_name", + "business_name", + "category_name", + "categoryboxes", + "content_hash", + "fetched_at", + "id", + "is_warehousing", + "open_salesman", + "payload", + "pid", + "sort", + "source_endpoint", + "source_file", + "tenant_goods_business_id", + "tenant_id" + ] + }, + "ODS_STORE_GOODS": { + "table_name": "billiards_ods.store_goods_master", + "endpoint": "/TenantGoods/GetGoodsInventoryList", + "api_records": 50, + "api_fields_count": 47, + "ods_columns_count": 50, + "missing_in_ods": [ + "commodity_code", + "not_sale" + ], + "ods_only": [], + "api_fields": [ + "able_discount", + "able_site_transfer", + "audit_status", + "average_monthly_sales", + "batch_stock_quantity", + "commodity_code", + "cost_price", + "cost_price_type", + "create_time", + "custom_label_type", + "days_available", + "enable_status", + "forbid_sell_status", + "freeze", + "goods_bar_code", + "goods_category_id", + "goods_cover", + "goods_name", + "goods_second_category_id", + "goods_state", + "id", + "is_delete", + "is_warehousing", + "min_discount_price", + "not_sale", + "onecategoryname", + "option_required", + "pinyin_initial", + "provisional_total_cost", + "remark", + "safe_stock", + "sale_channel", + "sale_num", + "sale_price", + "send_state", + "site_id", + "sitename", + "sort", + "stock", + "stock_a", + "tenant_goods_id", + "tenant_id", + "total_purchase_cost", + "total_sales", + "twocategoryname", + "unit", + "update_time" + ], + "ods_columns": [ + "able_discount", + "able_site_transfer", + "audit_status", + "average_monthly_sales", + "batch_stock_quantity", + "content_hash", + "cost_price", + "cost_price_type", + "create_time", + "custom_label_type", + "days_available", + "enable_status", + "fetched_at", + "forbid_sell_status", + "freeze", + "goods_bar_code", + "goods_category_id", + "goods_cover", + "goods_name", + "goods_second_category_id", + "goods_state", + "id", + "is_delete", + "is_warehousing", + "min_discount_price", + "onecategoryname", + "option_required", + "payload", + "pinyin_initial", + "provisional_total_cost", + "remark", + "safe_stock", + "sale_channel", + "sale_num", + "sale_price", + "send_state", + "site_id", + "sitename", + "sort", + "source_endpoint", + "source_file", + "stock", + "stock_a", + "tenant_goods_id", + "tenant_id", + "total_purchase_cost", + "total_sales", + "twocategoryname", + "unit", + "update_time" + ] + }, + "ODS_TABLE_FEE_DISCOUNT": { + "table_name": "billiards_ods.table_fee_discount_records", + "endpoint": "/Site/GetTaiFeeAdjustList", + "api_records": 50, + "api_fields_count": 53, + "ods_columns_count": 25, + "missing_in_ods": [ + "area_type_id", + "charge_free", + "site_table_area_id", + "site_table_area_name", + "sitename", + "table_name", + "table_price", + "tenant_name" + ], + "ods_only": [], + "api_fields": [ + "address", + "adjust_type", + "applicant_id", + "applicant_name", + "area_type_id", + "attendance_distance", + "attendance_enabled", + "auto_light", + "avatar", + "business_tel", + "charge_free", + "create_time", + "customer_service_qrcode", + "customer_service_wechat", + "ewelink_client_id", + "fixed_pay_qrcode", + "full_address", + "id", + "is_delete", + "latitude", + "ledger_amount", + "ledger_count", + "ledger_name", + "ledger_status", + "light_status", + "light_token", + "light_type", + "longitude", + "operator_id", + "operator_name", + "order_settle_id", + "order_trade_no", + "org_id", + "prod_env", + "shop_name", + "shop_status", + "site_id", + "site_label", + "site_table_area_id", + "site_table_area_name", + "site_table_id", + "site_type", + "sitename", + "siteprofile", + "table_name", + "table_price", + "tableprofile", + "tenant_id", + "tenant_name", + "tenant_site_region_id", + "tenant_table_area_id", + "wifi_name", + "wifi_password" + ], + "ods_columns": [ + "adjust_type", + "applicant_id", + "applicant_name", + "content_hash", + "create_time", + "fetched_at", + "id", + "is_delete", + "ledger_amount", + "ledger_count", + "ledger_name", + "ledger_status", + "operator_id", + "operator_name", + "order_settle_id", + "order_trade_no", + "payload", + "site_id", + "site_table_id", + "siteprofile", + "source_endpoint", + "source_file", + "tableprofile", + "tenant_id", + "tenant_table_area_id" + ] + }, + "ODS_TENANT_GOODS": { + "table_name": "billiards_ods.tenant_goods_master", + "endpoint": "/TenantGoods/QueryTenantGoods", + "api_records": 50, + "api_fields_count": 32, + "ods_columns_count": 36, + "missing_in_ods": [ + "not_sale" + ], + "ods_only": [], + "api_fields": [ + "able_discount", + "able_site_transfer", + "categoryname", + "commodity_code", + "commoditycode", + "common_sale_royalty", + "cost_price", + "cost_price_type", + "create_time", + "goods_bar_code", + "goods_category_id", + "goods_cover", + "goods_name", + "goods_number", + "goods_second_category_id", + "goods_state", + "id", + "is_delete", + "is_warehousing", + "isinsite", + "market_price", + "min_discount_price", + "not_sale", + "out_goods_id", + "pinyin_initial", + "point_sale_royalty", + "remark_name", + "sale_channel", + "supplier_id", + "tenant_id", + "unit", + "update_time" + ], + "ods_columns": [ + "able_discount", + "able_site_transfer", + "categoryname", + "commodity_code", + "commoditycode", + "common_sale_royalty", + "content_hash", + "cost_price", + "cost_price_type", + "create_time", + "fetched_at", + "goods_bar_code", + "goods_category_id", + "goods_cover", + "goods_name", + "goods_number", + "goods_second_category_id", + "goods_state", + "id", + "is_delete", + "is_warehousing", + "isinsite", + "market_price", + "min_discount_price", + "out_goods_id", + "payload", + "pinyin_initial", + "point_sale_royalty", + "remark_name", + "sale_channel", + "source_endpoint", + "source_file", + "supplier_id", + "tenant_id", + "unit", + "update_time" + ] + } +} \ No newline at end of file diff --git a/tmp/api_ods_issue_report.json b/tmp/api_ods_issue_report.json new file mode 100644 index 0000000..ac4029f --- /dev/null +++ b/tmp/api_ods_issue_report.json @@ -0,0 +1,355 @@ +{ + "generated_at": "2026-02-02T19:00:26.972834", + "missing_fields": [ + { + "task_code": "ODS_SETTLEMENT_RECORDS", + "table_name": "billiards_ods.settlement_records", + "endpoint": "/Site/GetAllOrderSettleList", + "missing_fields": [ + "electricityadjustmoney", + "electricitymoney", + "mervousalesamount", + "plcouponsaleamount", + "realelectricitymoney", + "settlelist", + "tenant_id" + ] + }, + { + "task_code": "ODS_TABLE_USE", + "table_name": "billiards_ods.table_fee_transactions", + "endpoint": "/Site/GetSiteTableOrderDetails", + "missing_fields": [ + "activity_discount_amount", + "order_consumption_type", + "real_service_money" + ] + }, + { + "task_code": "ODS_ASSISTANT_LEDGER", + "table_name": "billiards_ods.assistant_service_records", + "endpoint": "/AssistantPerformance/GetOrderAssistantDetails", + "missing_fields": [ + "assistantteamname", + "real_service_money" + ] + }, + { + "task_code": "ODS_ASSISTANT_ABOLISH", + "table_name": "billiards_ods.assistant_cancellation_records", + "endpoint": "/AssistantPerformance/GetAbolitionAssistant", + "missing_fields": [ + "tenant_id" + ] + }, + { + "task_code": "ODS_STORE_GOODS_SALES", + "table_name": "billiards_ods.store_goods_sales_records", + "endpoint": "/TenantGoods/GetGoodsSalesList", + "missing_fields": [ + "coupon_share_money" + ] + }, + { + "task_code": "ODS_PAYMENT", + "table_name": "billiards_ods.payment_transactions", + "endpoint": "/PayLog/GetPayLogListPage", + "missing_fields": [ + "tenant_id" + ] + }, + { + "task_code": "ODS_MEMBER", + "table_name": "billiards_ods.member_profiles", + "endpoint": "/MemberProfile/GetTenantMemberList", + "missing_fields": [ + "pay_money_sum", + "person_tenant_org_id", + "person_tenant_org_name", + "recharge_money_sum", + "register_source" + ] + }, + { + "task_code": "ODS_MEMBER_CARD", + "table_name": "billiards_ods.member_stored_value_cards", + "endpoint": "/MemberProfile/GetTenantMemberCardList", + "missing_fields": [ + "able_share_member_discount", + "electricity_deduct_radio", + "electricity_discount", + "electricitycarddeduct", + "member_grade", + "principal_balance", + "rechargefreezebalance" + ] + }, + { + "task_code": "ODS_MEMBER_BALANCE", + "table_name": "billiards_ods.member_balance_changes", + "endpoint": "/MemberProfile/GetMemberCardBalanceChange", + "missing_fields": [ + "principal_after", + "principal_before", + "principal_data" + ] + }, + { + "task_code": "ODS_RECHARGE_SETTLE", + "table_name": "billiards_ods.recharge_settlements", + "endpoint": "/Site/GetRechargeSettleList", + "missing_fields": [ + "electricityadjustmoney", + "electricitymoney", + "mervousalesamount", + "plcouponsaleamount", + "realelectricitymoney", + "settlelist", + "tenant_id" + ] + }, + { + "task_code": "ODS_GROUP_PACKAGE", + "table_name": "billiards_ods.group_buy_packages", + "endpoint": "/PackageCoupon/QueryPackageCouponList", + "missing_fields": [ + "is_first_limit", + "sort", + "tableareanamelist", + "tenantcouponsaleorderitemid", + "tenanttableareaidlist" + ] + }, + { + "task_code": "ODS_GROUP_BUY_REDEMPTION", + "table_name": "billiards_ods.group_buy_redemption_records", + "endpoint": "/Site/GetSiteTableUseDetails", + "missing_fields": [ + "assistant_service_share_money", + "assistant_share_money", + "coupon_sale_id", + "good_service_share_money", + "goods_share_money", + "member_discount_money", + "recharge_share_money", + "table_service_share_money", + "table_share_money" + ] + }, + { + "task_code": "ODS_TABLES", + "table_name": "billiards_ods.site_tables_master", + "endpoint": "/Table/GetSiteTables", + "missing_fields": [ + "order_id" + ] + }, + { + "task_code": "ODS_STORE_GOODS", + "table_name": "billiards_ods.store_goods_master", + "endpoint": "/TenantGoods/GetGoodsInventoryList", + "missing_fields": [ + "commodity_code", + "not_sale" + ] + }, + { + "task_code": "ODS_TABLE_FEE_DISCOUNT", + "table_name": "billiards_ods.table_fee_discount_records", + "endpoint": "/Site/GetTaiFeeAdjustList", + "missing_fields": [ + "area_type_id", + "charge_free", + "site_table_area_id", + "site_table_area_name", + "sitename", + "table_name", + "table_price", + "tenant_name" + ] + }, + { + "task_code": "ODS_TENANT_GOODS", + "table_name": "billiards_ods.tenant_goods_master", + "endpoint": "/TenantGoods/QueryTenantGoods", + "missing_fields": [ + "not_sale" + ] + } + ], + "zero_to_null_issues": [ + { + "task_code": "ODS_TABLE_USE", + "table_name": "billiards_ods.table_fee_transactions", + "checked_rows": 100, + "issues": [ + { + "column": "activity_discount_amount", + "count": 67, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "real_service_money", + "count": 67, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_ASSISTANT_LEDGER", + "table_name": "billiards_ods.assistant_service_records", + "checked_rows": 100, + "issues": [ + { + "column": "real_service_money", + "count": 90, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_STORE_GOODS_SALES", + "table_name": "billiards_ods.store_goods_sales_records", + "checked_rows": 100, + "issues": [ + { + "column": "coupon_share_money", + "count": 100, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_MEMBER", + "table_name": "billiards_ods.member_profiles", + "checked_rows": 100, + "issues": [ + { + "column": "person_tenant_org_id", + "count": 96, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "pay_money_sum", + "count": 40, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "recharge_money_sum", + "count": 12, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_MEMBER_CARD", + "table_name": "billiards_ods.member_stored_value_cards", + "checked_rows": 100, + "issues": [ + { + "column": "rechargefreezebalance", + "count": 100, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "principal_balance", + "count": 34, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "member_grade", + "count": 8, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_MEMBER_BALANCE", + "table_name": "billiards_ods.member_balance_changes", + "checked_rows": 100, + "issues": [ + { + "column": "principal_after", + "count": 18, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "principal_before", + "count": 18, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_GROUP_PACKAGE", + "table_name": "billiards_ods.group_buy_packages", + "checked_rows": 52, + "issues": [ + { + "column": "tenantcouponsaleorderitemid", + "count": 52, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_GROUP_BUY_REDEMPTION", + "table_name": "billiards_ods.group_buy_redemption_records", + "checked_rows": 100, + "issues": [ + { + "column": "assistant_service_share_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "assistant_share_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "coupon_sale_id", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "good_service_share_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "goods_share_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "member_discount_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "recharge_share_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + }, + { + "column": "table_service_share_money", + "count": 74, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + }, + { + "task_code": "ODS_TABLES", + "table_name": "billiards_ods.site_tables_master", + "checked_rows": 100, + "issues": [ + { + "column": "order_id", + "count": 19, + "issue": "API 中的 0 值在 ODS 中变成了 NULL" + } + ] + } + ] +} \ No newline at end of file diff --git a/tmp/backfill_dwd_from_ods.py b/tmp/backfill_dwd_from_ods.py new file mode 100644 index 0000000..393b46f --- /dev/null +++ b/tmp/backfill_dwd_from_ods.py @@ -0,0 +1,283 @@ +# -*- coding: utf-8 -*- +""" +从 ODS 同步回填 DWD 缺失的列值 +""" +import os +import sys +from pathlib import Path + +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + + +# DWD 回填配置: (dwd_table, ods_table, join_condition, [(dwd_col, ods_col), ...]) +BACKFILL_CONFIGS = [ + # dwd_settlement_head + ( + "billiards_dwd.dwd_settlement_head", + "billiards_ods.settlement_records", + "d.order_settle_id = o.id", + [ + ("pl_coupon_sale_amount", "plcouponsaleamount"), + ("mervou_sales_amount", "mervousalesamount"), + ("electricity_money", "electricitymoney"), + ("real_electricity_money", "realelectricitymoney"), + ("electricity_adjust_money", "electricityadjustmoney"), + ] + ), + # dwd_recharge_order + ( + "billiards_dwd.dwd_recharge_order", + "billiards_ods.recharge_settlements", + "d.recharge_order_id = o.id", + [ + ("pl_coupon_sale_amount", "plcouponsaleamount"), + ("mervou_sales_amount", "mervousalesamount"), + ("electricity_money", "electricitymoney"), + ("real_electricity_money", "realelectricitymoney"), + ("electricity_adjust_money", "electricityadjustmoney"), + ] + ), + # dwd_member_balance_change + ( + "billiards_dwd.dwd_member_balance_change", + "billiards_ods.member_balance_changes", + "d.balance_change_id = o.id", + [ + ("principal_before", "principal_before"), + ("principal_after", "principal_after"), + ("principal_change_amount", "principal_data"), + ] + ), + # dim_member + ( + "billiards_dwd.dim_member", + "billiards_ods.member_profiles", + "d.member_id = o.id", + [ + ("pay_money_sum", "pay_money_sum"), + ("recharge_money_sum", "recharge_money_sum"), + ] + ), + # dim_member_ex + ( + "billiards_dwd.dim_member_ex", + "billiards_ods.member_profiles", + "d.member_id = o.id", + [ + ("person_tenant_org_id", "person_tenant_org_id"), + ("person_tenant_org_name", "person_tenant_org_name"), + ("register_source", "register_source"), + ] + ), + # dim_member_card_account + ( + "billiards_dwd.dim_member_card_account", + "billiards_ods.member_stored_value_cards", + "d.member_card_id = o.id", + [ + ("principal_balance", "principal_balance"), + ("member_grade", "member_grade"), + ] + ), + # dim_member_card_account_ex + ( + "billiards_dwd.dim_member_card_account_ex", + "billiards_ods.member_stored_value_cards", + "d.member_card_id = o.id", + [ + ("able_share_member_discount", "able_share_member_discount"), + ("electricity_deduct_radio", "electricity_deduct_radio"), + ("electricity_discount", "electricity_discount"), + ("electricity_card_deduct", "electricitycarddeduct"), + ("recharge_freeze_balance", "rechargefreezebalance"), + ] + ), + # dwd_table_fee_log + ( + "billiards_dwd.dwd_table_fee_log", + "billiards_ods.table_fee_transactions", + "d.table_fee_log_id = o.id", + [ + ("activity_discount_amount", "activity_discount_amount"), + ("real_service_money", "real_service_money"), + ] + ), + # dwd_table_fee_log_ex + ( + "billiards_dwd.dwd_table_fee_log_ex", + "billiards_ods.table_fee_transactions", + "d.table_fee_log_id = o.id", + [ + ("order_consumption_type", "order_consumption_type"), + ] + ), + # dwd_assistant_service_log + ( + "billiards_dwd.dwd_assistant_service_log", + "billiards_ods.assistant_service_records", + "d.assistant_service_id = o.id", + [ + ("real_service_money", "real_service_money"), + ] + ), + # dwd_assistant_service_log_ex + ( + "billiards_dwd.dwd_assistant_service_log_ex", + "billiards_ods.assistant_service_records", + "d.assistant_service_id = o.id", + [ + ("assistant_team_name", "assistantteamname"), + ] + ), + # dwd_store_goods_sale + ( + "billiards_dwd.dwd_store_goods_sale", + "billiards_ods.store_goods_sales_records", + "d.store_goods_sale_id = o.id", + [ + ("coupon_share_money", "coupon_share_money"), + ] + ), + # dwd_groupbuy_redemption + ( + "billiards_dwd.dwd_groupbuy_redemption", + "billiards_ods.group_buy_redemption_records", + "d.redemption_id = o.id", + [ + ("coupon_sale_id", "coupon_sale_id"), + ("member_discount_money", "member_discount_money"), + ] + ), + # dwd_groupbuy_redemption_ex + ( + "billiards_dwd.dwd_groupbuy_redemption_ex", + "billiards_ods.group_buy_redemption_records", + "d.redemption_id = o.id", + [ + ("assistant_share_money", "assistant_share_money"), + ("table_share_money", "table_share_money"), + ("goods_share_money", "goods_share_money"), + ("recharge_share_money", "recharge_share_money"), + ] + ), + # dim_table + ( + "billiards_dwd.dim_table", + "billiards_ods.site_tables_master", + "d.table_id = o.id", + [ + ("order_id", "order_id"), + ] + ), + # dim_store_goods + ( + "billiards_dwd.dim_store_goods", + "billiards_ods.store_goods_master", + "d.site_goods_id = o.id", + [ + ("commodity_code", "commodity_code"), + ("not_sale", "not_sale"), + ] + ), + # dim_tenant_goods + ( + "billiards_dwd.dim_tenant_goods", + "billiards_ods.tenant_goods_master", + "d.tenant_goods_id = o.id", + [ + ("not_sale", "not_sale"), + ] + ), + # dim_groupbuy_package + ( + "billiards_dwd.dim_groupbuy_package", + "billiards_ods.group_buy_packages", + "d.groupbuy_package_id = o.id", + [ + ("sort", "sort"), + ("is_first_limit", "is_first_limit"), + ] + ), +] + + +def column_exists(db, table: str, column: str) -> bool: + schema, tbl = table.split(".") + result = db.query(""" + SELECT 1 FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = %s + """, (schema, tbl, column.lower())) + return bool(result) + + +def main(): + dsn = os.getenv("PG_DSN") + if not dsn: + print("Error: PG_DSN not set") + return + + db = DatabaseConnection(dsn) + + print("=" * 70) + print("DWD Backfill from ODS Script") + print("=" * 70) + + total_updates = 0 + errors = [] + + for dwd_table, ods_table, join_cond, columns in BACKFILL_CONFIGS: + print(f"\n[{dwd_table}]") + + for dwd_col, ods_col in columns: + # Check column exists in both tables + if not column_exists(db, dwd_table, dwd_col): + print(f" {dwd_col}: SKIP (DWD column not found)") + continue + if not column_exists(db, ods_table, ods_col): + print(f" {dwd_col}: SKIP (ODS column {ods_col} not found)") + continue + + # Build UPDATE SQL + sql = f""" + UPDATE {dwd_table} d + SET "{dwd_col}" = o."{ods_col}" + FROM {ods_table} o + WHERE {join_cond} + AND d."{dwd_col}" IS NULL + AND o."{ods_col}" IS NOT NULL + """ + + try: + db.execute(sql) + db.commit() + + # Count non-null + count_sql = f'SELECT COUNT(*) as cnt FROM {dwd_table} WHERE "{dwd_col}" IS NOT NULL' + cnt = db.query(count_sql)[0]["cnt"] + print(f" {dwd_col}: OK (now {cnt} non-null)") + total_updates += 1 + + except Exception as e: + db.rollback() + err_msg = str(e).split("\n")[0][:80] + print(f" {dwd_col}: ERROR - {err_msg}") + errors.append((dwd_table, dwd_col, err_msg)) + + print("\n" + "=" * 70) + print(f"Completed: {total_updates} columns processed") + if errors: + print(f"Errors: {len(errors)}") + for t, c, e in errors: + print(f" - {t}.{c}: {e}") + + db.close() + + +if __name__ == "__main__": + main() diff --git a/tmp/backfill_ods_from_payload.py b/tmp/backfill_ods_from_payload.py new file mode 100644 index 0000000..54c7a5b --- /dev/null +++ b/tmp/backfill_ods_from_payload.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +""" +从 ODS payload 回填缺失的列值 +""" +import os +import sys +from pathlib import Path + +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + + +# 回填配置: (表名, [(db_col, payload_jsonb_expr), ...]) +BACKFILL_CONFIGS = [ + # settlement_records - settleList 内的字段 + ("billiards_ods.settlement_records", [ + ("plcouponsaleamount", "(payload->'settleList')->>'plCouponSaleAmount'"), + ("mervousalesamount", "(payload->'settleList')->>'merVouSalesAmount'"), + ("electricitymoney", "(payload->'settleList')->>'electricityMoney'"), + ("realelectricitymoney", "(payload->'settleList')->>'realElectricityMoney'"), + ("electricityadjustmoney", "(payload->'settleList')->>'electricityAdjustMoney'"), + ]), + # recharge_settlements + ("billiards_ods.recharge_settlements", [ + ("plcouponsaleamount", "(payload->'settleList')->>'plCouponSaleAmount'"), + ("mervousalesamount", "(payload->'settleList')->>'merVouSalesAmount'"), + ("electricitymoney", "(payload->'settleList')->>'electricityMoney'"), + ("realelectricitymoney", "(payload->'settleList')->>'realElectricityMoney'"), + ("electricityadjustmoney", "(payload->'settleList')->>'electricityAdjustMoney'"), + ]), + # member_balance_changes + ("billiards_ods.member_balance_changes", [ + ("principal_before", "payload->>'principal_before'"), + ("principal_after", "payload->>'principal_after'"), + ("principal_data", "payload->>'principal_data'"), + ]), + # member_stored_value_cards + ("billiards_ods.member_stored_value_cards", [ + ("principal_balance", "payload->>'principal_balance'"), + ("member_grade", "payload->>'member_grade'"), + ("rechargefreezebalance", "payload->>'rechargeFreezeBalance'"), + ("able_share_member_discount", "payload->>'able_share_member_discount'"), + ("electricity_deduct_radio", "payload->>'electricity_deduct_radio'"), + ("electricity_discount", "payload->>'electricity_discount'"), + ("electricitycarddeduct", "payload->>'electricityCardDeduct'"), + ]), + # member_profiles + ("billiards_ods.member_profiles", [ + ("pay_money_sum", "payload->>'pay_money_sum'"), + ("recharge_money_sum", "payload->>'recharge_money_sum'"), + ("person_tenant_org_id", "payload->>'person_tenant_org_id'"), + ("person_tenant_org_name", "payload->>'person_tenant_org_name'"), + ("register_source", "payload->>'register_source'"), + ]), + # table_fee_transactions + ("billiards_ods.table_fee_transactions", [ + ("activity_discount_amount", "payload->>'activity_discount_amount'"), + ("real_service_money", "payload->>'real_service_money'"), + ("order_consumption_type", "payload->>'order_consumption_type'"), + ]), + # assistant_service_records + ("billiards_ods.assistant_service_records", [ + ("real_service_money", "payload->>'real_service_money'"), + ("assistantteamname", "payload->>'assistantTeamName'"), + ]), + # store_goods_sales_records + ("billiards_ods.store_goods_sales_records", [ + ("coupon_share_money", "payload->>'coupon_share_money'"), + ]), + # group_buy_redemption_records + ("billiards_ods.group_buy_redemption_records", [ + ("coupon_sale_id", "payload->>'coupon_sale_id'"), + ("member_discount_money", "payload->>'member_discount_money'"), + ("assistant_share_money", "payload->>'assistant_share_money'"), + ("table_share_money", "payload->>'table_share_money'"), + ("goods_share_money", "payload->>'goods_share_money'"), + ("recharge_share_money", "payload->>'recharge_share_money'"), + ]), + # site_tables_master + ("billiards_ods.site_tables_master", [ + ("order_id", "payload->>'order_id'"), + ]), + # store_goods_master + ("billiards_ods.store_goods_master", [ + ("commodity_code", "payload->>'commodity_code'"), + ("not_sale", "payload->>'not_sale'"), + ]), + # table_fee_discount_records + ("billiards_ods.table_fee_discount_records", [ + ("table_name", "payload->>'table_name'"), + ("table_price", "payload->>'table_price'"), + ("charge_free", "payload->>'charge_free'"), + ("area_type_id", "payload->>'area_type_id'"), + ("site_table_area_id", "payload->>'site_table_area_id'"), + ("site_table_area_name", "payload->>'site_table_area_name'"), + ]), + # tenant_goods_master + ("billiards_ods.tenant_goods_master", [ + ("not_sale", "payload->>'not_sale'"), + ]), + # group_buy_packages + ("billiards_ods.group_buy_packages", [ + ("sort", "payload->>'sort'"), + ("is_first_limit", "payload->>'is_first_limit'"), + ]), +] + + +def column_exists(db, table: str, column: str) -> bool: + schema, tbl = table.split(".") + result = db.query(""" + SELECT 1 FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = %s + """, (schema, tbl, column.lower())) + return bool(result) + + +def get_column_type(db, table: str, column: str) -> str: + schema, tbl = table.split(".") + result = db.query(""" + SELECT data_type FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = %s + """, (schema, tbl, column.lower())) + return result[0]["data_type"] if result else "text" + + +def main(): + dsn = os.getenv("PG_DSN") + if not dsn: + print("Error: PG_DSN not set") + return + + db = DatabaseConnection(dsn) + + print("=" * 70) + print("ODS Payload Backfill Script") + print("=" * 70) + + total_updates = 0 + errors = [] + + for table, columns in BACKFILL_CONFIGS: + print(f"\n[{table}]") + + for db_col, payload_expr in columns: + # Check column exists + if not column_exists(db, table, db_col): + print(f" {db_col}: SKIP (column not found)") + continue + + # Get column type for proper casting + col_type = get_column_type(db, table, db_col) + + # Build UPDATE SQL with proper type casting + if col_type in ("numeric", "double precision", "real", "decimal"): + cast_expr = f"({payload_expr})::numeric" + elif col_type in ("integer", "bigint", "smallint"): + cast_expr = f"({payload_expr})::bigint" + elif col_type == "boolean": + cast_expr = f"({payload_expr})::boolean" + elif col_type in ("timestamp", "timestamp with time zone", "timestamp without time zone"): + cast_expr = f"({payload_expr})::timestamp" + else: + cast_expr = payload_expr # text, keep as is + + sql = f""" + UPDATE {table} + SET "{db_col}" = {cast_expr} + WHERE "{db_col}" IS NULL + AND {payload_expr} IS NOT NULL + """ + + try: + db.execute(sql) + db.commit() + + # Count updated + count_sql = f""" + SELECT COUNT(*) as cnt FROM {table} + WHERE "{db_col}" IS NOT NULL + """ + cnt = db.query(count_sql)[0]["cnt"] + print(f" {db_col}: OK (now {cnt} non-null)") + total_updates += 1 + + except Exception as e: + db.rollback() + err_msg = str(e).split("\n")[0][:80] + print(f" {db_col}: ERROR - {err_msg}") + errors.append((table, db_col, err_msg)) + + print("\n" + "=" * 70) + print(f"Completed: {total_updates} columns processed") + if errors: + print(f"Errors: {len(errors)}") + for t, c, e in errors: + print(f" - {t}.{c}: {e}") + + db.close() + + +if __name__ == "__main__": + main() diff --git a/tmp/bd_manual_diff.json b/tmp/bd_manual_diff.json new file mode 100644 index 0000000..969c35a --- /dev/null +++ b/tmp/bd_manual_diff.json @@ -0,0 +1,57 @@ +[ + { + "table": "dim_member_ex", + "missing_in_doc": [], + "extra_in_doc": [ + "1" + ], + "type_mismatches": [], + "doc_path": "etl_billiards\\docs\\bd_manual\\Ex\\BD_manual_dim_member_ex.md" + }, + { + "table": "dim_store_goods", + "missing_in_doc": [], + "extra_in_doc": [ + "1" + ], + "type_mismatches": [], + "doc_path": "etl_billiards\\docs\\bd_manual\\main\\BD_manual_dim_store_goods.md" + }, + { + "table": "dim_table", + "missing_in_doc": [], + "extra_in_doc": [ + "补时长" + ], + "type_mismatches": [], + "doc_path": "etl_billiards\\docs\\bd_manual\\main\\BD_manual_dim_table.md" + }, + { + "table": "dim_table_ex", + "missing_in_doc": [], + "extra_in_doc": [ + "1" + ], + "type_mismatches": [], + "doc_path": "etl_billiards\\docs\\bd_manual\\Ex\\BD_manual_dim_table_ex.md" + }, + { + "table": "dwd_member_balance_change", + "missing_in_doc": [], + "extra_in_doc": [ + "台费专用卡", + "最主要的消费卡种" + ], + "type_mismatches": [], + "doc_path": "etl_billiards\\docs\\bd_manual\\main\\BD_manual_dwd_member_balance_change.md" + }, + { + "table": "dwd_refund_ex", + "missing_in_doc": [], + "extra_in_doc": [ + "1" + ], + "type_mismatches": [], + "doc_path": "etl_billiards\\docs\\bd_manual\\Ex\\BD_manual_dwd_refund_ex.md" + } +] \ No newline at end of file diff --git a/tmp/check_api_ods_issues.py b/tmp/check_api_ods_issues.py new file mode 100644 index 0000000..120a00e --- /dev/null +++ b/tmp/check_api_ods_issues.py @@ -0,0 +1,295 @@ +# -*- coding: utf-8 -*- +""" +排查 API -> ODS 环节的问题： +1. 检测 API 字段在 ODS 表中缺失的列 +2. 检测 API 中的 0 值在 ODS 中是否变成了 NULL +""" +import json +import os +import sys +from datetime import datetime +from decimal import Decimal +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + + +def load_api_ods_comparison(): + """加载已有的 API-ODS 对比文件""" + comparison_file = Path(__file__).parent / "api_ods_comparison.json" + if comparison_file.exists(): + with open(comparison_file, "r", encoding="utf-8") as f: + return json.load(f) + return {} + + +def get_ods_tables_mapping(): + """获取 ODS 任务代码与表名的映射""" + return { + "ODS_ASSISTANT_ACCOUNT": "billiards_ods.assistant_accounts_master", + "ODS_SETTLEMENT_RECORDS": "billiards_ods.settlement_records", + "ODS_TABLE_USE": "billiards_ods.table_fee_transactions", + "ODS_ASSISTANT_LEDGER": "billiards_ods.assistant_service_records", + "ODS_ASSISTANT_ABOLISH": "billiards_ods.assistant_cancellation_records", + "ODS_STORE_GOODS_SALES": "billiards_ods.store_goods_sales_records", + "ODS_PAYMENT": "billiards_ods.payment_transactions", + "ODS_REFUND": "billiards_ods.refund_transactions", + "ODS_PLATFORM_COUPON": "billiards_ods.platform_coupon_redemption_records", + "ODS_MEMBER": "billiards_ods.member_profiles", + "ODS_MEMBER_CARD": "billiards_ods.member_stored_value_cards", + "ODS_MEMBER_BALANCE": "billiards_ods.member_balance_changes", + "ODS_RECHARGE_SETTLE": "billiards_ods.recharge_settlements", + "ODS_GROUP_PACKAGE": "billiards_ods.group_buy_packages", + "ODS_GROUP_BUY_REDEMPTION": "billiards_ods.group_buy_redemption_records", + "ODS_INVENTORY_STOCK": "billiards_ods.goods_stock_summary", + "ODS_INVENTORY_CHANGE": "billiards_ods.goods_stock_movements", + "ODS_TABLES": "billiards_ods.site_tables_master", + "ODS_GOODS_CATEGORY": "billiards_ods.stock_goods_category_tree", + "ODS_STORE_GOODS": "billiards_ods.store_goods_master", + "ODS_TABLE_FEE_DISCOUNT": "billiards_ods.table_fee_discount_records", + "ODS_TENANT_GOODS": "billiards_ods.tenant_goods_master", + } + + +def check_zero_to_null_issues(db: DatabaseConnection, table_name: str, limit: int = 100): + """ + 检查 ODS 表中是否存在 payload 里有 0 值但对应列为 NULL 的情况 + """ + issues = [] + + # 获取表的列信息 + schema, name = table_name.split(".", 1) if "." in table_name else ("public", table_name) + col_sql = """ + SELECT column_name, data_type, udt_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + ORDER BY ordinal_position + """ + + try: + cols = db.query(col_sql, (schema, name)) + except Exception as e: + return {"error": str(e), "issues": []} + + # 筛选数值类型列（可能存在 0 转 NULL 问题） + numeric_cols = [ + c["column_name"] for c in cols + if c["data_type"] in ("integer", "bigint", "smallint", "numeric", "double precision", "real", "decimal") + ] + + # 查询最近的记录，检查 payload 中的值与列值 + check_sql = f""" + SELECT payload, {', '.join(f'"{c}"' for c in numeric_cols)} + FROM {table_name} + WHERE payload IS NOT NULL + ORDER BY fetched_at DESC NULLS LAST + LIMIT %s + """ + + try: + rows = db.query(check_sql, (limit,)) + except Exception as e: + return {"error": str(e), "issues": []} + + zero_to_null_count = {} + + for row in rows: + payload = row.get("payload") + if not payload: + continue + + if isinstance(payload, str): + try: + payload = json.loads(payload) + except: + continue + + if not isinstance(payload, dict): + continue + + # 检查每个数值列 + for col in numeric_cols: + db_value = row.get(col) + + # 从 payload 中获取对应的值（不区分大小写） + payload_value = None + for k, v in payload.items(): + if k.lower() == col.lower(): + payload_value = v + break + + # 检查：payload 中是 0，但数据库中是 NULL + if payload_value == 0 and db_value is None: + if col not in zero_to_null_count: + zero_to_null_count[col] = 0 + zero_to_null_count[col] += 1 + + if zero_to_null_count: + issues = [ + {"column": col, "count": count, "issue": "API 中的 0 值在 ODS 中变成了 NULL"} + for col, count in zero_to_null_count.items() + ] + + return {"issues": issues, "checked_rows": len(rows)} + + +def generate_report(): + """生成完整的排查报告""" + print("=" * 80) + print("API -> ODS 字段排查报告") + print("生成时间:", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + print("=" * 80) + + # 加载对比数据 + comparison = load_api_ods_comparison() + + if not comparison: + print("\n[错误] 未找到 API-ODS 对比文件 (api_ods_comparison.json)") + print("请先运行 compare_api_ods_fields.py 生成对比数据") + return + + # 统计缺失字段 + print("\n" + "=" * 80) + print("一、API 字段在 ODS 表中缺失的情况") + print("=" * 80) + + missing_summary = [] + for task_code, data in comparison.items(): + missing = data.get("missing_in_ods", []) + if missing: + # 过滤掉 siteprofile 等嵌套对象和系统字段 + filtered_missing = [ + f for f in missing + if f.lower() not in ("siteprofile", "settleprofile", "tableprofile", "address", "avatar", + "business_tel", "customer_service_qrcode", "customer_service_wechat", + "fixed_pay_qrcode", "full_address", "latitude", "longitude", + "light_status", "light_token", "light_type", "org_id", "prod_env", + "shop_name", "shop_status", "site_label", "site_type", + "tenant_site_region_id", "wifi_name", "wifi_password", + "attendance_distance", "attendance_enabled", "auto_light") + ] + if filtered_missing: + missing_summary.append({ + "task_code": task_code, + "table_name": data.get("table_name"), + "endpoint": data.get("endpoint"), + "missing_fields": filtered_missing, + }) + + if missing_summary: + for item in missing_summary: + print(f"\n【{item['task_code']}】") + print(f" 表名: {item['table_name']}") + print(f" 端点: {item['endpoint']}") + print(f" 缺失字段 ({len(item['missing_fields'])} 个):") + for field in item['missing_fields']: + print(f" - {field}") + else: + print("\n没有发现明显缺失的业务字段。") + + # 检查 0 转 NULL 问题 + print("\n" + "=" * 80) + print("二、检查 API 中的 0 值在 ODS 中是否变成了 NULL") + print("=" * 80) + + try: + dsn = os.getenv("PG_DSN") + if not dsn: + print("[错误] 未找到 PG_DSN 环境变量") + return + db = DatabaseConnection(dsn) + tables = get_ods_tables_mapping() + + zero_null_issues = [] + for task_code, table_name in tables.items(): + print(f"\n检查 {task_code} ({table_name})...") + result = check_zero_to_null_issues(db, table_name) + + if result.get("error"): + print(f" [错误] {result['error']}") + continue + + if result.get("issues"): + zero_null_issues.append({ + "task_code": task_code, + "table_name": table_name, + "checked_rows": result["checked_rows"], + "issues": result["issues"], + }) + for issue in result["issues"]: + print(f" [发现问题] 列 '{issue['column']}': {issue['count']} 条记录 - {issue['issue']}") + else: + print(f" [正常] 检查了 {result['checked_rows']} 条记录，未发现 0 转 NULL 问题") + + db.close() + + except Exception as e: + print(f"\n[错误] 数据库连接失败: {e}") + zero_null_issues = [] + + # 生成汇总 + print("\n" + "=" * 80) + print("三、问题汇总") + print("=" * 80) + + print("\n1. 需要添加的 ODS 表列:") + if missing_summary: + all_ddl = [] + for item in missing_summary: + table_name = item['table_name'] + for field in item['missing_fields']: + # 根据字段名推断类型 + if field.endswith("_id") or field in ("tenant_id", "member_id", "site_id"): + col_type = "BIGINT" + elif field.endswith("_money") or field.endswith("_amount") or field.endswith("_price"): + col_type = "NUMERIC(18,2)" + elif field.endswith("_time") or field.startswith("create") or field.startswith("update"): + col_type = "TIMESTAMP" + elif field.startswith("is_") or field.endswith("_status"): + col_type = "INTEGER" + else: + col_type = "TEXT" + + ddl = f"ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS {field} {col_type};" + all_ddl.append(ddl) + + print("\n生成的 DDL 语句:") + for ddl in all_ddl: + print(f" {ddl}") + else: + print(" 无") + + print("\n2. 需要修复的 0 转 NULL 问题:") + if zero_null_issues: + for item in zero_null_issues: + print(f"\n 【{item['task_code']}】({item['table_name']})") + for issue in item['issues']: + print(f" - 列 '{issue['column']}': {issue['count']} 条记录受影响") + else: + print(" 未发现明显的 0 转 NULL 问题") + + # 保存报告 + report = { + "generated_at": datetime.now().isoformat(), + "missing_fields": missing_summary, + "zero_to_null_issues": zero_null_issues, + } + + report_path = Path(__file__).parent / "api_ods_issue_report.json" + with open(report_path, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + + print(f"\n报告已保存到: {report_path}") + + return report + + +if __name__ == "__main__": + generate_report() diff --git a/tmp/check_ddl_vs_db.py b/tmp/check_ddl_vs_db.py new file mode 100644 index 0000000..8843e3c --- /dev/null +++ b/tmp/check_ddl_vs_db.py @@ -0,0 +1,231 @@ +# -*- coding: utf-8 -*- +""" +检查 DDL 文件与数据库实际结构的差异 +""" +import psycopg2 +import re +from pathlib import Path + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +def get_db_columns(conn, schema): + """从数据库获取所有表和列""" + sql = """ + SELECT table_name, column_name, data_type, + character_maximum_length, numeric_precision, numeric_scale, + is_nullable + FROM information_schema.columns + WHERE table_schema = %s + ORDER BY table_name, ordinal_position + """ + with conn.cursor() as cur: + cur.execute(sql, (schema,)) + rows = cur.fetchall() + + tables = {} + for row in rows: + table_name = row[0] + col_name = row[1].lower() + data_type = row[2] + char_len = row[3] + num_prec = row[4] + num_scale = row[5] + + # 构建类型字符串 + if data_type == 'character varying': + type_str = f'VARCHAR({char_len})' if char_len else 'VARCHAR' + elif data_type == 'numeric': + type_str = f'NUMERIC({num_prec},{num_scale})' if num_prec else 'NUMERIC' + elif data_type == 'integer': + type_str = 'INTEGER' + elif data_type == 'bigint': + type_str = 'BIGINT' + elif data_type == 'smallint': + type_str = 'SMALLINT' + elif data_type == 'boolean': + type_str = 'BOOLEAN' + elif data_type == 'text': + type_str = 'TEXT' + elif data_type == 'jsonb': + type_str = 'JSONB' + elif data_type == 'json': + type_str = 'JSON' + elif data_type == 'date': + type_str = 'DATE' + elif data_type == 'timestamp with time zone': + type_str = 'TIMESTAMPTZ' + elif data_type == 'timestamp without time zone': + type_str = 'TIMESTAMP' + else: + type_str = data_type.upper() + + if table_name not in tables: + tables[table_name] = {} + tables[table_name][col_name] = type_str + + return tables + +def parse_ddl_file(filepath, default_schema=None): + """解析 DDL 文件，提取表和列定义""" + content = Path(filepath).read_text(encoding='utf-8') + + tables = {} + + # 匹配多种 CREATE TABLE 格式: + # 1. CREATE TABLE schema.table (...) + # 2. CREATE TABLE IF NOT EXISTS schema.table (...) + # 3. CREATE TABLE IF NOT EXISTS table (...) -- 需要 default_schema + # 4. CREATE TABLE table (...) -- 需要 default_schema + table_pattern = re.compile( + r'CREATE TABLE\s+(?:IF NOT EXISTS\s+)?(?:(\w+)\.)?(\w+)\s*\((.*?)\);', + re.DOTALL | re.IGNORECASE + ) + + for match in table_pattern.finditer(content): + schema = match.group(1) or default_schema + table_name = match.group(2) + columns_block = match.group(3) + + columns = {} + # 解析列定义 + for line in columns_block.split('\n'): + line = line.strip() + if not line or line.startswith('--'): + continue + # 跳过约束 + if line.upper().startswith(('PRIMARY KEY', 'CONSTRAINT', 'UNIQUE', 'FOREIGN KEY', 'CHECK', 'EXCLUDE')): + continue + + # 匹配列定义: column_name TYPE ... + col_match = re.match(r'^(\w+)\s+(\w+(?:\s*\([^)]+\))?)', line) + if col_match: + col_name = col_match.group(1).lower() + col_type = col_match.group(2).upper().replace(' ', '') + # 标准化类型 + if col_type == 'INT': + col_type = 'INTEGER' + columns[col_name] = col_type + + tables[table_name] = columns + + return tables + +def compare_schemas(db_tables, ddl_tables, schema_name): + """比较数据库和 DDL 的差异""" + differences = { + 'db_only_tables': [], + 'ddl_only_tables': [], + 'db_only_cols': [], + 'ddl_only_cols': [], + 'type_diff': [] + } + + # 检查 DDL 中有但数据库没有的表 + for table in ddl_tables: + if table not in db_tables: + differences['ddl_only_tables'].append(f"{schema_name}.{table}") + + # 检查数据库中有但 DDL 没有的表 + for table in db_tables: + if table not in ddl_tables: + differences['db_only_tables'].append(f"{schema_name}.{table}") + + # 检查共有表的列差异 + for table in set(db_tables.keys()) & set(ddl_tables.keys()): + db_cols = db_tables[table] + ddl_cols = ddl_tables[table] + + # DDL 有但 DB 没有的列 + for col in ddl_cols: + if col not in db_cols: + differences['ddl_only_cols'].append(f"{schema_name}.{table}.{col} ({ddl_cols[col]})") + + # DB 有但 DDL 没有的列 + for col in db_cols: + if col not in ddl_cols: + differences['db_only_cols'].append(f"{schema_name}.{table}.{col} ({db_cols[col]})") + + return differences + +def main(): + conn = psycopg2.connect(DSN) + + base_dir = Path(__file__).parent.parent / 'etl_billiards' / 'database' + + print("=" * 80) + print("DDL vs DB Structure Comparison") + print("=" * 80) + + # 检查 ODS + print("\n### billiards_ods ###\n") + ods_ddl_file = base_dir / 'schema_ODS_doc.sql' + if ods_ddl_file.exists(): + db_ods = get_db_columns(conn, 'billiards_ods') + ddl_ods = parse_ddl_file(ods_ddl_file, 'billiards_ods') + + print(f"DB tables: {len(db_ods)}") + print(f"DDL tables: {len(ddl_ods)}") + + diff_ods = compare_schemas(db_ods, ddl_ods, 'billiards_ods') + + total_diff = sum(len(v) for v in diff_ods.values()) + if total_diff > 0: + print(f"\nFound {total_diff} differences:") + if diff_ods['db_only_tables']: + print("\n [DB has, DDL missing] Tables:") + for t in sorted(diff_ods['db_only_tables']): + print(f" - {t}") + if diff_ods['ddl_only_tables']: + print("\n [DDL has, DB missing] Tables:") + for t in sorted(diff_ods['ddl_only_tables']): + print(f" - {t}") + if diff_ods['db_only_cols']: + print("\n [DB has, DDL missing] Columns:") + for c in sorted(diff_ods['db_only_cols']): + print(f" - {c}") + if diff_ods['ddl_only_cols']: + print("\n [DDL has, DB missing] Columns:") + for c in sorted(diff_ods['ddl_only_cols']): + print(f" - {c}") + else: + print("\nNo differences found.") + + # 检查 DWD + print("\n### billiards_dwd ###\n") + dwd_ddl_file = base_dir / 'schema_dwd_doc.sql' + if dwd_ddl_file.exists(): + db_dwd = get_db_columns(conn, 'billiards_dwd') + ddl_dwd = parse_ddl_file(dwd_ddl_file, 'billiards_dwd') + + print(f"DB tables: {len(db_dwd)}") + print(f"DDL tables: {len(ddl_dwd)}") + + diff_dwd = compare_schemas(db_dwd, ddl_dwd, 'billiards_dwd') + + total_diff = sum(len(v) for v in diff_dwd.values()) + if total_diff > 0: + print(f"\nFound {total_diff} differences:") + if diff_dwd['db_only_tables']: + print("\n [DB has, DDL missing] Tables:") + for t in sorted(diff_dwd['db_only_tables']): + print(f" - {t}") + if diff_dwd['ddl_only_tables']: + print("\n [DDL has, DB missing] Tables:") + for t in sorted(diff_dwd['ddl_only_tables']): + print(f" - {t}") + if diff_dwd['db_only_cols']: + print("\n [DB has, DDL missing] Columns:") + for c in sorted(diff_dwd['db_only_cols']): + print(f" - {c}") + if diff_dwd['ddl_only_cols']: + print("\n [DDL has, DB missing] Columns:") + for c in sorted(diff_dwd['ddl_only_cols']): + print(f" - {c}") + else: + print("\nNo differences found.") + + conn.close() + print("\n" + "=" * 80) + +if __name__ == '__main__': + main() diff --git a/tmp/check_field_variants.py b/tmp/check_field_variants.py new file mode 100644 index 0000000..0b23b06 --- /dev/null +++ b/tmp/check_field_variants.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +""" +检查缺失字段是否是拼写变体（驼峰式/下划线式、大小写差异等） +""" +import os +import sys +import json +import re +import psycopg2 +from psycopg2.extras import RealDictCursor + +# 配置 +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +def camel_to_snake(name): + """驼峰转下划线""" + s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name) + return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower() + +def snake_to_camel(name): + """下划线转驼峰""" + components = name.split('_') + return components[0] + ''.join(x.title() for x in components[1:]) + +def normalize_field(name): + """标准化字段名 - 去除下划线，全小写""" + return name.lower().replace('_', '') + +def find_variants(api_field, ods_columns): + """查找 API 字段在 ODS 中的可能变体""" + api_lower = api_field.lower() + api_normalized = normalize_field(api_field) + api_snake = camel_to_snake(api_field) + api_camel = snake_to_camel(api_field) + + matches = [] + for ods_col in ods_columns: + ods_lower = ods_col.lower() + ods_normalized = normalize_field(ods_col) + + # 完全匹配 + if api_lower == ods_lower: + matches.append((ods_col, 'exact')) + continue + + # 标准化后匹配（忽略下划线和大小写） + if api_normalized == ods_normalized: + matches.append((ods_col, 'normalized')) + continue + + # 驼峰转下划线匹配 + if api_snake == ods_lower: + matches.append((ods_col, 'camel_to_snake')) + continue + + # 下划线转驼峰匹配 + if api_camel.lower() == ods_lower: + matches.append((ods_col, 'snake_to_camel')) + continue + + # 部分匹配 - 一个是另一个的子串 + if len(api_normalized) > 3 and len(ods_normalized) > 3: + if api_normalized in ods_normalized or ods_normalized in api_normalized: + matches.append((ods_col, 'partial')) + continue + + return matches + +def get_ods_table_columns(conn, table_name): + """获取 ODS 表的字段结构""" + if '.' in table_name: + schema, name = table_name.split('.', 1) + else: + schema, name = 'public', table_name + + sql = """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + ORDER BY ordinal_position + """ + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute(sql, (schema, name)) + rows = cur.fetchall() + + return [row['column_name'] for row in rows] + +def main(): + # 读取之前的对比结果 + json_path = os.path.join(os.path.dirname(__file__), 'api_ods_comparison.json') + with open(json_path, 'r', encoding='utf-8') as f: + results = json.load(f) + + conn = psycopg2.connect(DSN) + + print("=" * 100) + print("缺失字段拼写变体检查") + print("=" * 100) + + all_findings = {} + + for code, data in results.items(): + missing = data.get('missing_in_ods', []) + if not missing: + continue + + table_name = data['table_name'] + ods_columns = get_ods_table_columns(conn, table_name) + + print(f"\n### {code}") + print(f"表名: {table_name}") + + findings = [] + true_missing = [] + + for api_field in missing: + variants = find_variants(api_field, ods_columns) + if variants: + for ods_col, match_type in variants: + findings.append({ + 'api_field': api_field, + 'ods_column': ods_col, + 'match_type': match_type + }) + print(f" [发现变体] API: `{api_field}` -> ODS: `{ods_col}` ({match_type})") + else: + true_missing.append(api_field) + + if findings: + all_findings[code] = { + 'table_name': table_name, + 'variants': findings, + 'true_missing': true_missing + } + + if true_missing: + print(f"\n **确认缺失 ({len(true_missing)}):**") + for f in true_missing: + print(f" - {f}") + + conn.close() + + # 输出汇总 + print("\n") + print("=" * 100) + print("汇总 - 发现的拼写变体") + print("=" * 100) + + for code, data in all_findings.items(): + if data['variants']: + print(f"\n### {code} (`{data['table_name']}`)") + print("\n| API 字段 | ODS 字段 | 匹配类型 |") + print("|----------|----------|----------|") + for v in data['variants']: + print(f"| `{v['api_field']}` | `{v['ods_column']}` | {v['match_type']} |") + + print("\n") + print("=" * 100) + print("汇总 - 确认缺失的字段（无变体）") + print("=" * 100) + + for code, data in results.items(): + missing = data.get('missing_in_ods', []) + if not missing: + continue + + if code in all_findings: + true_missing = all_findings[code]['true_missing'] + else: + true_missing = missing + + if true_missing: + print(f"\n### {code} (`{data['table_name']}`)") + print(f"缺失 {len(true_missing)} 个字段:") + print("\n| 字段名 | 说明 |") + print("|--------|------|") + for f in true_missing: + print(f"| `{f}` | |") + +if __name__ == '__main__': + main() diff --git a/tmp/check_new_fields_data.py b/tmp/check_new_fields_data.py new file mode 100644 index 0000000..f6cd524 --- /dev/null +++ b/tmp/check_new_fields_data.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- +"""检查新添加字段的数据完整性""" +import psycopg2 + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +# 新添加的字段列表 +NEW_FIELDS = { + # DWD 主表 + 'billiards_dwd.dwd_settlement_head': [ + 'electricity_money', 'real_electricity_money', 'electricity_adjust_money', + 'pl_coupon_sale_amount', 'mervou_sales_amount' + ], + 'billiards_dwd.dwd_table_fee_log': ['activity_discount_amount', 'real_service_money'], + 'billiards_dwd.dwd_table_fee_adjust': ['table_name', 'table_price', 'charge_free'], + 'billiards_dwd.dim_member': ['pay_money_sum', 'recharge_money_sum'], + 'billiards_dwd.dim_member_card_account': ['principal_balance', 'member_grade'], + 'billiards_dwd.dim_store_goods': ['commodity_code', 'not_sale'], + 'billiards_dwd.dim_table': ['order_id'], + 'billiards_dwd.dim_tenant_goods': ['not_sale'], + 'billiards_dwd.dim_groupbuy_package': ['sort', 'is_first_limit'], + 'billiards_dwd.dwd_assistant_service_log': ['real_service_money'], + 'billiards_dwd.dwd_assistant_trash_event': ['tenant_id'], + 'billiards_dwd.dwd_groupbuy_redemption': ['member_discount_money', 'coupon_sale_id'], + 'billiards_dwd.dwd_member_balance_change': ['principal_before', 'principal_after'], + 'billiards_dwd.dwd_payment': ['tenant_id'], + 'billiards_dwd.dwd_store_goods_sale': ['coupon_share_money'], +} + +def check_field_data(conn, schema_table, fields): + """检查字段的数据情况""" + results = [] + schema, table = schema_table.split('.') + + cur = conn.cursor() + + # 获取总行数 + cur.execute(f"SELECT COUNT(*) FROM {schema_table}") + total_rows = cur.fetchone()[0] + + for field in fields: + try: + # 非空计数 + cur.execute(f"SELECT COUNT(*) FROM {schema_table} WHERE {field} IS NOT NULL") + non_null_count = cur.fetchone()[0] + + # 非空非零计数（对于数值类型） + cur.execute(f""" + SELECT COUNT(*) FROM {schema_table} + WHERE {field} IS NOT NULL + AND CAST({field} AS TEXT) NOT IN ('0', '0.00', '0.0', '') + """) + non_zero_count = cur.fetchone()[0] + + results.append({ + 'field': field, + 'total': total_rows, + 'non_null': non_null_count, + 'non_zero': non_zero_count, + 'fill_rate': f"{non_null_count/total_rows*100:.1f}%" if total_rows > 0 else "N/A" + }) + except Exception as e: + results.append({ + 'field': field, + 'error': str(e)[:50] + }) + + cur.close() + return results + +def main(): + conn = psycopg2.connect(DSN) + + print("=" * 90) + print("New Fields Data Completeness Check") + print("=" * 90) + + for table, fields in NEW_FIELDS.items(): + print(f"\n### {table} ###\n") + results = check_field_data(conn, table, fields) + + print(f"{'Field':<30} {'Total':>8} {'Non-Null':>10} {'Non-Zero':>10} {'Fill Rate':>10}") + print("-" * 70) + + for r in results: + if 'error' in r: + print(f"{r['field']:<30} ERROR: {r['error']}") + else: + print(f"{r['field']:<30} {r['total']:>8} {r['non_null']:>10} {r['non_zero']:>10} {r['fill_rate']:>10}") + + conn.close() + print("\n" + "=" * 90) + +if __name__ == '__main__': + main() diff --git a/tmp/check_scd2_tables.py b/tmp/check_scd2_tables.py new file mode 100644 index 0000000..ae46f71 --- /dev/null +++ b/tmp/check_scd2_tables.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +"""检查 DWD 维度表 SCD2 配置""" +import os +import sys +from pathlib import Path + +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + +dsn = os.getenv("PG_DSN") +db = DatabaseConnection(dsn) + +print("=" * 70) +print("DWD Dimension Tables - SCD2 Analysis") +print("=" * 70) + +# 获取所有维度表 +tables = db.query(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'billiards_dwd' + AND table_name LIKE 'dim_%' + ORDER BY table_name +""") + +scd_cols = {"scd2_start_time", "scd2_end_time", "scd2_is_current", "scd2_version"} + +scd2_tables = [] +type1_tables = [] + +for t in tables: + tbl = t["table_name"] + cols = db.query(""" + SELECT column_name FROM information_schema.columns + WHERE table_schema = 'billiards_dwd' AND table_name = %s + """, (tbl,)) + col_names = {c["column_name"].lower() for c in cols} + + has_scd = col_names & scd_cols + if has_scd: + scd2_tables.append((tbl, has_scd)) + else: + type1_tables.append(tbl) + +print("\n[SCD2 Tables - History Tracking]") +print("-" * 50) +if scd2_tables: + for tbl, cols in scd2_tables: + print(f" {tbl}") + print(f" SCD2 cols: {', '.join(sorted(cols))}") +else: + print(" (none)") + +print(f"\n[Type1 Tables - Direct Overwrite] ({len(type1_tables)} tables)") +print("-" * 50) +for tbl in type1_tables: + print(f" {tbl}") + +print("\n" + "=" * 70) +print("Processing Logic") +print("=" * 70) +print(""" +Code path in dwd_load_task.py: + + if table.startswith('dim_'): + _merge_dim() + | + +-- if has SCD2 columns: + | _merge_dim_scd2() + | -> Compare data, close old version, insert new version + | -> Uses INSERT (no ON CONFLICT) + | -> SCD2 NOT affected by fact_upsert config + | + +-- else: + _merge_dim_type1_upsert() + -> Uses ON CONFLICT DO UPDATE + -> Direct overwrite (Type1) + else: + _load_fact_generic() + -> Uses ON CONFLICT DO UPDATE (if fact_upsert=true) + +CONCLUSION: SCD2 logic is INDEPENDENT, NOT affected by conflict mode settings. +""") + +db.close() diff --git a/tmp/check_seq.py b/tmp/check_seq.py new file mode 100644 index 0000000..fe0aeaa --- /dev/null +++ b/tmp/check_seq.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +import re +from pathlib import Path + +docs = list(Path('etl_billiards/docs/bd_manual/main').glob('*.md')) + \ + list(Path('etl_billiards/docs/bd_manual/Ex').glob('*.md')) + +for doc in docs: + content = doc.read_text(encoding='utf-8') + lines = content.split('\n') + seqs = [] + for line in lines: + match = re.match(r'\|\s*(\d+)\s*\|', line) + if match: + seq = int(match.group(1)) + seqs.append((seq, line[:70])) + + # 检查是否有重复序号 + seq_nums = [s[0] for s in seqs] + if len(seq_nums) != len(set(seq_nums)): + print(f'\n{doc.name}: Duplicate sequences found') + seen = set() + for seq, line in seqs: + if seq in seen or seq_nums.count(seq) > 1: + print(f' {seq}: {line}...') + seen.add(seq) diff --git a/tmp/compare_api_ods_fields.py b/tmp/compare_api_ods_fields.py new file mode 100644 index 0000000..8664756 --- /dev/null +++ b/tmp/compare_api_ods_fields.py @@ -0,0 +1,510 @@ +# -*- coding: utf-8 -*- +""" +对比 API 返回字段和 ODS 表字段，找出 ODS 中缺少的 API 字段 +""" +import os +import sys +import json +import requests +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import psycopg2 +from psycopg2.extras import RealDictCursor + +# 配置 +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' +API_BASE = 'https://pc.ficoo.vip/apiprod/admin/v1/' +API_TOKEN = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IktlbTVsdHRqZ2tSUExOcVA2ajhNakdQYnFrNW5mRzBQNzRvMHE0b295VVE9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzIvOCDkuIvljYg2OjU3OjA1IiwibmVlZENoZWNrVG9rZW4iOiJmYWxzZSIsImV4cCI6MTc3MDU0ODIyNSwiaXNzIjoidGVzdCIsImF1ZCI6IlVzZXIifQ.wJlm7pTqUzp769nUGdxx0e1bVMy4x9Prp9U_UMWQvlk' +STORE_ID = '2790685415443269' +TZ = ZoneInfo('Asia/Taipei') + +# ODS 任务配置 +ODS_SPECS = [ + { + 'code': 'ODS_ASSISTANT_ACCOUNT', + 'table_name': 'billiards_ods.assistant_accounts_master', + 'endpoint': '/PersonnelManagement/SearchAssistantInfo', + 'data_path': ['data'], + 'list_key': 'assistantInfos', + 'requires_window': True, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_SETTLEMENT_RECORDS', + 'table_name': 'billiards_ods.settlement_records', + 'endpoint': '/Site/GetAllOrderSettleList', + 'data_path': ['data'], + 'list_key': 'settleList', + 'requires_window': True, + 'time_fields': ('rangeStartTime', 'rangeEndTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_TABLE_USE', + 'table_name': 'billiards_ods.table_fee_transactions', + 'endpoint': '/Site/GetSiteTableOrderDetails', + 'data_path': ['data'], + 'list_key': 'siteTableUseDetailsList', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_ASSISTANT_LEDGER', + 'table_name': 'billiards_ods.assistant_service_records', + 'endpoint': '/AssistantPerformance/GetOrderAssistantDetails', + 'data_path': ['data'], + 'list_key': 'orderAssistantDetails', + 'requires_window': True, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_ASSISTANT_ABOLISH', + 'table_name': 'billiards_ods.assistant_cancellation_records', + 'endpoint': '/AssistantPerformance/GetAbolitionAssistant', + 'data_path': ['data'], + 'list_key': 'abolitionAssistants', + 'requires_window': True, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_STORE_GOODS_SALES', + 'table_name': 'billiards_ods.store_goods_sales_records', + 'endpoint': '/TenantGoods/GetGoodsSalesList', + 'data_path': ['data'], + 'list_key': 'orderGoodsLedgers', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_PAYMENT', + 'table_name': 'billiards_ods.payment_transactions', + 'endpoint': '/PayLog/GetPayLogListPage', + 'data_path': ['data'], + 'list_key': None, + 'requires_window': False, + 'time_fields': ('StartPayTime', 'EndPayTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_REFUND', + 'table_name': 'billiards_ods.refund_transactions', + 'endpoint': '/Order/GetRefundPayLogList', + 'data_path': ['data'], + 'list_key': None, + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_PLATFORM_COUPON', + 'table_name': 'billiards_ods.platform_coupon_redemption_records', + 'endpoint': '/Promotion/GetOfflineCouponConsumePageList', + 'data_path': ['data'], + 'list_key': None, + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_MEMBER', + 'table_name': 'billiards_ods.member_profiles', + 'endpoint': '/MemberProfile/GetTenantMemberList', + 'data_path': ['data'], + 'list_key': 'tenantMemberInfos', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_MEMBER_CARD', + 'table_name': 'billiards_ods.member_stored_value_cards', + 'endpoint': '/MemberProfile/GetTenantMemberCardList', + 'data_path': ['data'], + 'list_key': 'tenantMemberCards', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_MEMBER_BALANCE', + 'table_name': 'billiards_ods.member_balance_changes', + 'endpoint': '/MemberProfile/GetMemberCardBalanceChange', + 'data_path': ['data'], + 'list_key': 'tenantMemberCardLogs', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_RECHARGE_SETTLE', + 'table_name': 'billiards_ods.recharge_settlements', + 'endpoint': '/Site/GetRechargeSettleList', + 'data_path': ['data'], + 'list_key': 'settleList', + 'requires_window': True, + 'time_fields': ('rangeStartTime', 'rangeEndTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_GROUP_PACKAGE', + 'table_name': 'billiards_ods.group_buy_packages', + 'endpoint': '/PackageCoupon/QueryPackageCouponList', + 'data_path': ['data'], + 'list_key': 'packageCouponList', + 'requires_window': False, + 'time_fields': None, + 'include_site_id': True, + }, + { + 'code': 'ODS_GROUP_BUY_REDEMPTION', + 'table_name': 'billiards_ods.group_buy_redemption_records', + 'endpoint': '/Site/GetSiteTableUseDetails', + 'data_path': ['data'], + 'list_key': 'siteTableUseDetailsList', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_INVENTORY_STOCK', + 'table_name': 'billiards_ods.goods_stock_summary', + 'endpoint': '/TenantGoods/GetGoodsStockReport', + 'data_path': ['data'], + 'list_key': None, + 'requires_window': False, + 'time_fields': None, + 'include_site_id': True, + }, + { + 'code': 'ODS_INVENTORY_CHANGE', + 'table_name': 'billiards_ods.goods_stock_movements', + 'endpoint': '/GoodsStockManage/QueryGoodsOutboundReceipt', + 'data_path': ['data'], + 'list_key': 'queryDeliveryRecordsList', + 'requires_window': True, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_TABLES', + 'table_name': 'billiards_ods.site_tables_master', + 'endpoint': '/Table/GetSiteTables', + 'data_path': ['data'], + 'list_key': 'siteTables', + 'requires_window': False, + 'time_fields': None, + 'include_site_id': True, + }, + { + 'code': 'ODS_GOODS_CATEGORY', + 'table_name': 'billiards_ods.stock_goods_category_tree', + 'endpoint': '/TenantGoodsCategory/QueryPrimarySecondaryCategory', + 'data_path': ['data'], + 'list_key': 'goodsCategoryList', + 'requires_window': False, + 'time_fields': None, + 'include_site_id': True, + }, + { + 'code': 'ODS_STORE_GOODS', + 'table_name': 'billiards_ods.store_goods_master', + 'endpoint': '/TenantGoods/GetGoodsInventoryList', + 'data_path': ['data'], + 'list_key': 'orderGoodsList', + 'requires_window': False, + 'time_fields': None, + 'include_site_id': True, + 'site_id_array': True, # 需要数组格式 + }, + { + 'code': 'ODS_TABLE_FEE_DISCOUNT', + 'table_name': 'billiards_ods.table_fee_discount_records', + 'endpoint': '/Site/GetTaiFeeAdjustList', + 'data_path': ['data'], + 'list_key': 'taiFeeAdjustInfos', + 'requires_window': False, + 'time_fields': ('startTime', 'endTime'), + 'include_site_id': True, + }, + { + 'code': 'ODS_TENANT_GOODS', + 'table_name': 'billiards_ods.tenant_goods_master', + 'endpoint': '/TenantGoods/QueryTenantGoods', + 'data_path': ['data'], + 'list_key': 'tenantGoodsList', + 'requires_window': False, + 'time_fields': None, + 'include_site_id': True, + }, +] + + +def get_ods_table_columns(conn, table_name: str) -> dict: + """获取 ODS 表的字段结构""" + if '.' in table_name: + schema, name = table_name.split('.', 1) + else: + schema, name = 'public', table_name + + sql = """ + SELECT column_name, data_type, udt_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + ORDER BY ordinal_position + """ + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute(sql, (schema, name)) + rows = cur.fetchall() + + return {row['column_name'].lower(): row for row in rows} + + +def flatten_json_keys(obj, prefix='', depth=0) -> set: + """递归展平 JSON 获取所有字段名，限制深度""" + if depth > 3: # 限制深度 + return set() + keys = set() + if isinstance(obj, dict): + for k, v in obj.items(): + full_key = f"{prefix}.{k}" if prefix else k + keys.add(k) # 添加不带前缀的 + if isinstance(v, (dict, list)) and depth < 3: + keys.update(flatten_json_keys(v, full_key, depth + 1)) + elif isinstance(obj, list): + for item in obj[:5]: # 只检查前5个 + keys.update(flatten_json_keys(item, prefix, depth)) + return keys + + +def call_api(endpoint: str, params: dict) -> dict: + """调用 API""" + url = API_BASE.rstrip('/') + '/' + endpoint.lstrip('/') + headers = { + 'Authorization': f'Bearer {API_TOKEN}', + 'Content-Type': 'application/json', + 'Accept': 'application/json', + } + try: + resp = requests.post(url, json=params, headers=headers, timeout=30) + resp.raise_for_status() + return resp.json() + except Exception as e: + print(f" API 调用异常: {e}") + return {} + + +def extract_list(payload: dict, data_path: list, list_key: str = None) -> list: + """从响应中提取列表""" + cur = payload + for key in data_path: + if isinstance(cur, dict): + cur = cur.get(key) + else: + return [] + + if isinstance(cur, list): + return cur + + if isinstance(cur, dict): + if list_key and list_key in cur: + return cur[list_key] + # 尝试常见的列表键 + for k in ['list', 'rows', 'records', 'items', 'dataList']: + if k in cur and isinstance(cur[k], list): + return cur[k] + # 返回字典的第一个列表值 + for v in cur.values(): + if isinstance(v, list): + return v + + return [] + + +def get_api_sample_data(spec: dict, window_start: datetime, window_end: datetime) -> list: + """从 API 获取示例数据""" + params = {'page': 1, 'limit': 50} + + if spec.get('include_site_id'): + if spec.get('site_id_array'): + params['siteId'] = [int(STORE_ID)] + else: + params['siteId'] = int(STORE_ID) + + time_fields = spec.get('time_fields') + if time_fields: + start_key, end_key = time_fields + params[start_key] = window_start.strftime('%Y-%m-%d %H:%M:%S') + params[end_key] = window_end.strftime('%Y-%m-%d %H:%M:%S') + + payload = call_api(spec['endpoint'], params) + if not payload: + return [] + + records = extract_list(payload, spec['data_path'], spec.get('list_key')) + return records + + +def compare_fields(api_fields: set, ods_columns: dict) -> dict: + """比较 API 字段和 ODS 列""" + ods_col_names = set(ods_columns.keys()) + + # 需要排除的 ODS 系统字段 + system_cols = { + 'payload', 'source_file', 'source_endpoint', 'fetched_at', + 'content_hash', 'record_index', 'site_profile' + } + + # siteProfile 嵌套字段 - 忽略这些门店配置字段 + site_profile_fields = { + 'address', 'full_address', 'latitude', 'longitude', + 'shop_name', 'shop_status', 'site_label', 'site_type', + 'tenant_site_region_id', 'attendance_distance', 'attendance_enabled', + 'auto_light', 'avatar', 'business_tel', 'customer_service_qrcode', + 'customer_service_wechat', 'fixed_pay_qrcode', 'light_status', + 'light_token', 'light_type', 'prod_env', 'wifi_name', 'wifi_password', + 'org_id', 'siteprofile', 'ewelink_client_id' + } + + # API 字段标准化 + api_fields_lower = {f.lower() for f in api_fields} + + # 在 ODS 中缺失的 API 字段（排除系统字段和 siteProfile 字段） + missing_in_ods = api_fields_lower - ods_col_names - system_cols - site_profile_fields + + # 在 API 中没有但 ODS 有的字段（可能是衍生字段） + ods_only = ods_col_names - api_fields_lower - system_cols + + return { + 'api_fields': sorted(api_fields_lower), + 'ods_columns': sorted(ods_col_names), + 'missing_in_ods': sorted(missing_in_ods), + 'ods_only': sorted(ods_only), + } + + +def main(): + print("=" * 80) + print("API vs ODS 字段对比分析") + print("=" * 80) + + # 连接数据库 + conn = psycopg2.connect(DSN) + print(f"数据库连接成功") + + print(f"API: {API_BASE}") + print(f"门店 ID: {STORE_ID}") + + # 时间窗口：2025-12-01 到现在 + now = datetime.now(TZ) + window_end = now + window_start = datetime(2025, 12, 1, 0, 0, 0, tzinfo=TZ) + print(f"时间窗口: {window_start.strftime('%Y-%m-%d %H:%M:%S')} ~ {window_end.strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 80) + + results = {} + + for spec in ODS_SPECS: + print(f"\n处理: {spec['code']}") + print(f" 表名: {spec['table_name']}") + print(f" 端点: {spec['endpoint']}") + + # 获取 ODS 表结构 + ods_columns = get_ods_table_columns(conn, spec['table_name']) + if not ods_columns: + print(f" [跳过] ODS 表不存在或无字段") + continue + print(f" ODS 字段数: {len(ods_columns)}") + + # 获取 API 数据 + records = get_api_sample_data(spec, window_start, window_end) + print(f" API 返回记录数: {len(records)}") + + if not records: + results[spec['code']] = { + 'table_name': spec['table_name'], + 'endpoint': spec['endpoint'], + 'api_records': 0, + 'ods_columns': list(ods_columns.keys()), + 'missing_in_ods': [], + 'note': 'API 无返回数据' + } + continue + + # 提取 API 字段 + api_fields = set() + for rec in records[:20]: # 检查前20条 + if isinstance(rec, dict): + api_fields.update(flatten_json_keys(rec)) + print(f" API 字段数: {len(api_fields)}") + + # 对比 + comparison = compare_fields(api_fields, ods_columns) + + results[spec['code']] = { + 'table_name': spec['table_name'], + 'endpoint': spec['endpoint'], + 'api_records': len(records), + 'api_fields_count': len(comparison['api_fields']), + 'ods_columns_count': len(comparison['ods_columns']), + 'missing_in_ods': comparison['missing_in_ods'], + 'ods_only': comparison['ods_only'], + 'api_fields': comparison['api_fields'], + 'ods_columns': comparison['ods_columns'], + } + + if comparison['missing_in_ods']: + print(f" [!] ODS 缺少 {len(comparison['missing_in_ods'])} 个字段:") + for f in comparison['missing_in_ods'][:10]: + print(f" - {f}") + if len(comparison['missing_in_ods']) > 10: + print(f" ... 还有 {len(comparison['missing_in_ods']) - 10} 个") + else: + print(f" [OK] ODS 已包含所有 API 字段") + + conn.close() + + # 输出汇总表格 + print("\n") + print("=" * 80) + print("汇总报告 - 每个 ODS 表缺少的 API 字段") + print("=" * 80) + + for code, data in results.items(): + missing = data.get('missing_in_ods', []) + if missing or data.get('note'): + print(f"\n### {code}") + print(f"表名: `{data['table_name']}`") + print(f"端点: `{data['endpoint']}`") + print(f"API 记录数: {data.get('api_records', 0)}") + + if missing: + print(f"\n**ODS 缺少的字段 ({len(missing)}):**\n") + print("| 字段名 | 说明 |") + print("|--------|------|") + for f in missing: + print(f"| `{f}` | |") + elif data.get('note'): + print(f"\n备注: {data['note']}") + + # 显示没有缺失的表 + print("\n\n### 已完整的表（无缺失字段）") + for code, data in results.items(): + missing = data.get('missing_in_ods', []) + if not missing and not data.get('note'): + print(f"- {code}: `{data['table_name']}` [OK]") + + # 保存详细结果 + output_file = os.path.join(os.path.dirname(__file__), 'api_ods_comparison.json') + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(results, f, ensure_ascii=False, indent=2) + print(f"\n\n详细结果已保存至: {output_file}") + + +if __name__ == '__main__': + main() diff --git a/tmp/detailed_field_compare.py b/tmp/detailed_field_compare.py new file mode 100644 index 0000000..9a6d93c --- /dev/null +++ b/tmp/detailed_field_compare.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +""" +详细双向对比 - 针对可能相关的字段 +""" +import os +import json +import re +import psycopg2 +from psycopg2.extras import RealDictCursor + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +# 需要详细审核的字段对 +REVIEW_PAIRS = [ + { + 'code': 'ODS_TABLE_USE', + 'table': 'billiards_ods.table_fee_transactions', + 'keywords': ['service', 'money', 'real'], + }, + { + 'code': 'ODS_ASSISTANT_LEDGER', + 'table': 'billiards_ods.assistant_service_records', + 'keywords': ['service', 'money', 'real'], + }, + { + 'code': 'ODS_MEMBER_CARD', + 'table': 'billiards_ods.member_stored_value_cards', + 'keywords': ['balance', 'principal', 'freeze', 'recharge'], + }, + { + 'code': 'ODS_MEMBER_BALANCE', + 'table': 'billiards_ods.member_balance_changes', + 'keywords': ['before', 'after', 'principal', 'change'], + }, + { + 'code': 'ODS_SETTLEMENT_RECORDS', + 'table': 'billiards_ods.settlement_records', + 'keywords': ['coupon', 'sale', 'amount', 'pl', 'tenant'], + }, + { + 'code': 'ODS_RECHARGE_SETTLE', + 'table': 'billiards_ods.recharge_settlements', + 'keywords': ['coupon', 'sale', 'amount', 'pl', 'tenant'], + }, + { + 'code': 'ODS_GROUP_PACKAGE', + 'table': 'billiards_ods.group_buy_packages', + 'keywords': ['table', 'area', 'name', 'list', 'tenant'], + }, +] + +def get_ods_columns(conn, table_name): + """获取 ODS 表字段""" + if '.' in table_name: + schema, name = table_name.split('.', 1) + else: + schema, name = 'public', table_name + + sql = """ + SELECT column_name, data_type + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + ORDER BY ordinal_position + """ + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute(sql, (schema, name)) + return {row['column_name']: row['data_type'] for row in cur.fetchall()} + +def normalize(s): + """标准化：去除下划线，全小写""" + return s.lower().replace('_', '') + +def filter_by_keywords(fields, keywords): + """按关键词筛选字段""" + result = [] + for f in fields: + f_norm = normalize(f) + for kw in keywords: + if kw in f_norm: + result.append(f) + break + return sorted(set(result)) + +def main(): + # 读取 API 字段 + json_path = os.path.join(os.path.dirname(__file__), 'api_ods_comparison.json') + with open(json_path, 'r', encoding='utf-8') as f: + results = json.load(f) + + conn = psycopg2.connect(DSN) + + print("=" * 100) + print("双向详细对比 - 可能相关的字段") + print("=" * 100) + + for review in REVIEW_PAIRS: + code = review['code'] + table = review['table'] + keywords = review['keywords'] + + if code not in results: + continue + + data = results[code] + api_fields = data.get('api_fields', []) + + # 获取 ODS 字段 + ods_columns = get_ods_columns(conn, table) + + # 按关键词筛选 + api_related = filter_by_keywords(api_fields, keywords) + ods_related = filter_by_keywords(ods_columns.keys(), keywords) + + print(f"\n{'='*80}") + print(f"### {code}") + print(f"表: {table}") + print(f"关键词: {keywords}") + print(f"{'='*80}") + + print(f"\n**API 相关字段 ({len(api_related)}):**") + for f in api_related: + print(f" - {f}") + + print(f"\n**ODS 相关字段 ({len(ods_related)}):**") + for f in ods_related: + dtype = ods_columns.get(f, '') + print(f" - {f} ({dtype})") + + # 匹配分析 + print(f"\n**匹配分析:**") + + # 建立映射 + matched_api = set() + matched_ods = set() + mappings = [] + + for api_f in api_related: + api_norm = normalize(api_f) + for ods_f in ods_related: + ods_norm = normalize(ods_f) + + # 完全匹配 + if api_norm == ods_norm: + mappings.append((api_f, ods_f, 'exact', '完全匹配')) + matched_api.add(api_f) + matched_ods.add(ods_f) + # 包含关系 + elif api_norm in ods_norm or ods_norm in api_norm: + if api_f not in matched_api: + mappings.append((api_f, ods_f, 'partial', '部分匹配')) + + if mappings: + print("\n| API 字段 | ODS 字段 | 类型 | 说明 |") + print("|----------|----------|------|------|") + for api_f, ods_f, mtype, desc in mappings: + print(f"| `{api_f}` | `{ods_f}` | {mtype} | {desc} |") + + # 未匹配的 API 字段 + unmatched_api = set(api_related) - matched_api + if unmatched_api: + print(f"\n**API 未匹配字段:**") + for f in sorted(unmatched_api): + print(f" - {f}") + + # 未匹配的 ODS 字段 + unmatched_ods = set(ods_related) - matched_ods + if unmatched_ods: + print(f"\n**ODS 未匹配字段:**") + for f in sorted(unmatched_ods): + print(f" - {f}") + + conn.close() + + # 输出最终结论 + print("\n") + print("=" * 100) + print("最终审核结论") + print("=" * 100) + +if __name__ == '__main__': + main() diff --git a/tmp/dwd_schema.json b/tmp/dwd_schema.json new file mode 100644 index 0000000..d059e8b --- /dev/null +++ b/tmp/dwd_schema.json @@ -0,0 +1,5404 @@ +{ + "dim_assistant": [ + { + "column": "assistant_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_no", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "real_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "nickname", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "mobile", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "team_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "team_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "level", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "entry_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "resign_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "leave_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "assistant_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_assistant_ex": [ + { + "column": "assistant_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "gender", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "birth_date", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "avatar", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "introduce", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "video_introduction_url", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "height", + "type": "numeric", + "nullable": "YES", + "max_length": 5 + }, + { + "column": "weight", + "type": "numeric", + "nullable": "YES", + "max_length": 5 + }, + { + "column": "shop_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "group_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "group_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "person_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "staff_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "staff_profile_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_grade", + "type": "double precision", + "nullable": "YES", + "max_length": 53 + }, + { + "column": "sum_grade", + "type": "double precision", + "nullable": "YES", + "max_length": 53 + }, + { + "column": "get_grade_times", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "charge_way", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "allow_cx", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_guaranteed", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "salary_grant_enabled", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "entry_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "entry_sign_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "resign_sign_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "work_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "show_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "show_sort", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "online_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "criticism_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "update_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "start_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "last_table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "last_table_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "last_update_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ding_talk_synced", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "site_light_cfg_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "light_equipment_id", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "light_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_team_leader", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "serial_number", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_goods_category": [ + { + "column": "category_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "category_name", + "type": "character varying", + "nullable": "YES", + "max_length": 50 + }, + { + "column": "alias_name", + "type": "character varying", + "nullable": "YES", + "max_length": 50 + }, + { + "column": "parent_category_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "business_name", + "type": "character varying", + "nullable": "YES", + "max_length": 50 + }, + { + "column": "tenant_goods_business_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "category_level", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_leaf", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "open_salesman", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sort_order", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_warehousing", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_groupbuy_package": [ + { + "column": "groupbuy_package_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "package_name", + "type": "character varying", + "nullable": "YES", + "max_length": 200 + }, + { + "column": "package_template_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "selling_price", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "coupon_face_value", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "duration_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "start_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "table_area_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "is_enabled", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_table_area_id_list", + "type": "character varying", + "nullable": "YES", + "max_length": 512 + }, + { + "column": "card_type_ids", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sort", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_first_limit", + "type": "boolean", + "nullable": "YES", + "max_length": null + } + ], + "dim_groupbuy_package_ex": [ + { + "column": "groupbuy_package_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "usable_count", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "date_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "usable_range", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "date_info", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "start_clock", + "type": "character varying", + "nullable": "YES", + "max_length": 16 + }, + { + "column": "end_clock", + "type": "character varying", + "nullable": "YES", + "max_length": 16 + }, + { + "column": "add_start_clock", + "type": "character varying", + "nullable": "YES", + "max_length": 16 + }, + { + "column": "add_end_clock", + "type": "character varying", + "nullable": "YES", + "max_length": 16 + }, + { + "column": "area_tag_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_area_id_list", + "type": "character varying", + "nullable": "YES", + "max_length": 512 + }, + { + "column": "group_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "system_group_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "package_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "effective_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "max_selectable_categories", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "creator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "tenant_coupon_sale_order_item_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dim_member": [ + { + "column": "member_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "system_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "register_site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "mobile", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "nickname", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_card_grade_code", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_card_grade_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "update_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "pay_money_sum", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "recharge_money_sum", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dim_member_card_account": [ + { + "column": "member_card_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "register_site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "system_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "card_type_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_card_grade_code", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_card_grade_code_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_card_type_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_mobile", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "balance", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "start_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "last_consume_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "principal_balance", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "member_grade", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_member_card_account_ex": [ + { + "column": "member_card_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenantavatar", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "effect_site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "able_cross_site", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "card_physics_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "card_no", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "bind_password", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "use_scene", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "denomination", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "disable_start_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "disable_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_allow_give", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_allow_order_deduct", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sort", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "table_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "goods_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "assistant_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "assistant_reward_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "table_service_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "goods_service_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "assistant_service_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "coupon_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "table_discount_sub_switch", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "goods_discount_sub_switch", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "assistant_discount_sub_switch", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "assistant_reward_discount_sub_switch", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "goods_discount_range_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "table_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "goods_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "assistant_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "table_service_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "goods_service_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "assistant_service_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "assistant_reward_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "coupon_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "cardsettlededuct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "tablecarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "tableservicecarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goodscardeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goodsservicecarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistantcarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistantservicecarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistantrewardcarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "couponcarddeduct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "deliveryfeededuct", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "tableareaid", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "goodscategoryid", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "pdassisnatlevel", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "cxassisnatlevel", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "able_share_member_discount", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "electricity_deduct_radio", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "electricity_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "electricity_card_deduct", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "recharge_freeze_balance", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dim_member_ex": [ + { + "column": "member_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "referrer_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "point", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "register_site_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "growth_value", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "user_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "person_tenant_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "person_tenant_org_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "register_source", + "type": "text", + "nullable": "YES", + "max_length": null + } + ], + "dim_site": [ + { + "column": "site_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "shop_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "site_label", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "full_address", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "address", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "longitude", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "latitude", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "tenant_site_region_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "business_tel", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "site_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "shop_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_site_ex": [ + { + "column": "site_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "avatar", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "address", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "longitude", + "type": "numeric", + "nullable": "YES", + "max_length": 9 + }, + { + "column": "latitude", + "type": "numeric", + "nullable": "YES", + "max_length": 9 + }, + { + "column": "tenant_site_region_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "auto_light", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "light_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "light_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "light_token", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "site_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "site_label", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "attendance_enabled", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "attendance_distance", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "customer_service_qrcode", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "customer_service_wechat", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "fixed_pay_qrcode", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "prod_env", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "shop_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "update_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_store_goods": [ + { + "column": "site_goods_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_goods_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "goods_category_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_second_category_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "category_level1_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "category_level2_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "batch_stock_qty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sale_qty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "total_sales_qty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sale_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "created_at", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "updated_at", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "avg_monthly_sales", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_state", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "enable_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "send_state", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "commodity_code", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "not_sale", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_store_goods_ex": [ + { + "column": "site_goods_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "unit", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "goods_barcode", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "goods_cover_url", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "pinyin_initial", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "stock_qty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "stock_secondary_qty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "safety_stock_qty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "cost_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "cost_price_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "provisional_total_cost", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "total_purchase_cost", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "min_discount_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "is_discountable", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "days_on_shelf", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "audit_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sale_channel", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_warehousing", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "freeze_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "forbid_sell_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "able_site_transfer", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "custom_label_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "option_required", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "remark", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "sort_order", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_table": [ + { + "column": "table_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "site_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_area_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "order_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dim_table_ex": [ + { + "column": "table_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "show_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_online_reservation", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "table_cloth_use_time", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "table_cloth_use_cycle", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "table_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_tenant_goods": [ + { + "column": "tenant_goods_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "supplier_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "category_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_category_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_second_category_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "goods_number", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "unit", + "type": "character varying", + "nullable": "YES", + "max_length": 16 + }, + { + "column": "market_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_state", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "update_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "not_sale", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dim_tenant_goods_ex": [ + { + "column": "tenant_goods_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "remark_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "pinyin_initial", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "goods_cover", + "type": "character varying", + "nullable": "YES", + "max_length": 512 + }, + { + "column": "goods_bar_code", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "commodity_code", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "commodity_code_list", + "type": "character varying", + "nullable": "YES", + "max_length": 256 + }, + { + "column": "min_discount_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "cost_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "cost_price_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "able_discount", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sale_channel", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_warehousing", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_in_site", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "able_site_transfer", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "common_sale_royalty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "point_sale_royalty", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "out_goods_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "scd2_start_time", + "type": "timestamp with time zone", + "nullable": "NO", + "max_length": null + }, + { + "column": "scd2_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "scd2_is_current", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "scd2_version", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ], + "dwd_assistant_service_log": [ + { + "column": "assistant_service_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_pay_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_assistant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_assistant_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "system_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_no", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "nickname", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_assistant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_team_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "person_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_level", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "level_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "skill_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "skill_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_unit_price", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "ledger_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "projected_income", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "coupon_deduct_money", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "income_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "real_use_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "add_clock", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "start_use_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "last_use_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "real_service_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dwd_assistant_service_log_ex": [ + { + "column": "assistant_service_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "table_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "ledger_group_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "ledger_count", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "member_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "manual_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "service_money", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "returns_clock", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_start_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "ledger_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "ledger_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_confirm", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_single_order", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_not_responding", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_trash", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "trash_applicant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "trash_applicant_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "trash_reason", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "salesman_user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "skill_grade", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "service_grade", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "composite_grade", + "type": "numeric", + "nullable": "YES", + "max_length": 5 + }, + { + "column": "sum_grade", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "get_grade_times", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "grade_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "composite_grade_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "assistant_team_name", + "type": "text", + "nullable": "YES", + "max_length": null + } + ], + "dwd_assistant_trash_event": [ + { + "column": "assistant_trash_event_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "assistant_no", + "type": "character varying", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "assistant_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "charge_minutes_raw", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "abolish_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "trash_reason", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dwd_assistant_trash_event_ex": [ + { + "column": "assistant_trash_event_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "table_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_area_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + } + ], + "dwd_groupbuy_redemption": [ + { + "column": "redemption_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_charge_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_coupon_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "coupon_origin_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "promotion_activity_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "promotion_coupon_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_coupon_channel", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_unit_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "ledger_count", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "coupon_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "promotion_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "coupon_code", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "is_single_order", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_discount_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "coupon_sale_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dwd_groupbuy_redemption_ex": [ + { + "column": "redemption_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_area_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_pay_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_option_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "table_service_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_service_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "reward_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "recharge_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "offer_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "operator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_role_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_group_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "table_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "table_service_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "good_service_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_service_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "recharge_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dwd_member_balance_change": [ + { + "column": "balance_change_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "register_site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "system_member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_member_card_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "card_type_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "card_type_name", + "type": "character varying", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "member_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_mobile", + "type": "character varying", + "nullable": "YES", + "max_length": 20 + }, + { + "column": "balance_before", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "change_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "balance_after", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "from_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "payment_method", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "change_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "remark", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "principal_before", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "principal_after", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dwd_member_balance_change_ex": [ + { + "column": "balance_change_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "pay_site_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "register_site_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "refund_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "operator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "principal_data", + "type": "text", + "nullable": "YES", + "max_length": null + } + ], + "dwd_payment": [ + { + "column": "payment_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "relate_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "relate_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "pay_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "pay_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "payment_method", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "online_pay_channel", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "pay_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "pay_date", + "type": "date", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dwd_platform_coupon_redemption": [ + { + "column": "platform_coupon_redemption_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "coupon_code", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "coupon_channel", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "coupon_name", + "type": "character varying", + "nullable": "YES", + "max_length": 200 + }, + { + "column": "sale_price", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "coupon_money", + "type": "numeric", + "nullable": "YES", + "max_length": 10 + }, + { + "column": "coupon_free_time", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "channel_deal_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "deal_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "group_package_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_order_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "certificate_id", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "verify_id", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "use_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "consume_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + } + ], + "dwd_platform_coupon_redemption_ex": [ + { + "column": "platform_coupon_redemption_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "coupon_cover", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "coupon_remark", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "groupon_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "operator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 50 + } + ], + "dwd_recharge_order": [ + { + "column": "recharge_order_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_name_snapshot", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_phone_snapshot", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_member_card_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_card_type_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "settle_relate_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "settle_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "settle_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_first", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "pay_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "refund_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "cash_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "payment_method", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "pay_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + } + ], + "dwd_recharge_order_ex": [ + { + "column": "recharge_order_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "site_name_snapshot", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "settle_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_bind_member", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_activity", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_use_coupon", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_use_discount", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "can_be_revoked", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "online_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "balance_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "coupon_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "recharge_card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "gift_card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "prepay_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "consume_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_goods_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "table_charge_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "service_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "activity_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "all_coupon_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_pd_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_cx_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_manual_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "coupon_sale_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "member_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_discount_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_discount_cost", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "adjust_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "rounding_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "operator_name_snapshot", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "salesman_user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "order_remark", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "table_id", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "serial_number", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "revoke_order_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "revoke_order_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "revoke_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + } + ], + "dwd_refund": [ + { + "column": "refund_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "relate_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "relate_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "pay_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "channel_fee", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "pay_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "payment_method", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_card_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dwd_refund_ex": [ + { + "column": "refund_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "pay_sn", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "refund_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "round_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "balance_frozen_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "card_frozen_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "pay_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "action_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_revoke", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "check_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "online_pay_channel", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "online_pay_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "pay_terminal", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "pay_config_id", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "cashier_point_id", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "channel_payer_id", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "channel_pay_no", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + } + ], + "dwd_settlement_head": [ + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "settle_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "pay_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "settle_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "revoke_order_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "member_phone", + "type": "character varying", + "nullable": "YES", + "max_length": 50 + }, + { + "column": "member_card_account_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_card_type_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "is_bind_member", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "member_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "consume_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "table_charge_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_goods_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_pd_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_cx_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "adjust_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "pay_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "balance_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "recharge_card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "gift_card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "coupon_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "rounding_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "electricity_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_electricity_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "electricity_adjust_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "pl_coupon_sale_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "mervou_sales_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dwd_settlement_head_ex": [ + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "serial_number", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "settle_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "can_be_revoked", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "revoke_order_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "revoke_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_first_order", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "service_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "cash_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "online_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "refund_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "prepay_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "payment_method", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "coupon_sale_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "all_coupon_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "goods_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_promotion_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "activity_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "assistant_manual_discount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_discount_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_discount_cost", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "is_use_coupon", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_use_discount", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "is_activity", + "type": "boolean", + "nullable": "YES", + "max_length": null + }, + { + "column": "operator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "salesman_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "order_remark", + "type": "character varying", + "nullable": "YES", + "max_length": 255 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "settle_list", + "type": "jsonb", + "nullable": "YES", + "max_length": null + } + ], + "dwd_store_goods_sale": [ + { + "column": "store_goods_sale_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_pay_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_goods_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_goods_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_goods_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_goods_category_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_goods_business_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_name", + "type": "character varying", + "nullable": "YES", + "max_length": 200 + }, + { + "column": "ledger_group_name", + "type": "character varying", + "nullable": "YES", + "max_length": 100 + }, + { + "column": "ledger_unit_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "ledger_count", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "discount_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_goods_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "cost_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "ledger_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "coupon_share_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dwd_store_goods_sale_ex": [ + { + "column": "store_goods_sale_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "legacy_order_goods_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "legacy_site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "goods_remark", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "option_value_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "operator_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "open_salesman_flag", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "salesman_user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "salesman_role_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "discount_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "returns_number", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "coupon_deduct_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "member_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_discount_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "point_discount_money_cost", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "package_coupon_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_coupon_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_coupon_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "option_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "option_member_discount_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "option_coupon_deduct_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "push_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "is_single_order", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "sales_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + } + ], + "dwd_table_fee_adjust": [ + { + "column": "table_fee_adjust_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "table_area_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "ledger_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "adjust_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "table_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "table_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "charge_free", + "type": "boolean", + "nullable": "YES", + "max_length": null + } + ], + "dwd_table_fee_adjust_ex": [ + { + "column": "table_fee_adjust_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "adjust_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_count", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_name", + "type": "character varying", + "nullable": "YES", + "max_length": 128 + }, + { + "column": "applicant_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "operator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "applicant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "area_type_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_area_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "site_name", + "type": "text", + "nullable": "YES", + "max_length": null + }, + { + "column": "tenant_name", + "type": "text", + "nullable": "YES", + "max_length": null + } + ], + "dwd_table_fee_log": [ + { + "column": "table_fee_log_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "order_trade_no", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_settle_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_pay_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "site_table_area_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "tenant_table_area_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "member_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "ledger_unit_price", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "ledger_count", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "ledger_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_table_charge_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "coupon_promotion_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "member_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "adjust_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_table_use_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "add_clock_seconds", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "start_use_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "ledger_end_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "create_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "ledger_status", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_single_order", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "is_delete", + "type": "integer", + "nullable": "YES", + "max_length": 32 + }, + { + "column": "activity_discount_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "real_service_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + } + ], + "dwd_table_fee_log_ex": [ + { + "column": "table_fee_log_id", + "type": "bigint", + "nullable": "NO", + "max_length": 64 + }, + { + "column": "operator_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_name", + "type": "character varying", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "used_card_amount", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "service_money", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "mgmt_fee", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "fee_total", + "type": "numeric", + "nullable": "YES", + "max_length": 18 + }, + { + "column": "ledger_start_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "last_use_time", + "type": "timestamp with time zone", + "nullable": "YES", + "max_length": null + }, + { + "column": "operator_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_user_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "salesman_org_id", + "type": "bigint", + "nullable": "YES", + "max_length": 64 + }, + { + "column": "order_consumption_type", + "type": "integer", + "nullable": "YES", + "max_length": 32 + } + ] +} \ No newline at end of file diff --git a/tmp/field_coverage_report.json b/tmp/field_coverage_report.json new file mode 100644 index 0000000..a8678f6 --- /dev/null +++ b/tmp/field_coverage_report.json @@ -0,0 +1,38 @@ +{ + "generated_at": "2026-02-02T19:14:29.766314", + "ods_coverage": [ + { + "table": "billiards_ods.table_fee_transactions", + "column": "activity_discount_amount", + "total": 28162, + "non_null": 33, + "coverage": 0.1171791776152262, + "zero_count": 33 + }, + { + "table": "billiards_ods.table_fee_transactions", + "column": "real_service_money", + "total": 28162, + "non_null": 33, + "coverage": 0.1171791776152262, + "zero_count": 33 + }, + { + "table": "billiards_ods.table_fee_transactions", + "column": "order_consumption_type", + "total": 28162, + "non_null": 33, + "coverage": 0.1171791776152262, + "zero_count": 0 + }, + { + "table": "billiards_ods.assistant_service_records", + "column": "real_service_money", + "total": 10093, + "non_null": 10, + "coverage": 0.09907856930545923, + "zero_count": 10 + } + ], + "dwd_coverage": [] +} \ No newline at end of file diff --git a/tmp/fix_bd_manual.py b/tmp/fix_bd_manual.py new file mode 100644 index 0000000..7e9c6bc --- /dev/null +++ b/tmp/fix_bd_manual.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +"""自动修复 bd_manual 文档中的类型不匹配问题""" +import json +import re +from pathlib import Path + +def fix_type_in_doc(doc_path, type_mismatches): + """修复文档中的类型不匹配""" + if not Path(doc_path).exists(): + print(f" SKIP: {doc_path} not found") + return False + + content = Path(doc_path).read_text(encoding='utf-8') + original = content + + for m in type_mismatches: + col_name = m['column'] + old_type = m['doc_type'] + new_type = m['db_type'] + + # 匹配字段行并替换类型 + # 格式: | 序号 | 字段名 | 类型 | 可空 | ... + pattern = rf'(\|\s*\d+\s*\|\s*{col_name}\s*\|\s*){re.escape(old_type)}(\s*\|)' + replacement = rf'\g<1>{new_type}\g<2>' + content, count = re.subn(pattern, replacement, content) + + if count > 0: + print(f" Fixed: {col_name}: {old_type} -> {new_type}") + else: + # 尝试更宽松的匹配 + pattern2 = rf'(\|\s*{col_name}\s*\|\s*){re.escape(old_type)}(\s*\|)' + content, count = re.subn(pattern2, replacement.replace(r'\g<1>', r'\1').replace(r'\g<2>', r'\2'), content) + if count > 0: + print(f" Fixed (alt): {col_name}: {old_type} -> {new_type}") + else: + print(f" WARN: Could not fix {col_name}") + + if content != original: + Path(doc_path).write_text(content, encoding='utf-8') + return True + return False + +def add_missing_field(doc_path, table_name, field_name, db_schema): + """向文档中添加缺失的字段""" + if not Path(doc_path).exists(): + return False + + # 从 db_schema 获取字段信息 + field_info = None + for col in db_schema.get(table_name, []): + if col['column'] == field_name: + field_info = col + break + + if not field_info: + print(f" WARN: Could not find {field_name} in db_schema") + return False + + content = Path(doc_path).read_text(encoding='utf-8') + + # 找到字段表格的最后一行，在其后添加新字段 + # 格式: | 序号 | 字段名 | 类型 | 可空 | 主键 | 说明 | + lines = content.split('\n') + insert_idx = None + last_seq = 0 + + for i, line in enumerate(lines): + # 匹配字段行 + match = re.match(r'\|\s*(\d+)\s*\|\s*(\w+)\s*\|', line) + if match: + seq = int(match.group(1)) + if seq > last_seq: + last_seq = seq + insert_idx = i + + if insert_idx is not None: + new_seq = last_seq + 1 + nullable = 'YES' if field_info['nullable'] == 'YES' else 'NO' + new_line = f"| {new_seq} | {field_name} | {field_info['type']} | {nullable} | | 调整时间 |" + lines.insert(insert_idx + 1, new_line) + + Path(doc_path).write_text('\n'.join(lines), encoding='utf-8') + print(f" Added: {field_name} (type: {field_info['type']})") + return True + + return False + +def main(): + # 加载差异数据 + with open('tmp/bd_manual_diff.json', 'r', encoding='utf-8') as f: + diffs = json.load(f) + + # 加载数据库 schema (需要重新获取带详细信息的) + import psycopg2 + DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + + conn = psycopg2.connect(DSN) + cur = conn.cursor() + cur.execute(""" + SELECT table_name, column_name, data_type, is_nullable, + COALESCE(character_maximum_length, numeric_precision) as max_length, + numeric_scale + FROM information_schema.columns + WHERE table_schema = 'billiards_dwd' + ORDER BY table_name, ordinal_position + """) + + db_schema = {} + TYPE_MAP = { + 'bigint': 'BIGINT', + 'integer': 'INTEGER', + 'smallint': 'SMALLINT', + 'numeric': 'NUMERIC', + 'text': 'TEXT', + 'character varying': 'VARCHAR', + 'boolean': 'BOOLEAN', + 'timestamp with time zone': 'TIMESTAMPTZ', + 'timestamp without time zone': 'TIMESTAMP', + 'date': 'DATE', + 'jsonb': 'JSONB', + 'json': 'JSON', + } + + for row in cur.fetchall(): + table_name, col_name, data_type, nullable, max_len, scale = row + if table_name not in db_schema: + db_schema[table_name] = [] + + type_str = TYPE_MAP.get(data_type, data_type.upper()) + if data_type == 'numeric' and max_len and scale is not None: + type_str = f'NUMERIC({max_len},{scale})' + elif data_type == 'character varying' and max_len: + type_str = f'VARCHAR({max_len})' + + db_schema[table_name].append({ + 'column': col_name, + 'type': type_str, + 'nullable': nullable, + }) + + cur.close() + conn.close() + + print("=" * 80) + print("Fixing BD Manual Documents") + print("=" * 80) + + fixed_count = 0 + + for diff in diffs: + table = diff['table'] + doc_path = diff.get('doc_path', '') + + if not doc_path: + continue + + has_changes = False + + # 修复类型不匹配 + if diff.get('type_mismatches'): + print(f"\n### {table} (type fixes) ###") + if fix_type_in_doc(doc_path, diff['type_mismatches']): + has_changes = True + + # 添加缺失字段 + if diff.get('missing_in_doc'): + print(f"\n### {table} (missing fields) ###") + for field in diff['missing_in_doc']: + if add_missing_field(doc_path, table, field, db_schema): + has_changes = True + + if has_changes: + fixed_count += 1 + + print("\n" + "=" * 80) + print(f"Fixed {fixed_count} documents") + print("=" * 80) + +if __name__ == '__main__': + main() diff --git a/tmp/fix_not_sale_type.py b/tmp/fix_not_sale_type.py new file mode 100644 index 0000000..bcb5f14 --- /dev/null +++ b/tmp/fix_not_sale_type.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +"""修复 not_sale 字段类型""" +import psycopg2 + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +def fix_column_type(): + conn = psycopg2.connect(DSN) + conn.autocommit = True + cur = conn.cursor() + + # 修复 ODS store_goods_master 表 + tables_to_fix = [ + ('billiards_ods', 'store_goods_master', 'not_sale', 'INTEGER'), + ('billiards_ods', 'tenant_goods_master', 'not_sale', 'INTEGER'), + ('billiards_dwd', 'dim_store_goods', 'not_sale', 'INTEGER'), + ('billiards_dwd', 'dim_tenant_goods', 'not_sale', 'INTEGER'), + ] + + for schema, table, column, new_type in tables_to_fix: + try: + # 检查表和列是否存在 + cur.execute(""" + SELECT data_type FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = %s + """, (schema, table, column)) + result = cur.fetchone() + + if result: + current_type = result[0] + print(f"{schema}.{table}.{column}: current type = {current_type}") + + if current_type == 'boolean': + # 先删除列，再重新添加为 INTEGER + sql_drop = f'ALTER TABLE {schema}.{table} DROP COLUMN "{column}"' + sql_add = f'ALTER TABLE {schema}.{table} ADD COLUMN "{column}" {new_type}' + + cur.execute(sql_drop) + print(f" Dropped column") + cur.execute(sql_add) + print(f" Re-added as {new_type}") + else: + print(f" Already {current_type}, skipping") + else: + print(f"{schema}.{table}.{column}: column not found") + + except Exception as e: + print(f"Error fixing {schema}.{table}.{column}: {e}") + + cur.close() + conn.close() + print("\nDone!") + +if __name__ == '__main__': + fix_column_type() diff --git a/tmp/fix_remaining_issues.py b/tmp/fix_remaining_issues.py new file mode 100644 index 0000000..b8d0025 --- /dev/null +++ b/tmp/fix_remaining_issues.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +import os +import sys +from pathlib import Path + +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + +dsn = os.getenv("PG_DSN") +db = DatabaseConnection(dsn) + +print("=== Fixing remaining issues ===") + +# 1. Fix principal_change_amount type mismatch +db.execute(""" + UPDATE billiards_dwd.dwd_member_balance_change d + SET principal_change_amount = o.principal_data::numeric + FROM billiards_ods.member_balance_changes o + WHERE d.balance_change_id = o.id + AND d.principal_change_amount IS NULL + AND o.principal_data IS NOT NULL +""") +db.commit() +print("principal_change_amount: fixed") + +# 2. Add missing DWD columns for dwd_recharge_order +missing_cols = [ + ("pl_coupon_sale_amount", "NUMERIC(18,2)"), + ("mervou_sales_amount", "NUMERIC(18,2)"), + ("electricity_money", "NUMERIC(18,2)"), + ("real_electricity_money", "NUMERIC(18,2)"), + ("electricity_adjust_money", "NUMERIC(18,2)"), +] +for col, dtype in missing_cols: + try: + db.execute(f'ALTER TABLE billiards_dwd.dwd_recharge_order ADD COLUMN IF NOT EXISTS "{col}" {dtype}') + db.commit() + print(f"dwd_recharge_order.{col}: column added") + except Exception as e: + db.rollback() + print(f"dwd_recharge_order.{col}: {str(e)[:50]}") + +# 3. Backfill dwd_recharge_order from ODS +db.execute(""" + UPDATE billiards_dwd.dwd_recharge_order d + SET pl_coupon_sale_amount = o.plcouponsaleamount, + mervou_sales_amount = o.mervousalesamount, + electricity_money = o.electricitymoney, + real_electricity_money = o.realelectricitymoney, + electricity_adjust_money = o.electricityadjustmoney + FROM billiards_ods.recharge_settlements o + WHERE d.recharge_order_id = o.id +""") +db.commit() +print("dwd_recharge_order: backfilled") + +db.close() +print("Done") diff --git a/tmp/full_reload_validation.py b/tmp/full_reload_validation.py new file mode 100644 index 0000000..57e3d4b --- /dev/null +++ b/tmp/full_reload_validation.py @@ -0,0 +1,237 @@ +# -*- coding: utf-8 -*- +""" +全量数据回写验证脚本 +从 2025-07-01 到现在，重新获取 API 数据并入库 +""" +import json +import os +import sys +from datetime import datetime, timedelta +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + + +def check_ods_field_coverage(db: DatabaseConnection): + """检查 ODS 表中新增字段的数据覆盖情况""" + + # 需要检查的新增字段 + fields_to_check = [ + ("billiards_ods.table_fee_transactions", ["activity_discount_amount", "real_service_money", "order_consumption_type"]), + ("billiards_ods.assistant_service_records", ["real_service_money", "assistantteamname"]), + ("billiards_ods.assistant_cancellation_records", ["tenant_id"]), + ("billiards_ods.store_goods_sales_records", ["coupon_share_money"]), + ("billiards_ods.payment_transactions", ["tenant_id"]), + ("billiards_ods.member_profiles", ["pay_money_sum", "person_tenant_org_id", "recharge_money_sum", "register_source"]), + ("billiards_ods.member_stored_value_cards", ["principal_balance", "member_grade", "rechargefreezebalance"]), + ("billiards_ods.member_balance_changes", ["principal_after", "principal_before", "principal_data"]), + ("billiards_ods.settlement_records", ["tenant_id"]), + ("billiards_ods.recharge_settlements", ["tenant_id"]), + ("billiards_ods.group_buy_packages", ["sort", "is_first_limit", "tenantcouponsaleorderitemid"]), + ("billiards_ods.group_buy_redemption_records", ["coupon_sale_id", "member_discount_money"]), + ("billiards_ods.site_tables_master", ["order_id"]), + ("billiards_ods.store_goods_master", ["commodity_code", "not_sale"]), + ("billiards_ods.table_fee_discount_records", ["table_name", "table_price", "charge_free"]), + ("billiards_ods.tenant_goods_master", ["not_sale"]), + ] + + print("\n" + "=" * 80) + print("ODS 新增字段数据覆盖检查") + print("=" * 80) + + results = [] + + for table, columns in fields_to_check: + print(f"\n检查表: {table}") + + # 获取总记录数 + try: + total_rows = db.query(f"SELECT COUNT(*) as cnt FROM {table}")[0]["cnt"] + except Exception as e: + print(f" [错误] 无法获取记录数: {e}") + continue + + for col in columns: + try: + # 检查列是否存在 + schema, name = table.split(".", 1) + col_check = db.query(""" + SELECT COUNT(*) as cnt FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = %s + """, (schema, name, col.lower())) + + if col_check[0]["cnt"] == 0: + print(f" 列 {col}: [不存在]") + continue + + # 统计非空值数量 + non_null_rows = db.query(f'SELECT COUNT(*) as cnt FROM {table} WHERE "{col}" IS NOT NULL')[0]["cnt"] + zero_rows = db.query(f'SELECT COUNT(*) as cnt FROM {table} WHERE "{col}" = 0')[0]["cnt"] + + coverage = (non_null_rows / total_rows * 100) if total_rows > 0 else 0 + + print(f" 列 {col}:") + print(f" - 总记录: {total_rows}, 非空: {non_null_rows} ({coverage:.1f}%), 值为0: {zero_rows}") + + results.append({ + "table": table, + "column": col, + "total": total_rows, + "non_null": non_null_rows, + "coverage": coverage, + "zero_count": zero_rows, + }) + + except Exception as e: + print(f" 列 {col}: [错误] {e}") + + return results + + +def check_dwd_field_coverage(db: DatabaseConnection): + """检查 DWD 表中新增字段的数据覆盖情况""" + + # 需要检查的新增字段 + fields_to_check = [ + ("billiards_dwd.dwd_table_fee_log", ["activity_discount_amount", "real_service_money"]), + ("billiards_dwd.dwd_assistant_service_log", ["real_service_money"]), + ("billiards_dwd.dwd_assistant_trash_event", ["tenant_id"]), + ("billiards_dwd.dwd_store_goods_sale", ["coupon_share_money"]), + ("billiards_dwd.dwd_payment", ["tenant_id"]), + ("billiards_dwd.dim_member", ["pay_money_sum", "recharge_money_sum"]), + ("billiards_dwd.dim_member_ex", ["person_tenant_org_id", "register_source"]), + ("billiards_dwd.dim_member_card_account", ["principal_balance", "member_grade"]), + ("billiards_dwd.dwd_member_balance_change", ["principal_after", "principal_before", "principal_change_amount"]), + ("billiards_dwd.dwd_settlement_head", ["tenant_id"]), + ("billiards_dwd.dwd_recharge_order", ["tenant_id"]), + ("billiards_dwd.dim_groupbuy_package", ["sort", "is_first_limit"]), + ("billiards_dwd.dwd_groupbuy_redemption", ["coupon_sale_id", "member_discount_money"]), + ("billiards_dwd.dim_table", ["order_id"]), + ("billiards_dwd.dim_store_goods", ["commodity_code", "not_sale"]), + ("billiards_dwd.dwd_table_fee_adjust", ["table_name", "table_price", "charge_free"]), + ("billiards_dwd.dim_tenant_goods", ["not_sale"]), + ] + + print("\n" + "=" * 80) + print("DWD 新增字段数据覆盖检查") + print("=" * 80) + + results = [] + + for table, columns in fields_to_check: + print(f"\n检查表: {table}") + + # 获取总记录数 + try: + total_rows = db.query(f"SELECT COUNT(*) as cnt FROM {table}")[0]["cnt"] + except Exception as e: + print(f" [错误] 无法获取记录数: {e}") + continue + + for col in columns: + try: + # 检查列是否存在 + schema, name = table.split(".", 1) + col_check = db.query(""" + SELECT COUNT(*) as cnt FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s AND column_name = %s + """, (schema, name, col.lower())) + + if col_check[0]["cnt"] == 0: + print(f" 列 {col}: [不存在]") + continue + + # 统计非空值数量 + non_null_rows = db.query(f'SELECT COUNT(*) as cnt FROM {table} WHERE "{col}" IS NOT NULL')[0]["cnt"] + + coverage = (non_null_rows / total_rows * 100) if total_rows > 0 else 0 + + print(f" 列 {col}: 总记录: {total_rows}, 非空: {non_null_rows} ({coverage:.1f}%)") + + results.append({ + "table": table, + "column": col, + "total": total_rows, + "non_null": non_null_rows, + "coverage": coverage, + }) + + except Exception as e: + print(f" 列 {col}: [错误] {e}") + + return results + + +def main(): + print("=" * 80) + print("全量数据回写验证") + print("时间:", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + print("=" * 80) + + # 连接数据库 + dsn = os.getenv("PG_DSN") + if not dsn: + print("[错误] 未找到 PG_DSN 环境变量") + return False + + db = DatabaseConnection(dsn) + + # 检查 ODS 字段覆盖 + ods_results = check_ods_field_coverage(db) + + # 检查 DWD 字段覆盖 + dwd_results = check_dwd_field_coverage(db) + + db.close() + + # 生成汇总 + print("\n" + "=" * 80) + print("汇总") + print("=" * 80) + + print("\nODS 新增字段覆盖率统计:") + for r in ods_results: + if r["coverage"] < 50: + status = "[需关注]" + elif r["coverage"] < 80: + status = "[一般]" + else: + status = "[良好]" + print(f" {r['table']}.{r['column']}: {r['coverage']:.1f}% {status}") + + print("\nDWD 新增字段覆盖率统计:") + for r in dwd_results: + if r["coverage"] < 50: + status = "[需关注]" + elif r["coverage"] < 80: + status = "[一般]" + else: + status = "[良好]" + print(f" {r['table']}.{r['column']}: {r['coverage']:.1f}% {status}") + + # 保存报告 + report = { + "generated_at": datetime.now().isoformat(), + "ods_coverage": ods_results, + "dwd_coverage": dwd_results, + } + + report_file = Path(__file__).parent / "field_coverage_report.json" + with open(report_file, "w", encoding="utf-8") as f: + json.dump(report, f, ensure_ascii=False, indent=2) + + print(f"\n报告已保存到: {report_file}") + + return True + + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) diff --git a/tmp/get_dwd_schema.py b/tmp/get_dwd_schema.py new file mode 100644 index 0000000..51a8139 --- /dev/null +++ b/tmp/get_dwd_schema.py @@ -0,0 +1,48 @@ +# -*- coding: utf-8 -*- +"""获取 DWD 所有表的结构""" +import psycopg2 +import json + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +def get_all_tables(conn, schema='billiards_dwd'): + """获取所有表及其列""" + cur = conn.cursor() + cur.execute(""" + SELECT table_name, column_name, data_type, is_nullable, + COALESCE(character_maximum_length, numeric_precision) as max_length + FROM information_schema.columns + WHERE table_schema = %s + ORDER BY table_name, ordinal_position + """, (schema,)) + + tables = {} + for row in cur.fetchall(): + table_name, col_name, data_type, nullable, max_len = row + if table_name not in tables: + tables[table_name] = [] + tables[table_name].append({ + 'column': col_name, + 'type': data_type, + 'nullable': nullable, + 'max_length': max_len + }) + + cur.close() + return tables + +def main(): + conn = psycopg2.connect(DSN) + tables = get_all_tables(conn) + conn.close() + + # 保存到 JSON 文件 + with open('tmp/dwd_schema.json', 'w', encoding='utf-8') as f: + json.dump(tables, f, ensure_ascii=False, indent=2) + + print(f"Found {len(tables)} tables") + for table_name, columns in sorted(tables.items()): + print(f" {table_name}: {len(columns)} columns") + +if __name__ == '__main__': + main() diff --git a/tmp/list_dwd_tables.py b/tmp/list_dwd_tables.py new file mode 100644 index 0000000..f23ff6a --- /dev/null +++ b/tmp/list_dwd_tables.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +import psycopg2 +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' +conn = psycopg2.connect(DSN) +cur = conn.cursor() +cur.execute("SELECT table_name FROM information_schema.tables WHERE table_schema = 'billiards_dwd' ORDER BY table_name") +for r in cur.fetchall(): + print(r[0]) +conn.close() diff --git a/tmp/query_schema.py b/tmp/query_schema.py new file mode 100644 index 0000000..8fb9342 --- /dev/null +++ b/tmp/query_schema.py @@ -0,0 +1,19 @@ +import psycopg2 +from collections import defaultdict + +conn = psycopg2.connect('postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test') +cur = conn.cursor() +cur.execute("SELECT table_name, column_name, data_type, ordinal_position FROM information_schema.columns WHERE table_schema = 'billiards_dwd' ORDER BY table_name, ordinal_position") +results = cur.fetchall() +tables = defaultdict(list) +for row in results: + tables[row[0]].append((row[1], row[2], row[3])) + +for table in sorted(tables.keys()): + print(f'\n琛ㄥ悕: {table}') + cols = tables[table] + for col, dtype, pos in cols: + print(f' {pos}. {col} ({dtype})') + +cur.close() +conn.close() diff --git a/tmp/schema_output.txt b/tmp/schema_output.txt new file mode 100644 index 0000000..cf4b29c --- /dev/null +++ b/tmp/schema_output.txt @@ -0,0 +1,994 @@ + +?? assistant_accounts_master +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. assistant_no (text) + 5. nickname (text) + 6. real_name (text) + 7. mobile (text) + 8. team_id (bigint) + 9. team_name (text) + 10. user_id (bigint) + 11. level (text) + 12. assistant_status (integer) + 13. work_status (integer) + 14. leave_status (integer) + 15. entry_time (timestamp without time zone) + 16. resign_time (timestamp without time zone) + 17. start_time (timestamp without time zone) + 18. end_time (timestamp without time zone) + 19. create_time (timestamp without time zone) + 20. update_time (timestamp without time zone) + 21. order_trade_no (text) + 22. staff_id (bigint) + 23. staff_profile_id (bigint) + 24. system_role_id (bigint) + 25. avatar (text) + 26. birth_date (timestamp without time zone) + 27. gender (integer) + 28. height (numeric) + 29. weight (numeric) + 30. job_num (text) + 31. show_status (integer) + 32. show_sort (integer) + 33. sum_grade (numeric) + 34. assistant_grade (numeric) + 35. get_grade_times (integer) + 36. introduce (text) + 37. video_introduction_url (text) + 38. group_id (bigint) + 39. group_name (text) + 40. shop_name (text) + 41. charge_way (integer) + 42. entry_type (integer) + 43. allow_cx (integer) + 44. is_guaranteed (integer) + 45. salary_grant_enabled (integer) + 46. light_status (integer) + 47. online_status (integer) + 48. is_delete (integer) + 49. cx_unit_price (numeric) + 50. pd_unit_price (numeric) + 51. last_table_id (bigint) + 52. last_table_name (text) + 53. person_org_id (bigint) + 54. serial_number (bigint) + 55. is_team_leader (integer) + 56. criticism_status (integer) + 57. last_update_name (text) + 58. ding_talk_synced (integer) + 59. site_light_cfg_id (bigint) + 60. light_equipment_id (text) + 61. entry_sign_status (integer) + 62. resign_sign_status (integer) + 63. source_file (text) + 64. source_endpoint (text) + 65. fetched_at (timestamp with time zone) + 66. payload (jsonb) + 67. content_hash (text) + +?? assistant_cancellation_records +------------------------------------------------------------ + 1. id (bigint) + 2. siteid (bigint) + 3. siteprofile (jsonb) + 4. assistantname (text) + 5. assistantabolishamount (numeric) + 6. assistanton (integer) + 7. pdchargeminutes (integer) + 8. tableareaid (bigint) + 9. tablearea (text) + 10. tableid (bigint) + 11. tablename (text) + 12. trashreason (text) + 13. createtime (timestamp without time zone) + 14. source_file (text) + 15. source_endpoint (text) + 16. fetched_at (timestamp with time zone) + 17. payload (jsonb) + 18. content_hash (text) + +?? assistant_service_records +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. siteprofile (jsonb) + 5. site_table_id (bigint) + 6. order_settle_id (bigint) + 7. order_trade_no (text) + 8. order_pay_id (bigint) + 9. order_assistant_id (bigint) + 10. order_assistant_type (integer) + 11. assistantname (text) + 12. assistantno (text) + 13. assistant_level (text) + 14. levelname (text) + 15. site_assistant_id (bigint) + 16. skill_id (bigint) + 17. skillname (text) + 18. system_member_id (bigint) + 19. tablename (text) + 20. tenant_member_id (bigint) + 21. user_id (bigint) + 22. assistant_team_id (bigint) + 23. nickname (text) + 24. ledger_name (text) + 25. ledger_group_name (text) + 26. ledger_amount (numeric) + 27. ledger_count (numeric) + 28. ledger_unit_price (numeric) + 29. ledger_status (integer) + 30. ledger_start_time (timestamp without time zone) + 31. ledger_end_time (timestamp without time zone) + 32. manual_discount_amount (numeric) + 33. member_discount_amount (numeric) + 34. coupon_deduct_money (numeric) + 35. service_money (numeric) + 36. projected_income (numeric) + 37. real_use_seconds (integer) + 38. income_seconds (integer) + 39. start_use_time (timestamp without time zone) + 40. last_use_time (timestamp without time zone) + 41. create_time (timestamp without time zone) + 42. is_single_order (integer) + 43. is_delete (integer) + 44. is_trash (integer) + 45. trash_reason (text) + 46. trash_applicant_id (bigint) + 47. trash_applicant_name (text) + 48. operator_id (bigint) + 49. operator_name (text) + 50. salesman_name (text) + 51. salesman_org_id (bigint) + 52. salesman_user_id (bigint) + 53. person_org_id (bigint) + 54. add_clock (integer) + 55. returns_clock (integer) + 56. composite_grade (numeric) + 57. composite_grade_time (timestamp without time zone) + 58. skill_grade (numeric) + 59. service_grade (numeric) + 60. sum_grade (numeric) + 61. grade_status (integer) + 62. get_grade_times (integer) + 63. is_not_responding (integer) + 64. is_confirm (integer) + 65. payload (jsonb) + 66. source_file (text) + 67. source_endpoint (text) + 68. fetched_at (timestamp with time zone) + 69. content_hash (text) + +?? goods_stock_movements +------------------------------------------------------------ + 1. sitegoodsstockid (bigint) + 2. tenantid (bigint) + 3. siteid (bigint) + 4. sitegoodsid (bigint) + 5. goodsname (text) + 6. goodscategoryid (bigint) + 7. goodssecondcategoryid (bigint) + 8. unit (text) + 9. price (numeric) + 10. stocktype (integer) + 11. changenum (numeric) + 12. startnum (numeric) + 13. endnum (numeric) + 14. changenuma (numeric) + 15. startnuma (numeric) + 16. endnuma (numeric) + 17. remark (text) + 18. operatorname (text) + 19. createtime (timestamp without time zone) + 20. source_file (text) + 21. source_endpoint (text) + 22. fetched_at (timestamp with time zone) + 23. payload (jsonb) + 24. content_hash (text) + +?? goods_stock_summary +------------------------------------------------------------ + 1. sitegoodsid (bigint) + 2. goodsname (text) + 3. goodsunit (text) + 4. goodscategoryid (bigint) + 5. goodscategorysecondid (bigint) + 6. categoryname (text) + 7. rangestartstock (numeric) + 8. rangeendstock (numeric) + 9. rangein (numeric) + 10. rangeout (numeric) + 11. rangesale (numeric) + 12. rangesalemoney (numeric) + 13. rangeinventory (numeric) + 14. currentstock (numeric) + 15. source_file (text) + 16. source_endpoint (text) + 17. fetched_at (timestamp with time zone) + 18. payload (jsonb) + 19. content_hash (text) + +?? group_buy_packages +------------------------------------------------------------ + 1. id (bigint) + 2. package_id (bigint) + 3. package_name (text) + 4. selling_price (numeric) + 5. coupon_money (numeric) + 6. date_type (integer) + 7. date_info (text) + 8. start_time (timestamp without time zone) + 9. end_time (timestamp without time zone) + 10. start_clock (text) + 11. end_clock (text) + 12. add_start_clock (text) + 13. add_end_clock (text) + 14. duration (integer) + 15. usable_count (integer) + 16. usable_range (integer) + 17. table_area_id (bigint) + 18. table_area_name (text) + 19. table_area_id_list (jsonb) + 20. tenant_table_area_id (bigint) + 21. tenant_table_area_id_list (jsonb) + 22. site_id (bigint) + 23. site_name (text) + 24. tenant_id (bigint) + 25. card_type_ids (jsonb) + 26. group_type (integer) + 27. system_group_type (integer) + 28. type (integer) + 29. effective_status (integer) + 30. is_enabled (integer) + 31. is_delete (integer) + 32. max_selectable_categories (integer) + 33. area_tag_type (integer) + 34. creator_name (text) + 35. create_time (timestamp without time zone) + 36. source_file (text) + 37. source_endpoint (text) + 38. fetched_at (timestamp with time zone) + 39. payload (jsonb) + 40. content_hash (text) + +?? group_buy_redemption_records +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. sitename (text) + 5. table_id (bigint) + 6. tablename (text) + 7. tableareaname (text) + 8. tenant_table_area_id (bigint) + 9. order_trade_no (text) + 10. order_settle_id (bigint) + 11. order_pay_id (bigint) + 12. order_coupon_id (bigint) + 13. order_coupon_channel (integer) + 14. coupon_code (text) + 15. coupon_money (numeric) + 16. coupon_origin_id (bigint) + 17. ledger_name (text) + 18. ledger_group_name (text) + 19. ledger_amount (numeric) + 20. ledger_count (numeric) + 21. ledger_unit_price (numeric) + 22. ledger_status (integer) + 23. table_charge_seconds (integer) + 24. promotion_activity_id (bigint) + 25. promotion_coupon_id (bigint) + 26. promotion_seconds (integer) + 27. offer_type (integer) + 28. assistant_promotion_money (numeric) + 29. assistant_service_promotion_money (numeric) + 30. table_service_promotion_money (numeric) + 31. goods_promotion_money (numeric) + 32. recharge_promotion_money (numeric) + 33. reward_promotion_money (numeric) + 34. goodsoptionprice (numeric) + 35. salesman_name (text) + 36. sales_man_org_id (bigint) + 37. salesman_role_id (bigint) + 38. salesman_user_id (bigint) + 39. operator_id (bigint) + 40. operator_name (text) + 41. is_single_order (integer) + 42. is_delete (integer) + 43. create_time (timestamp without time zone) + 44. payload (jsonb) + 45. source_file (text) + 46. source_endpoint (text) + 47. fetched_at (timestamp with time zone) + 48. content_hash (text) + +?? member_balance_changes +------------------------------------------------------------ + 1. tenant_id (bigint) + 2. site_id (bigint) + 3. register_site_id (bigint) + 4. registersitename (text) + 5. paysitename (text) + 6. id (bigint) + 7. tenant_member_id (bigint) + 8. tenant_member_card_id (bigint) + 9. system_member_id (bigint) + 10. membername (text) + 11. membermobile (text) + 12. card_type_id (bigint) + 13. membercardtypename (text) + 14. account_data (numeric) + 15. before (numeric) + 16. after (numeric) + 17. refund_amount (numeric) + 18. from_type (integer) + 19. payment_method (integer) + 20. relate_id (bigint) + 21. remark (text) + 22. operator_id (bigint) + 23. operator_name (text) + 24. is_delete (integer) + 25. create_time (timestamp without time zone) + 26. source_file (text) + 27. source_endpoint (text) + 28. fetched_at (timestamp with time zone) + 29. payload (jsonb) + 30. content_hash (text) + +?? member_profiles +------------------------------------------------------------ + 1. tenant_id (bigint) + 2. register_site_id (bigint) + 3. site_name (text) + 4. id (bigint) + 5. system_member_id (bigint) + 6. member_card_grade_code (bigint) + 7. member_card_grade_name (text) + 8. mobile (text) + 9. nickname (text) + 10. point (numeric) + 11. growth_value (numeric) + 12. referrer_member_id (bigint) + 13. status (integer) + 14. user_status (integer) + 15. create_time (timestamp without time zone) + 16. source_file (text) + 17. source_endpoint (text) + 18. fetched_at (timestamp with time zone) + 19. payload (jsonb) + 20. content_hash (text) + +?? member_stored_value_cards +------------------------------------------------------------ + 1. tenant_id (bigint) + 2. tenant_member_id (bigint) + 3. system_member_id (bigint) + 4. register_site_id (bigint) + 5. site_name (text) + 6. id (bigint) + 7. member_card_grade_code (bigint) + 8. member_card_grade_code_name (text) + 9. member_card_type_name (text) + 10. member_name (text) + 11. member_mobile (text) + 12. card_type_id (bigint) + 13. card_no (text) + 14. card_physics_type (text) + 15. balance (numeric) + 16. denomination (numeric) + 17. table_discount (numeric) + 18. goods_discount (numeric) + 19. assistant_discount (numeric) + 20. assistant_reward_discount (numeric) + 21. table_service_discount (numeric) + 22. assistant_service_discount (numeric) + 23. coupon_discount (numeric) + 24. goods_service_discount (numeric) + 25. assistant_discount_sub_switch (integer) + 26. table_discount_sub_switch (integer) + 27. goods_discount_sub_switch (integer) + 28. assistant_reward_discount_sub_switch (integer) + 29. table_service_deduct_radio (numeric) + 30. assistant_service_deduct_radio (numeric) + 31. goods_service_deduct_radio (numeric) + 32. assistant_deduct_radio (numeric) + 33. table_deduct_radio (numeric) + 34. goods_deduct_radio (numeric) + 35. coupon_deduct_radio (numeric) + 36. assistant_reward_deduct_radio (numeric) + 37. tablecarddeduct (numeric) + 38. tableservicecarddeduct (numeric) + 39. goodscardeduct (numeric) + 40. goodsservicecarddeduct (numeric) + 41. assistantcarddeduct (numeric) + 42. assistantservicecarddeduct (numeric) + 43. assistantrewardcarddeduct (numeric) + 44. cardsettlededuct (numeric) + 45. couponcarddeduct (numeric) + 46. deliveryfeededuct (numeric) + 47. use_scene (integer) + 48. able_cross_site (integer) + 49. able_site_transfer (integer) + 50. is_allow_give (integer) + 51. is_allow_order_deduct (integer) + 52. is_delete (integer) + 53. bind_password (text) + 54. goods_discount_range_type (integer) + 55. goodscategoryid (bigint) + 56. tableareaid (bigint) + 57. effect_site_id (bigint) + 58. start_time (timestamp without time zone) + 59. end_time (timestamp without time zone) + 60. disable_start_time (timestamp without time zone) + 61. disable_end_time (timestamp without time zone) + 62. last_consume_time (timestamp without time zone) + 63. create_time (timestamp without time zone) + 64. status (integer) + 65. sort (integer) + 66. tenantavatar (text) + 67. tenantname (text) + 68. pdassisnatlevel (text) + 69. cxassisnatlevel (text) + 70. source_file (text) + 71. source_endpoint (text) + 72. fetched_at (timestamp with time zone) + 73. payload (jsonb) + 74. content_hash (text) + +?? payment_transactions +------------------------------------------------------------ + 1. id (bigint) + 2. site_id (bigint) + 3. siteprofile (jsonb) + 4. relate_type (integer) + 5. relate_id (bigint) + 6. pay_amount (numeric) + 7. pay_status (integer) + 8. pay_time (timestamp without time zone) + 9. create_time (timestamp without time zone) + 10. payment_method (integer) + 11. online_pay_channel (integer) + 12. source_file (text) + 13. source_endpoint (text) + 14. fetched_at (timestamp with time zone) + 15. payload (jsonb) + 16. content_hash (text) + +?? platform_coupon_redemption_records +------------------------------------------------------------ + 1. id (bigint) + 2. verify_id (bigint) + 3. certificate_id (text) + 4. coupon_code (text) + 5. coupon_name (text) + 6. coupon_channel (integer) + 7. groupon_type (integer) + 8. group_package_id (bigint) + 9. sale_price (numeric) + 10. coupon_money (numeric) + 11. coupon_free_time (numeric) + 12. coupon_cover (text) + 13. coupon_remark (text) + 14. use_status (integer) + 15. consume_time (timestamp without time zone) + 16. create_time (timestamp without time zone) + 17. deal_id (text) + 18. channel_deal_id (text) + 19. site_id (bigint) + 20. site_order_id (bigint) + 21. table_id (bigint) + 22. tenant_id (bigint) + 23. operator_id (bigint) + 24. operator_name (text) + 25. is_delete (integer) + 26. siteprofile (jsonb) + 27. source_file (text) + 28. source_endpoint (text) + 29. fetched_at (timestamp with time zone) + 30. payload (jsonb) + 31. content_hash (text) + +?? recharge_settlements +------------------------------------------------------------ + 1. id (bigint) + 2. tenantid (bigint) + 3. siteid (bigint) + 4. sitename (text) + 5. balanceamount (numeric) + 6. cardamount (numeric) + 7. cashamount (numeric) + 8. couponamount (numeric) + 9. createtime (timestamp with time zone) + 10. memberid (bigint) + 11. membername (text) + 12. tenantmembercardid (bigint) + 13. membercardtypename (text) + 14. memberphone (text) + 15. tableid (bigint) + 16. consumemoney (numeric) + 17. onlineamount (numeric) + 18. operatorid (bigint) + 19. operatorname (text) + 20. revokeorderid (bigint) + 21. revokeordername (text) + 22. revoketime (timestamp with time zone) + 23. payamount (numeric) + 24. pointamount (numeric) + 25. refundamount (numeric) + 26. settlename (text) + 27. settlerelateid (bigint) + 28. settlestatus (integer) + 29. settletype (integer) + 30. paytime (timestamp with time zone) + 31. roundingamount (numeric) + 32. paymentmethod (integer) + 33. adjustamount (numeric) + 34. assistantcxmoney (numeric) + 35. assistantpdmoney (numeric) + 36. couponsaleamount (numeric) + 37. memberdiscountamount (numeric) + 38. tablechargemoney (numeric) + 39. goodsmoney (numeric) + 40. realgoodsmoney (numeric) + 41. servicemoney (numeric) + 42. prepaymoney (numeric) + 43. salesmanname (text) + 44. orderremark (text) + 45. salesmanuserid (bigint) + 46. canberevoked (boolean) + 47. pointdiscountprice (numeric) + 48. pointdiscountcost (numeric) + 49. activitydiscount (numeric) + 50. serialnumber (bigint) + 51. assistantmanualdiscount (numeric) + 52. allcoupondiscount (numeric) + 53. goodspromotionmoney (numeric) + 54. assistantpromotionmoney (numeric) + 55. isusecoupon (boolean) + 56. isusediscount (boolean) + 57. isactivity (boolean) + 58. isbindmember (boolean) + 59. isfirst (integer) + 60. rechargecardamount (numeric) + 61. giftcardamount (numeric) + 62. source_file (text) + 63. source_endpoint (text) + 64. fetched_at (timestamp with time zone) + 65. payload (jsonb) + 66. content_hash (text) + +?? refund_transactions +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. tenantname (text) + 4. site_id (bigint) + 5. siteprofile (jsonb) + 6. relate_type (integer) + 7. relate_id (bigint) + 8. pay_sn (text) + 9. pay_amount (numeric) + 10. refund_amount (numeric) + 11. round_amount (numeric) + 12. pay_status (integer) + 13. pay_time (timestamp without time zone) + 14. create_time (timestamp without time zone) + 15. payment_method (integer) + 16. pay_terminal (integer) + 17. pay_config_id (bigint) + 18. online_pay_channel (integer) + 19. online_pay_type (integer) + 20. channel_fee (numeric) + 21. channel_payer_id (text) + 22. channel_pay_no (text) + 23. member_id (bigint) + 24. member_card_id (bigint) + 25. cashier_point_id (bigint) + 26. operator_id (bigint) + 27. action_type (integer) + 28. check_status (integer) + 29. is_revoke (integer) + 30. is_delete (integer) + 31. balance_frozen_amount (numeric) + 32. card_frozen_amount (numeric) + 33. source_file (text) + 34. source_endpoint (text) + 35. fetched_at (timestamp with time zone) + 36. payload (jsonb) + 37. content_hash (text) + +?? settlement_records +------------------------------------------------------------ + 1. id (bigint) + 2. tenantid (bigint) + 3. siteid (bigint) + 4. sitename (text) + 5. balanceamount (numeric) + 6. cardamount (numeric) + 7. cashamount (numeric) + 8. couponamount (numeric) + 9. createtime (timestamp with time zone) + 10. memberid (bigint) + 11. membername (text) + 12. tenantmembercardid (bigint) + 13. membercardtypename (text) + 14. memberphone (text) + 15. tableid (bigint) + 16. consumemoney (numeric) + 17. onlineamount (numeric) + 18. operatorid (bigint) + 19. operatorname (text) + 20. revokeorderid (bigint) + 21. revokeordername (text) + 22. revoketime (timestamp with time zone) + 23. payamount (numeric) + 24. pointamount (numeric) + 25. refundamount (numeric) + 26. settlename (text) + 27. settlerelateid (bigint) + 28. settlestatus (integer) + 29. settletype (integer) + 30. paytime (timestamp with time zone) + 31. roundingamount (numeric) + 32. paymentmethod (integer) + 33. adjustamount (numeric) + 34. assistantcxmoney (numeric) + 35. assistantpdmoney (numeric) + 36. couponsaleamount (numeric) + 37. memberdiscountamount (numeric) + 38. tablechargemoney (numeric) + 39. goodsmoney (numeric) + 40. realgoodsmoney (numeric) + 41. servicemoney (numeric) + 42. prepaymoney (numeric) + 43. salesmanname (text) + 44. orderremark (text) + 45. salesmanuserid (bigint) + 46. canberevoked (boolean) + 47. pointdiscountprice (numeric) + 48. pointdiscountcost (numeric) + 49. activitydiscount (numeric) + 50. serialnumber (bigint) + 51. assistantmanualdiscount (numeric) + 52. allcoupondiscount (numeric) + 53. goodspromotionmoney (numeric) + 54. assistantpromotionmoney (numeric) + 55. isusecoupon (boolean) + 56. isusediscount (boolean) + 57. isactivity (boolean) + 58. isbindmember (boolean) + 59. isfirst (integer) + 60. rechargecardamount (numeric) + 61. giftcardamount (numeric) + 62. source_file (text) + 63. source_endpoint (text) + 64. fetched_at (timestamp with time zone) + 65. payload (jsonb) + 66. content_hash (text) + +?? settlement_ticket_details +------------------------------------------------------------ + 1. ordersettleid (bigint) + 2. actualpayment (numeric) + 3. adjustamount (numeric) + 4. assistantmanualdiscount (numeric) + 5. balanceamount (numeric) + 6. cashiername (text) + 7. consumemoney (numeric) + 8. couponamount (numeric) + 9. deliveryaddress (text) + 10. deliveryfee (numeric) + 11. ledgeramount (numeric) + 12. memberdeductamount (numeric) + 13. memberofferamount (numeric) + 14. onlinereturnamount (numeric) + 15. orderremark (text) + 16. ordersettlenumber (bigint) + 17. paymemberbalance (numeric) + 18. paytime (timestamp without time zone) + 19. paymentmethod (integer) + 20. pointdiscountcost (numeric) + 21. pointdiscountprice (numeric) + 22. prepaymoney (numeric) + 23. refundamount (numeric) + 24. returngoodsamount (numeric) + 25. rewardname (text) + 26. settletype (text) + 27. siteaddress (text) + 28. sitebusinesstel (text) + 29. siteid (bigint) + 30. sitename (text) + 31. tenantid (bigint) + 32. tenantname (text) + 33. ticketcustomcontent (text) + 34. ticketremark (text) + 35. vouchermoney (numeric) + 36. memberprofile (jsonb) + 37. orderitem (jsonb) + 38. tenantmembercardlogs (jsonb) + 39. payload (jsonb) + 40. source_file (text) + 41. source_endpoint (text) + 42. fetched_at (timestamp with time zone) + 43. content_hash (text) + +?? site_tables_master +------------------------------------------------------------ + 1. id (bigint) + 2. site_id (bigint) + 3. sitename (text) + 4. appletQrCodeUrl (text) + 5. areaname (text) + 6. audit_status (integer) + 7. charge_free (integer) + 8. create_time (timestamp without time zone) + 9. delay_lights_time (integer) + 10. is_online_reservation (integer) + 11. is_rest_area (integer) + 12. light_status (integer) + 13. only_allow_groupon (integer) + 14. order_delay_time (integer) + 15. self_table (integer) + 16. show_status (integer) + 17. site_table_area_id (bigint) + 18. tablestatusname (text) + 19. table_cloth_use_cycle (integer) + 20. table_cloth_use_time (timestamp without time zone) + 21. table_name (text) + 22. table_price (numeric) + 23. table_status (integer) + 24. temporary_light_second (integer) + 25. virtual_table (integer) + 26. source_file (text) + 27. source_endpoint (text) + 28. fetched_at (timestamp with time zone) + 29. payload (jsonb) + 30. content_hash (text) + +?? stock_goods_category_tree +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. category_name (text) + 4. alias_name (text) + 5. pid (bigint) + 6. business_name (text) + 7. tenant_goods_business_id (bigint) + 8. open_salesman (integer) + 9. categoryboxes (jsonb) + 10. sort (integer) + 11. is_warehousing (integer) + 12. source_file (text) + 13. source_endpoint (text) + 14. fetched_at (timestamp with time zone) + 15. payload (jsonb) + 16. content_hash (text) + +?? store_goods_master +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. sitename (text) + 5. tenant_goods_id (bigint) + 6. goods_name (text) + 7. goods_bar_code (text) + 8. goods_category_id (bigint) + 9. goods_second_category_id (bigint) + 10. onecategoryname (text) + 11. twocategoryname (text) + 12. unit (text) + 13. sale_price (numeric) + 14. cost_price (numeric) + 15. cost_price_type (integer) + 16. min_discount_price (numeric) + 17. safe_stock (numeric) + 18. stock (numeric) + 19. stock_a (numeric) + 20. sale_num (numeric) + 21. total_purchase_cost (numeric) + 22. total_sales (numeric) + 23. average_monthly_sales (numeric) + 24. batch_stock_quantity (numeric) + 25. days_available (integer) + 26. provisional_total_cost (numeric) + 27. enable_status (integer) + 28. audit_status (integer) + 29. goods_state (integer) + 30. is_delete (integer) + 31. is_warehousing (integer) + 32. able_discount (integer) + 33. able_site_transfer (integer) + 34. forbid_sell_status (integer) + 35. freeze (integer) + 36. send_state (integer) + 37. custom_label_type (integer) + 38. option_required (integer) + 39. sale_channel (integer) + 40. sort (integer) + 41. remark (text) + 42. pinyin_initial (text) + 43. goods_cover (text) + 44. create_time (timestamp without time zone) + 45. update_time (timestamp without time zone) + 46. payload (jsonb) + 47. source_file (text) + 48. source_endpoint (text) + 49. fetched_at (timestamp with time zone) + 50. content_hash (text) + +?? store_goods_sales_records +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. siteid (bigint) + 5. sitename (text) + 6. site_goods_id (bigint) + 7. tenant_goods_id (bigint) + 8. order_settle_id (bigint) + 9. order_trade_no (text) + 10. order_goods_id (bigint) + 11. ordergoodsid (bigint) + 12. order_pay_id (bigint) + 13. order_coupon_id (bigint) + 14. ledger_name (text) + 15. ledger_group_name (text) + 16. ledger_amount (numeric) + 17. ledger_count (numeric) + 18. ledger_unit_price (numeric) + 19. ledger_status (integer) + 20. discount_money (numeric) + 21. discount_price (numeric) + 22. coupon_deduct_money (numeric) + 23. member_discount_amount (numeric) + 24. option_coupon_deduct_money (numeric) + 25. option_member_discount_money (numeric) + 26. point_discount_money (numeric) + 27. point_discount_money_cost (numeric) + 28. real_goods_money (numeric) + 29. cost_money (numeric) + 30. push_money (numeric) + 31. sales_type (integer) + 32. is_single_order (integer) + 33. is_delete (integer) + 34. goods_remark (text) + 35. option_price (numeric) + 36. option_value_name (text) + 37. option_name (text) + 38. member_coupon_id (bigint) + 39. package_coupon_id (bigint) + 40. sales_man_org_id (bigint) + 41. salesman_name (text) + 42. salesman_role_id (bigint) + 43. salesman_user_id (bigint) + 44. operator_id (bigint) + 45. operator_name (text) + 46. opensalesman (text) + 47. returns_number (integer) + 48. site_table_id (bigint) + 49. tenant_goods_business_id (bigint) + 50. tenant_goods_category_id (bigint) + 51. create_time (timestamp without time zone) + 52. payload (jsonb) + 53. source_file (text) + 54. source_endpoint (text) + 55. fetched_at (timestamp with time zone) + 56. content_hash (text) + +?? table_fee_discount_records +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. siteprofile (jsonb) + 5. site_table_id (bigint) + 6. tableprofile (jsonb) + 7. tenant_table_area_id (bigint) + 8. adjust_type (integer) + 9. ledger_amount (numeric) + 10. ledger_count (numeric) + 11. ledger_name (text) + 12. ledger_status (integer) + 13. applicant_id (bigint) + 14. applicant_name (text) + 15. operator_id (bigint) + 16. operator_name (text) + 17. order_settle_id (bigint) + 18. order_trade_no (text) + 19. is_delete (integer) + 20. create_time (timestamp without time zone) + 21. source_file (text) + 22. source_endpoint (text) + 23. fetched_at (timestamp with time zone) + 24. payload (jsonb) + 25. content_hash (text) + +?? table_fee_transactions +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. site_id (bigint) + 4. siteprofile (jsonb) + 5. site_table_id (bigint) + 6. site_table_area_id (bigint) + 7. site_table_area_name (text) + 8. tenant_table_area_id (bigint) + 9. order_trade_no (text) + 10. order_pay_id (bigint) + 11. order_settle_id (bigint) + 12. ledger_name (text) + 13. ledger_amount (numeric) + 14. ledger_count (numeric) + 15. ledger_unit_price (numeric) + 16. ledger_status (integer) + 17. ledger_start_time (timestamp without time zone) + 18. ledger_end_time (timestamp without time zone) + 19. start_use_time (timestamp without time zone) + 20. last_use_time (timestamp without time zone) + 21. real_table_use_seconds (integer) + 22. real_table_charge_money (numeric) + 23. add_clock_seconds (integer) + 24. adjust_amount (numeric) + 25. coupon_promotion_amount (numeric) + 26. member_discount_amount (numeric) + 27. used_card_amount (numeric) + 28. mgmt_fee (numeric) + 29. service_money (numeric) + 30. fee_total (numeric) + 31. is_single_order (integer) + 32. is_delete (integer) + 33. member_id (bigint) + 34. operator_id (bigint) + 35. operator_name (text) + 36. salesman_name (text) + 37. salesman_org_id (bigint) + 38. salesman_user_id (bigint) + 39. create_time (timestamp without time zone) + 40. payload (jsonb) + 41. source_file (text) + 42. source_endpoint (text) + 43. fetched_at (timestamp with time zone) + 44. content_hash (text) + +?? tenant_goods_master +------------------------------------------------------------ + 1. id (bigint) + 2. tenant_id (bigint) + 3. goods_name (text) + 4. goods_bar_code (text) + 5. goods_category_id (bigint) + 6. goods_second_category_id (bigint) + 7. categoryname (text) + 8. unit (text) + 9. goods_number (text) + 10. out_goods_id (text) + 11. goods_state (integer) + 12. sale_channel (integer) + 13. able_discount (integer) + 14. able_site_transfer (integer) + 15. is_delete (integer) + 16. is_warehousing (integer) + 17. isinsite (integer) + 18. cost_price (numeric) + 19. cost_price_type (integer) + 20. market_price (numeric) + 21. min_discount_price (numeric) + 22. common_sale_royalty (numeric) + 23. point_sale_royalty (numeric) + 24. pinyin_initial (text) + 25. commoditycode (text) + 26. commodity_code (text) + 27. goods_cover (text) + 28. supplier_id (bigint) + 29. remark_name (text) + 30. create_time (timestamp without time zone) + 31. update_time (timestamp without time zone) + 32. payload (jsonb) + 33. source_file (text) + 34. source_endpoint (text) + 35. fetched_at (timestamp with time zone) + 36. content_hash (text) diff --git a/tmp/sync_api_to_ods_columns.py b/tmp/sync_api_to_ods_columns.py new file mode 100644 index 0000000..7d501be --- /dev/null +++ b/tmp/sync_api_to_ods_columns.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +""" +同步 API 字段到 ODS 数据库表 +1. 检测 API JSON 字段与 ODS 表列的差异 +2. 生成并执行 DDL 添加缺失列 +3. 忽略 siteProfile 等嵌套对象字段 +""" +import json +import os +import sys +from datetime import datetime +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + + +# 忽略的 siteProfile 相关字段和其他非业务字段 +IGNORED_FIELDS = { + # siteProfile 内嵌字段 + "siteprofile", "address", "avatar", "business_tel", "customer_service_qrcode", + "customer_service_wechat", "fixed_pay_qrcode", "full_address", "latitude", "longitude", + "light_status", "light_token", "light_type", "org_id", "prod_env", "shop_name", + "shop_status", "site_label", "site_type", "tenant_site_region_id", "wifi_name", + "wifi_password", "attendance_distance", "attendance_enabled", "auto_light", + "ewelink_client_id", + # tableprofile 内嵌字段 + "tableprofile", + # 已有的系统字段 + "content_hash", "payload", "source_file", "source_endpoint", "fetched_at", "record_index", +} + +# API 字段类型推断规则 +def infer_column_type(field_name: str, sample_value=None) -> str: + """根据字段名和样本值推断 PostgreSQL 列类型""" + fn = field_name.lower() + + # ID 字段 + if fn.endswith("_id") or fn in ("id", "tenant_id", "member_id", "site_id", "table_id", + "operator_id", "relate_id", "order_id"): + return "BIGINT" + + # 金额字段 + if any(x in fn for x in ("_money", "_amount", "_price", "_cost", "_discount", "_balance", + "_deduct", "_fee", "_charge", "money", "amount", "price")): + return "NUMERIC(18,2)" + + # 时间字段 + if any(x in fn for x in ("_time", "time", "_date", "date")) or fn.startswith("create") or fn.startswith("update"): + return "TIMESTAMP" + + # 布尔/状态字段 + if fn.startswith("is_") or fn.startswith("can_") or fn.startswith("able_"): + return "INTEGER" + + # 数量/计数字段 + if any(x in fn for x in ("_count", "_num", "_seconds", "_minutes", "count", "num", "seconds")): + return "INTEGER" + + # 比率/折扣率 + if any(x in fn for x in ("_radio", "_ratio", "_rate")): + return "NUMERIC(10,4)" + + # 根据样本值推断 + if sample_value is not None: + if isinstance(sample_value, bool): + return "BOOLEAN" + if isinstance(sample_value, int): + if sample_value > 2147483647 or sample_value < -2147483648: + return "BIGINT" + return "INTEGER" + if isinstance(sample_value, float): + return "NUMERIC(18,2)" + if isinstance(sample_value, (list, dict)): + return "JSONB" + + # 默认文本 + return "TEXT" + + +def get_db_table_columns(db: DatabaseConnection, table_name: str) -> set: + """获取数据库表的所有列名""" + schema, name = table_name.split(".", 1) if "." in table_name else ("public", table_name) + sql = """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + """ + rows = db.query(sql, (schema, name)) + return {r["column_name"].lower() for r in rows} + + +def get_api_fields_from_comparison(comparison_file: Path) -> dict: + """从对比文件获取 API 字段""" + if not comparison_file.exists(): + return {} + with open(comparison_file, "r", encoding="utf-8") as f: + return json.load(f) + + +def generate_ddl_for_missing_fields(table_name: str, missing_fields: list, api_data: dict = None) -> list: + """生成添加缺失列的 DDL""" + ddl_list = [] + for field in missing_fields: + # 尝试从 API 数据获取样本值来推断类型 + sample_value = None + if api_data: + for record in api_data.get("data", [])[:10]: + if isinstance(record, dict) and field in record: + sample_value = record[field] + break + + col_type = infer_column_type(field, sample_value) + ddl = f'ALTER TABLE {table_name} ADD COLUMN IF NOT EXISTS "{field}" {col_type};' + ddl_list.append(ddl) + + return ddl_list + + +def main(): + print("=" * 80) + print("API → ODS 字段同步脚本") + print("时间:", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + print("=" * 80) + + # 连接数据库 + dsn = os.getenv("PG_DSN") + if not dsn: + print("[错误] 未找到 PG_DSN 环境变量") + return + + db = DatabaseConnection(dsn) + + # 加载对比数据 + comparison_file = Path(__file__).parent / "api_ods_comparison.json" + comparison = get_api_fields_from_comparison(comparison_file) + + if not comparison: + print("[错误] 未找到对比文件 api_ods_comparison.json") + db.close() + return + + all_ddl = [] + executed_ddl = [] + failed_ddl = [] + + for task_code, data in comparison.items(): + table_name = data.get("table_name") + missing = data.get("missing_in_ods", []) + + if not table_name or not missing: + continue + + # 过滤忽略的字段 + filtered_missing = [ + f for f in missing + if f.lower() not in IGNORED_FIELDS + ] + + if not filtered_missing: + continue + + # 获取数据库当前列 + current_cols = get_db_table_columns(db, table_name) + + # 二次过滤：排除已存在的列 + truly_missing = [ + f for f in filtered_missing + if f.lower() not in current_cols + ] + + if not truly_missing: + print(f"\n【{task_code}】({table_name})") + print(f" 所有缺失字段已在数据库中存在，跳过") + continue + + print(f"\n【{task_code}】({table_name})") + print(f" 需要添加 {len(truly_missing)} 列: {', '.join(truly_missing)}") + + # 生成 DDL + ddl_list = generate_ddl_for_missing_fields(table_name, truly_missing) + all_ddl.extend(ddl_list) + + # 执行 DDL + for ddl in ddl_list: + try: + db.execute(ddl) + db.commit() + executed_ddl.append(ddl) + print(f" [成功] {ddl[:80]}...") + except Exception as e: + db.rollback() + failed_ddl.append((ddl, str(e))) + print(f" [失败] {ddl[:60]}... - {e}") + + db.close() + + # 汇总 + print("\n" + "=" * 80) + print("执行汇总") + print("=" * 80) + print(f"总计生成 DDL: {len(all_ddl)} 条") + print(f"执行成功: {len(executed_ddl)} 条") + print(f"执行失败: {len(failed_ddl)} 条") + + if failed_ddl: + print("\n失败的 DDL:") + for ddl, err in failed_ddl: + print(f" - {ddl}") + print(f" 错误: {err}") + + # 保存执行日志 + log_file = Path(__file__).parent / "sync_ods_columns_log.json" + log = { + "executed_at": datetime.now().isoformat(), + "total_ddl": len(all_ddl), + "success_count": len(executed_ddl), + "failed_count": len(failed_ddl), + "executed_ddl": executed_ddl, + "failed_ddl": [{"ddl": d, "error": e} for d, e in failed_ddl], + } + with open(log_file, "w", encoding="utf-8") as f: + json.dump(log, f, ensure_ascii=False, indent=2) + + print(f"\n执行日志已保存到: {log_file}") + + return len(failed_ddl) == 0 + + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) diff --git a/tmp/sync_bd_manual.py b/tmp/sync_bd_manual.py new file mode 100644 index 0000000..ec13f6c --- /dev/null +++ b/tmp/sync_bd_manual.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +"""校验并同步 bd_manual 文档与数据库结构""" +import json +import re +from pathlib import Path +import psycopg2 + +DSN = 'postgresql://local-Python:Neo-local-1991125@100.64.0.4:5432/LLZQ-test' + +# 类型映射 (PostgreSQL -> 文档显示格式) +TYPE_MAP = { + 'bigint': 'BIGINT', + 'integer': 'INTEGER', + 'smallint': 'SMALLINT', + 'numeric': 'NUMERIC', + 'text': 'TEXT', + 'character varying': 'VARCHAR', + 'boolean': 'BOOLEAN', + 'timestamp with time zone': 'TIMESTAMPTZ', + 'timestamp without time zone': 'TIMESTAMP', + 'date': 'DATE', + 'jsonb': 'JSONB', + 'json': 'JSON', +} + +def get_db_schema(): + """获取数据库 schema""" + conn = psycopg2.connect(DSN) + cur = conn.cursor() + cur.execute(""" + SELECT table_name, column_name, data_type, is_nullable, + COALESCE(character_maximum_length, numeric_precision) as max_length, + numeric_scale + FROM information_schema.columns + WHERE table_schema = 'billiards_dwd' + ORDER BY table_name, ordinal_position + """) + + tables = {} + for row in cur.fetchall(): + table_name, col_name, data_type, nullable, max_len, scale = row + if table_name not in tables: + tables[table_name] = [] + + # 格式化类型 + type_str = TYPE_MAP.get(data_type, data_type.upper()) + if data_type == 'numeric' and max_len and scale is not None: + type_str = f'NUMERIC({max_len},{scale})' + elif data_type == 'character varying' and max_len: + type_str = f'VARCHAR({max_len})' + + tables[table_name].append({ + 'column': col_name, + 'type': type_str, + 'nullable': 'YES' if nullable == 'YES' else 'NO', + }) + + cur.close() + conn.close() + return tables + +def parse_md_fields(content): + """解析 MD 文档中的字段列表""" + fields = {} + # 匹配字段表格行 + pattern = r'\|\s*\d+\s*\|\s*(\w+)\s*\|\s*([^|]+)\s*\|\s*(\w+)\s*\|' + for match in re.finditer(pattern, content): + col_name = match.group(1).strip() + col_type = match.group(2).strip() + nullable = match.group(3).strip() + fields[col_name] = {'type': col_type, 'nullable': nullable} + return fields + +def compare_and_report(table_name, db_cols, doc_path): + """对比数据库和文档，返回差异""" + if not doc_path.exists(): + return {'missing_doc': True, 'table': table_name} + + content = doc_path.read_text(encoding='utf-8') + doc_fields = parse_md_fields(content) + + db_field_names = {c['column'] for c in db_cols} + doc_field_names = set(doc_fields.keys()) + + # 找出差异 + missing_in_doc = db_field_names - doc_field_names + extra_in_doc = doc_field_names - db_field_names + type_mismatches = [] + + for col in db_cols: + col_name = col['column'] + if col_name in doc_fields: + # 检查类型是否匹配 (忽略大小写和空格) + db_type = col['type'].upper().replace(' ', '') + doc_type = doc_fields[col_name]['type'].upper().replace(' ', '') + if db_type != doc_type: + type_mismatches.append({ + 'column': col_name, + 'db_type': col['type'], + 'doc_type': doc_fields[col_name]['type'] + }) + + return { + 'table': table_name, + 'missing_in_doc': list(missing_in_doc), + 'extra_in_doc': list(extra_in_doc), + 'type_mismatches': type_mismatches, + 'doc_path': str(doc_path), + } + +def main(): + db_schema = get_db_schema() + + main_dir = Path('etl_billiards/docs/bd_manual/main') + ex_dir = Path('etl_billiards/docs/bd_manual/Ex') + + all_diffs = [] + + for table_name, columns in sorted(db_schema.items()): + # 确定文档路径 + if table_name.endswith('_ex'): + base_name = table_name[:-3] # 去掉 _ex + doc_path = ex_dir / f'BD_manual_{table_name}.md' + else: + doc_path = main_dir / f'BD_manual_{table_name}.md' + + diff = compare_and_report(table_name, columns, doc_path) + if diff.get('missing_in_doc') or diff.get('extra_in_doc') or diff.get('type_mismatches') or diff.get('missing_doc'): + all_diffs.append(diff) + + # 输出报告 + print("=" * 80) + print("BD Manual vs Database Schema Comparison Report") + print("=" * 80) + + total_missing = 0 + total_extra = 0 + total_type_mismatch = 0 + + for diff in all_diffs: + table = diff['table'] + if diff.get('missing_doc'): + print(f"\n### {table}: MISSING DOCUMENT ###") + continue + + has_issues = False + + if diff['missing_in_doc']: + if not has_issues: + print(f"\n### {table} ###") + has_issues = True + print(f" Missing in doc ({len(diff['missing_in_doc'])}): {', '.join(sorted(diff['missing_in_doc']))}") + total_missing += len(diff['missing_in_doc']) + + if diff['extra_in_doc']: + if not has_issues: + print(f"\n### {table} ###") + has_issues = True + print(f" Extra in doc ({len(diff['extra_in_doc'])}): {', '.join(sorted(diff['extra_in_doc']))}") + total_extra += len(diff['extra_in_doc']) + + if diff['type_mismatches']: + if not has_issues: + print(f"\n### {table} ###") + has_issues = True + print(f" Type mismatches ({len(diff['type_mismatches'])}):") + for m in diff['type_mismatches']: + print(f" - {m['column']}: doc={m['doc_type']}, db={m['db_type']}") + total_type_mismatch += len(diff['type_mismatches']) + + print("\n" + "=" * 80) + print(f"Summary: {total_missing} missing, {total_extra} extra, {total_type_mismatch} type mismatches") + print("=" * 80) + + # 保存详细结果到 JSON + with open('tmp/bd_manual_diff.json', 'w', encoding='utf-8') as f: + json.dump(all_diffs, f, ensure_ascii=False, indent=2) + print(f"\nDetailed results saved to tmp/bd_manual_diff.json") + +if __name__ == '__main__': + main() diff --git a/tmp/sync_dwd_columns_log.json b/tmp/sync_dwd_columns_log.json new file mode 100644 index 0000000..5d02d69 --- /dev/null +++ b/tmp/sync_dwd_columns_log.json @@ -0,0 +1,10 @@ +{ + "executed_at": "2026-02-02T19:12:00.539963", + "total_ddl": 1, + "success_count": 1, + "failed_count": 0, + "executed_ddl": [ + "ALTER TABLE billiards_dwd.dwd_member_balance_change ADD COLUMN IF NOT EXISTS \"principal_change_amount\" NUMERIC(18,2);" + ], + "failed_ddl": [] +} \ No newline at end of file diff --git a/tmp/sync_ods_columns_log.json b/tmp/sync_ods_columns_log.json new file mode 100644 index 0000000..2394b1e --- /dev/null +++ b/tmp/sync_ods_columns_log.json @@ -0,0 +1,13 @@ +{ + "executed_at": "2026-02-02T19:10:13.492902", + "total_ddl": 4, + "success_count": 4, + "failed_count": 0, + "executed_ddl": [ + "ALTER TABLE billiards_ods.settlement_records ADD COLUMN IF NOT EXISTS \"tenant_id\" BIGINT;", + "ALTER TABLE billiards_ods.recharge_settlements ADD COLUMN IF NOT EXISTS \"tenant_id\" BIGINT;", + "ALTER TABLE billiards_ods.group_buy_packages ADD COLUMN IF NOT EXISTS \"tableareanamelist\" TEXT;", + "ALTER TABLE billiards_ods.group_buy_packages ADD COLUMN IF NOT EXISTS \"tenanttableareaidlist\" TEXT;" + ], + "failed_ddl": [] +} \ No newline at end of file diff --git a/tmp/sync_ods_to_dwd_columns.py b/tmp/sync_ods_to_dwd_columns.py new file mode 100644 index 0000000..d7a7fde --- /dev/null +++ b/tmp/sync_ods_to_dwd_columns.py @@ -0,0 +1,259 @@ +# -*- coding: utf-8 -*- +""" +同步 ODS 字段到 DWD 数据库表 +1. 检测 ODS 新增字段对应的 DWD 表是否缺失列 +2. 根据 dwd_load_task.py 的 FACT_MAPPINGS 生成 DDL +""" +import json +import os +import sys +from datetime import datetime +from pathlib import Path + +# 添加项目路径 +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + + +# ODS -> DWD 表映射（从 dwd_load_task.py 提取） +ODS_TO_DWD_MAP = { + "billiards_ods.table_fee_transactions": [ + "billiards_dwd.dim_site", "billiards_dwd.dim_site_ex", + "billiards_dwd.dwd_table_fee_log", "billiards_dwd.dwd_table_fee_log_ex", + ], + "billiards_ods.site_tables_master": [ + "billiards_dwd.dim_table", "billiards_dwd.dim_table_ex", + ], + "billiards_ods.assistant_accounts_master": [ + "billiards_dwd.dim_assistant", "billiards_dwd.dim_assistant_ex", + ], + "billiards_ods.assistant_service_records": [ + "billiards_dwd.dwd_assistant_service_log", "billiards_dwd.dwd_assistant_service_log_ex", + ], + "billiards_ods.assistant_cancellation_records": [ + "billiards_dwd.dwd_assistant_trash_event", "billiards_dwd.dwd_assistant_trash_event_ex", + ], + "billiards_ods.store_goods_sales_records": [ + "billiards_dwd.dwd_store_goods_sale", "billiards_dwd.dwd_store_goods_sale_ex", + ], + "billiards_ods.payment_transactions": [ + "billiards_dwd.dwd_payment", + ], + "billiards_ods.member_profiles": [ + "billiards_dwd.dim_member", "billiards_dwd.dim_member_ex", + ], + "billiards_ods.member_stored_value_cards": [ + "billiards_dwd.dim_member_card_account", "billiards_dwd.dim_member_card_account_ex", + ], + "billiards_ods.member_balance_changes": [ + "billiards_dwd.dwd_member_balance_change", "billiards_dwd.dwd_member_balance_change_ex", + ], + "billiards_ods.settlement_records": [ + "billiards_dwd.dwd_settlement_head", "billiards_dwd.dwd_settlement_head_ex", + ], + "billiards_ods.recharge_settlements": [ + "billiards_dwd.dwd_recharge_order", "billiards_dwd.dwd_recharge_order_ex", + ], + "billiards_ods.group_buy_packages": [ + "billiards_dwd.dim_groupbuy_package", "billiards_dwd.dim_groupbuy_package_ex", + ], + "billiards_ods.group_buy_redemption_records": [ + "billiards_dwd.dwd_groupbuy_redemption", "billiards_dwd.dwd_groupbuy_redemption_ex", + ], + "billiards_ods.table_fee_discount_records": [ + "billiards_dwd.dwd_table_fee_adjust", "billiards_dwd.dwd_table_fee_adjust_ex", + ], + "billiards_ods.tenant_goods_master": [ + "billiards_dwd.dim_tenant_goods", "billiards_dwd.dim_tenant_goods_ex", + ], + "billiards_ods.store_goods_master": [ + "billiards_dwd.dim_store_goods", "billiards_dwd.dim_store_goods_ex", + ], +} + +# 需要同步到 DWD 的新增 ODS 字段（从排查报告中获取） +# 格式: {ods_table: [(ods_col, dwd_col, dwd_table, col_type), ...]} +NEW_FIELDS_TO_DWD = { + "billiards_ods.table_fee_transactions": [ + ("activity_discount_amount", "activity_discount_amount", "billiards_dwd.dwd_table_fee_log", "NUMERIC(18,2)"), + ("real_service_money", "real_service_money", "billiards_dwd.dwd_table_fee_log", "NUMERIC(18,2)"), + ("order_consumption_type", "order_consumption_type", "billiards_dwd.dwd_table_fee_log_ex", "INTEGER"), + ], + "billiards_ods.assistant_service_records": [ + ("real_service_money", "real_service_money", "billiards_dwd.dwd_assistant_service_log", "NUMERIC(18,2)"), + ("assistantteamname", "assistant_team_name", "billiards_dwd.dwd_assistant_service_log_ex", "TEXT"), + ], + "billiards_ods.assistant_cancellation_records": [ + ("tenant_id", "tenant_id", "billiards_dwd.dwd_assistant_trash_event", "BIGINT"), + ], + "billiards_ods.store_goods_sales_records": [ + ("coupon_share_money", "coupon_share_money", "billiards_dwd.dwd_store_goods_sale", "NUMERIC(18,2)"), + ], + "billiards_ods.payment_transactions": [ + ("tenant_id", "tenant_id", "billiards_dwd.dwd_payment", "BIGINT"), + ], + "billiards_ods.member_profiles": [ + ("pay_money_sum", "pay_money_sum", "billiards_dwd.dim_member", "NUMERIC(18,2)"), + ("recharge_money_sum", "recharge_money_sum", "billiards_dwd.dim_member", "NUMERIC(18,2)"), + ("person_tenant_org_id", "person_tenant_org_id", "billiards_dwd.dim_member_ex", "BIGINT"), + ("person_tenant_org_name", "person_tenant_org_name", "billiards_dwd.dim_member_ex", "TEXT"), + ("register_source", "register_source", "billiards_dwd.dim_member_ex", "TEXT"), + ], + "billiards_ods.member_stored_value_cards": [ + ("principal_balance", "principal_balance", "billiards_dwd.dim_member_card_account", "NUMERIC(18,2)"), + ("member_grade", "member_grade", "billiards_dwd.dim_member_card_account", "INTEGER"), + ("able_share_member_discount", "able_share_member_discount", "billiards_dwd.dim_member_card_account_ex", "BOOLEAN"), + ("electricity_deduct_radio", "electricity_deduct_radio", "billiards_dwd.dim_member_card_account_ex", "NUMERIC(10,4)"), + ("electricity_discount", "electricity_discount", "billiards_dwd.dim_member_card_account_ex", "NUMERIC(10,4)"), + ("electricitycarddeduct", "electricity_card_deduct", "billiards_dwd.dim_member_card_account_ex", "BOOLEAN"), + ("rechargefreezebalance", "recharge_freeze_balance", "billiards_dwd.dim_member_card_account_ex", "NUMERIC(18,2)"), + ], + "billiards_ods.member_balance_changes": [ + ("principal_after", "principal_after", "billiards_dwd.dwd_member_balance_change", "NUMERIC(18,2)"), + ("principal_before", "principal_before", "billiards_dwd.dwd_member_balance_change", "NUMERIC(18,2)"), + ("principal_data", "principal_change_amount", "billiards_dwd.dwd_member_balance_change", "NUMERIC(18,2)"), + ], + "billiards_ods.settlement_records": [ + ("tenant_id", "tenant_id", "billiards_dwd.dwd_settlement_head", "BIGINT"), + ], + "billiards_ods.recharge_settlements": [ + ("tenant_id", "tenant_id", "billiards_dwd.dwd_recharge_order", "BIGINT"), + ], + "billiards_ods.group_buy_packages": [ + ("sort", "sort", "billiards_dwd.dim_groupbuy_package", "INTEGER"), + ("is_first_limit", "is_first_limit", "billiards_dwd.dim_groupbuy_package", "BOOLEAN"), + ("tenantcouponsaleorderitemid", "tenant_coupon_sale_order_item_id", "billiards_dwd.dim_groupbuy_package_ex", "BIGINT"), + ], + "billiards_ods.group_buy_redemption_records": [ + ("coupon_sale_id", "coupon_sale_id", "billiards_dwd.dwd_groupbuy_redemption", "BIGINT"), + ("member_discount_money", "member_discount_money", "billiards_dwd.dwd_groupbuy_redemption", "NUMERIC(18,2)"), + ("assistant_share_money", "assistant_share_money", "billiards_dwd.dwd_groupbuy_redemption_ex", "NUMERIC(18,2)"), + ("table_share_money", "table_share_money", "billiards_dwd.dwd_groupbuy_redemption_ex", "NUMERIC(18,2)"), + ("goods_share_money", "goods_share_money", "billiards_dwd.dwd_groupbuy_redemption_ex", "NUMERIC(18,2)"), + ("recharge_share_money", "recharge_share_money", "billiards_dwd.dwd_groupbuy_redemption_ex", "NUMERIC(18,2)"), + ], + "billiards_ods.site_tables_master": [ + ("order_id", "order_id", "billiards_dwd.dim_table", "BIGINT"), + ], + "billiards_ods.store_goods_master": [ + ("commodity_code", "commodity_code", "billiards_dwd.dim_store_goods", "TEXT"), + ("not_sale", "not_sale", "billiards_dwd.dim_store_goods", "INTEGER"), + ], + "billiards_ods.table_fee_discount_records": [ + ("table_name", "table_name", "billiards_dwd.dwd_table_fee_adjust", "TEXT"), + ("table_price", "table_price", "billiards_dwd.dwd_table_fee_adjust", "NUMERIC(18,2)"), + ("charge_free", "charge_free", "billiards_dwd.dwd_table_fee_adjust", "BOOLEAN"), + ("area_type_id", "area_type_id", "billiards_dwd.dwd_table_fee_adjust_ex", "BIGINT"), + ("site_table_area_id", "site_table_area_id", "billiards_dwd.dwd_table_fee_adjust_ex", "BIGINT"), + ("site_table_area_name", "site_table_area_name", "billiards_dwd.dwd_table_fee_adjust_ex", "TEXT"), + ("sitename", "site_name", "billiards_dwd.dwd_table_fee_adjust_ex", "TEXT"), + ("tenant_name", "tenant_name", "billiards_dwd.dwd_table_fee_adjust_ex", "TEXT"), + ], + "billiards_ods.tenant_goods_master": [ + ("not_sale", "not_sale", "billiards_dwd.dim_tenant_goods", "INTEGER"), + ], +} + + +def get_db_table_columns(db: DatabaseConnection, table_name: str) -> set: + """获取数据库表的所有列名""" + schema, name = table_name.split(".", 1) if "." in table_name else ("public", table_name) + sql = """ + SELECT column_name + FROM information_schema.columns + WHERE table_schema = %s AND table_name = %s + """ + rows = db.query(sql, (schema, name)) + return {r["column_name"].lower() for r in rows} + + +def main(): + print("=" * 80) + print("ODS → DWD 字段同步脚本") + print("时间:", datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + print("=" * 80) + + # 连接数据库 + dsn = os.getenv("PG_DSN") + if not dsn: + print("[错误] 未找到 PG_DSN 环境变量") + return False + + db = DatabaseConnection(dsn) + + all_ddl = [] + executed_ddl = [] + failed_ddl = [] + + for ods_table, fields in NEW_FIELDS_TO_DWD.items(): + print(f"\n处理 ODS 表: {ods_table}") + + for ods_col, dwd_col, dwd_table, col_type in fields: + # 检查 DWD 表是否存在该列 + try: + dwd_cols = get_db_table_columns(db, dwd_table) + except Exception as e: + print(f" [跳过] DWD 表 {dwd_table} 不存在或无法访问: {e}") + continue + + if dwd_col.lower() in dwd_cols: + print(f" [存在] {dwd_table}.{dwd_col}") + continue + + # 生成 DDL + ddl = f'ALTER TABLE {dwd_table} ADD COLUMN IF NOT EXISTS "{dwd_col}" {col_type};' + all_ddl.append(ddl) + + # 执行 DDL + try: + db.execute(ddl) + db.commit() + executed_ddl.append(ddl) + print(f" [新增] {dwd_table}.{dwd_col} ({col_type})") + except Exception as e: + db.rollback() + failed_ddl.append((ddl, str(e))) + print(f" [失败] {dwd_table}.{dwd_col} - {e}") + + db.close() + + # 汇总 + print("\n" + "=" * 80) + print("执行汇总") + print("=" * 80) + print(f"总计生成 DDL: {len(all_ddl)} 条") + print(f"执行成功: {len(executed_ddl)} 条") + print(f"执行失败: {len(failed_ddl)} 条") + + if failed_ddl: + print("\n失败的 DDL:") + for ddl, err in failed_ddl: + print(f" - {ddl}") + print(f" 错误: {err}") + + # 保存执行日志 + log_file = Path(__file__).parent / "sync_dwd_columns_log.json" + log = { + "executed_at": datetime.now().isoformat(), + "total_ddl": len(all_ddl), + "success_count": len(executed_ddl), + "failed_count": len(failed_ddl), + "executed_ddl": executed_ddl, + "failed_ddl": [{"ddl": d, "error": e} for d, e in failed_ddl], + } + with open(log_file, "w", encoding="utf-8") as f: + json.dump(log, f, ensure_ascii=False, indent=2) + + print(f"\n执行日志已保存到: {log_file}") + + return len(failed_ddl) == 0 + + +if __name__ == "__main__": + success = main() + sys.exit(0 if success else 1) diff --git a/tmp/test_backfill_feature.py b/tmp/test_backfill_feature.py new file mode 100644 index 0000000..d5da1b9 --- /dev/null +++ b/tmp/test_backfill_feature.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +""" +测试 ODS 回填特性 +""" +import os +import sys +from pathlib import Path + +project_root = Path(__file__).parent.parent / "etl_billiards" +sys.path.insert(0, str(project_root)) + +from dotenv import load_dotenv +load_dotenv(project_root / ".env") + +from database.connection import DatabaseConnection + +dsn = os.getenv("PG_DSN") +db = DatabaseConnection(dsn) + +print("=== 测试 ODS 回填特性 ===") + +# 1. 创建一个测试场景：找一条有 NULL 值的记录 +result = db.query(""" + SELECT id, plcouponsaleamount, mervousalesamount, + payload->'settleList'->>'plCouponSaleAmount' as payload_val + FROM billiards_ods.settlement_records + WHERE plcouponsaleamount IS NOT NULL + LIMIT 1 +""") + +if result: + row = result[0] + print(f"找到测试记录: id={row['id']}") + print(f" 当前 plcouponsaleamount: {row['plcouponsaleamount']}") + print(f" payload 中的值: {row['payload_val']}") +else: + print("未找到测试记录") + +# 2. 模拟生成的 SQL 语句 +print("\n=== 生成的 SQL 示例 ===") + +# 获取表结构 +cols = db.query(""" + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'billiards_ods' + AND table_name = 'settlement_records' + ORDER BY ordinal_position +""") + +col_names = [c["column_name"] for c in cols] +pk_cols = ["id"] # 假设主键是 id + +meta_cols = {"payload", "source_file", "source_endpoint", "fetched_at", "content_hash"} +pk_cols_lower = {c.lower() for c in pk_cols} +update_cols = [ + c for c in col_names + if c.lower() not in pk_cols_lower and c.lower() not in meta_cols +] + +print(f"表有 {len(col_names)} 列") +print(f"可更新列: {len(update_cols)} 列") + +# 生成 SQL +table = "billiards_ods.settlement_records" +pk_clause = ", ".join(f'"{c}"' for c in pk_cols) +set_clause = ", ".join( + f'"{c}" = COALESCE({table}."{c}", EXCLUDED."{c}")' + for c in update_cols[:3] # 只显示前3个 +) +where_clause = " OR ".join(f'{table}."{c}" IS NULL' for c in update_cols[:3]) + +print(f"\nSQL 示例 (前3列):") +print(f"INSERT INTO {table} (...) VALUES ...") +print(f"ON CONFLICT ({pk_clause}) DO UPDATE SET") +print(f" {set_clause}") +print(f"WHERE {where_clause}") + +print("\n=== 特性说明 ===") +print("1. 新记录 -> 正常插入") +print("2. 已存在记录 -> 只更新 NULL 列 (COALESCE)") +print("3. 已有值的列 -> 保持不变") +print("4. 可通过配置 run.ods_backfill_null_columns=false 禁用") + +db.close() +print("\n测试完成!") diff --git a/tmp/test_conflict_modes.py b/tmp/test_conflict_modes.py new file mode 100644 index 0000000..0778ebd --- /dev/null +++ b/tmp/test_conflict_modes.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +""" +测试 ODS 冲突处理三种模式 +""" +print("=" * 70) +print("ODS 冲突处理模式说明") +print("=" * 70) + +modes = [ + ("nothing", "跳过已存在记录", """ +INSERT INTO table (...) VALUES (...) +ON CONFLICT (pk) DO NOTHING + +行为: 已存在的记录完全跳过，不做任何更新 +适用: 严格保留原始快照，不允许修改历史数据 +"""), + + ("backfill", "回填 NULL 列", """ +INSERT INTO table (...) VALUES (...) +ON CONFLICT (pk) DO UPDATE SET + col1 = COALESCE(table.col1, EXCLUDED.col1), + col2 = COALESCE(table.col2, EXCLUDED.col2) +WHERE table.col1 IS NULL OR table.col2 IS NULL + +行为: 只填充数据库中为 NULL 的字段，已有值保持不变 +适用: 新增字段后回填历史数据，但不覆盖已有值 +"""), + + ("update", "全字段对比更新 (默认)", """ +INSERT INTO table (...) VALUES (...) +ON CONFLICT (pk) DO UPDATE SET + col1 = EXCLUDED.col1, + col2 = EXCLUDED.col2 +WHERE table.col1 IS DISTINCT FROM EXCLUDED.col1 + OR table.col2 IS DISTINCT FROM EXCLUDED.col2 + +行为: 对比所有字段，有变化则更新 +适用: 数据同步，保持与 API 一致 +"""), +] + +for mode, title, sql in modes: + print(f"\n【模式: {mode}】{title}") + print("-" * 50) + print(sql) + +print("=" * 70) +print("配置方式 (在 .env 中设置)") +print("=" * 70) +print(""" +# 方式1: 直接设置模式 +run.ods_conflict_mode=update # 全字段对比更新 (默认) +run.ods_conflict_mode=backfill # 只回填 NULL +run.ods_conflict_mode=nothing # 跳过已存在 + +# 方式2: 兼容旧配置 +run.ods_backfill_null_columns=false # 等同于 nothing 模式 +""") + +print("=" * 70) +print("对比表") +print("=" * 70) +print(""" +| 场景 | nothing | backfill | update | +|--------------------------|---------|----------|--------| +| 新记录 | 插入 | 插入 | 插入 | +| 已存在 + 字段已有值 | 跳过 | 保留原值 | 更新 | +| 已存在 + 字段为 NULL | 跳过 | 填充新值 | 填充 | +| 已存在 + API值与DB相同 | 跳过 | 跳过 | 跳过 | +""")