Neo
2a7a5d68aa
主线 1: rns1-customer-coach-api + 04-miniapp-core-business 后端实施
- 新增 GET /xcx/coaches/{id}/banner 轻量接口
- performance/records 加 coach_id 参数 + view_board_coach 权限分流
- coach/customer/performance/board/task 服务层重构
- fdw_queries 结算单粒度聚合 + consumption_summary 视图统一
- task_generator 回访宽限 72h + UPSERT 替代策略 + Step 5 保底清理
- recall_detector settle_type=3 双重限制 + 门店级 resolved
主线 2: 小程序权限分流 + 新增 coach-service-records 管理者视角业绩明细页
- perf-progress 共享模块去重 task-list/coach-detail 动画逻辑
- isScattered 散客标记端到端
- foodDetail/phoneFull/creator* 字段透传
主线 3: P19 指数回测框架 Phase 1+2
- 3 个指数表 stat_date 日快照模式
- 新增 DWS_INDEX_BACKFILL / DWS_TASK_SIMULATION 工具任务
- task_engine 升级 HTTP 实时 + 推演回测双模式
主线 4: Core 维度层启用
- 新增 CORE_DIM_SYNC 任务（DWD → core 4 维度表）
- 修复 app 视图空查询问题
主线 5: member_project_tag 改为 LAST_30_VISITS 消费次数窗口
主线 6: 2 个迁移 SQL 已执行（stat_date + member_project_tag 新窗口）
- schema 基线与 DDL 快照同步
主线 7: 开发机路径迁移 C:\NeoZQYY → C:\Project\NeoZQYY（约 95% 改动量）
附带: 新建运维脚本（churned_customer_report / simulate_historical_tasks /
backfill_index_snapshots）+ tools/task-analysis/ 任务分析工具
合计 157 文件。未包含中间产物（tmp/ .playwright-mcp/ inspect-* excel/sheet 分析 txt）。
审计记录见下一个 commit。
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
218 lines
12 KiB
Python
218 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
||
# AI_CHANGELOG
|
||
# - 2026-02-14 | 删除废弃代码残留：移除重复的 ODS_TASK_CLASSES 注册循环（底部），保留唯一一处（顶部）
|
||
# 直接原因: 原文件有两处 `for code, task_cls in ODS_TASK_CLASSES.items()` 循环，导致重复注册
|
||
# 验证: `python -c "from orchestration.task_registry import default_registry; print(len(default_registry.get_all_task_codes()))"` → 52
|
||
# CHANGE [2026-02-14] intent: 移除 14 个废弃独立 ODS 任务 + 3 个废弃独立 DWD 任务的导入与注册
|
||
# assumptions: 这些任务写入不存在的 billiards.* schema，已被通用 ODS 任务（ods.*）替代
|
||
# prompt: "删除废弃的独立 ODS/DWD 任务及其 loader"
|
||
# 验证: pytest tests/unit -x 确认无 import 错误
|
||
"""任务注册表"""
|
||
from dataclasses import dataclass, field
|
||
# ODS 层任务（仅保留通用 ODS 任务工厂 + JSON 归档）
|
||
from tasks.ods.ods_tasks import ODS_TASK_CLASSES
|
||
from tasks.ods.ods_json_archive_task import OdsJsonArchiveTask
|
||
|
||
# DWD 层任务（仅保留核心装载 + 质量检查）
|
||
from tasks.dwd.dwd_load_task import DwdLoadTask
|
||
from tasks.dwd.dwd_quality_task import DwdQualityTask
|
||
|
||
# 工具类任务
|
||
from tasks.utility.manual_ingest_task import ManualIngestTask
|
||
from tasks.utility.init_schema_task import InitOdsSchemaTask
|
||
from tasks.utility.init_dwd_schema_task import InitDwdSchemaTask
|
||
from tasks.utility.init_dws_schema_task import InitDwsSchemaTask
|
||
from tasks.utility.check_cutoff_task import CheckCutoffTask
|
||
from tasks.utility.dws_build_order_summary_task import DwsBuildOrderSummaryTask
|
||
from tasks.utility.data_integrity_task import DataIntegrityTask
|
||
from tasks.utility.seed_dws_config_task import SeedDwsConfigTask
|
||
|
||
# DWS 层任务导入
|
||
from tasks.dws import (
|
||
AssistantDailyTask,
|
||
AssistantOrderContributionTask,
|
||
AssistantMonthlyTask,
|
||
AssistantCustomerTask,
|
||
AssistantSalaryTask,
|
||
AssistantFinanceTask,
|
||
MemberConsumptionTask,
|
||
MemberVisitTask,
|
||
AssistantProjectTagTask,
|
||
MemberProjectTagTask,
|
||
FinanceDailyTask,
|
||
FinanceRechargeTask,
|
||
FinanceIncomeStructureTask,
|
||
FinanceDiscountDetailTask,
|
||
FinanceAreaDailyTask,
|
||
# 库存汇总任务
|
||
GoodsStockDailyTask,
|
||
GoodsStockWeeklyTask,
|
||
GoodsStockMonthlyTask,
|
||
# 指数算法任务
|
||
WinbackIndexTask,
|
||
NewconvIndexTask,
|
||
MlManualImportTask,
|
||
RelationIndexTask,
|
||
SpendingPowerIndexTask,
|
||
)
|
||
# CHANGE [2026-07-14] intent: 合并 MV 刷新 + 数据清理为 DWS_MAINTENANCE
|
||
from tasks.dws.maintenance_task import DwsMaintenanceTask
|
||
from tasks.dws.core_dim_sync_task import CoreDimSyncTask
|
||
# CHANGE 2026-03-29 | DWS_TASK_ENGINE：编排后端任务引擎（完成检查→过期检查→任务生成）
|
||
from tasks.dws.task_engine import DwsTaskEngineTask
|
||
# CHANGE 2026-04-12 | 指数回填工具任务
|
||
from tasks.utility.index_backfill_task import IndexBackfillTask
|
||
|
||
|
||
@dataclass
|
||
class TaskMeta:
|
||
"""任务元数据"""
|
||
task_class: type
|
||
requires_db_config: bool = True
|
||
layer: str | None = None # "ODS" / "DWD" / "DWS" / "INDEX" / None
|
||
task_type: str = "etl" # "etl" / "utility" / "verification"
|
||
depends_on: list[str] = field(default_factory=list) # 依赖的任务代码列表
|
||
|
||
|
||
class TaskRegistry:
|
||
"""任务注册和工厂"""
|
||
|
||
def __init__(self):
|
||
self._tasks: dict[str, TaskMeta] = {}
|
||
|
||
def register(
|
||
self,
|
||
task_code: str,
|
||
task_class: type,
|
||
requires_db_config: bool = True,
|
||
layer: str | None = None,
|
||
task_type: str = "etl",
|
||
depends_on: list[str] | None = None,
|
||
):
|
||
"""注册任务类及其元数据。向后兼容：仅传 task_code 和 task_class 时使用默认值。"""
|
||
self._tasks[task_code.upper()] = TaskMeta(
|
||
task_class=task_class,
|
||
requires_db_config=requires_db_config,
|
||
layer=layer,
|
||
task_type=task_type,
|
||
depends_on=depends_on or [],
|
||
)
|
||
|
||
def create_task(self, task_code: str, config, db_connection, api_client, logger):
|
||
"""创建任务实例"""
|
||
task_code = task_code.upper()
|
||
if task_code not in self._tasks:
|
||
raise ValueError(f"未知的任务类型: {task_code}")
|
||
|
||
task_class = self._tasks[task_code].task_class
|
||
return task_class(config, db_connection, api_client, logger)
|
||
|
||
def get_metadata(self, task_code: str) -> TaskMeta | None:
|
||
"""查询任务元数据。"""
|
||
return self._tasks.get(task_code.upper())
|
||
|
||
def get_tasks_by_layer(self, layer: str) -> list[str]:
|
||
"""获取指定层的所有任务代码。"""
|
||
return [
|
||
code for code, meta in self._tasks.items()
|
||
if meta.layer and meta.layer.upper() == layer.upper()
|
||
]
|
||
|
||
def is_utility_task(self, task_code: str) -> bool:
|
||
"""判断是否为工具类任务（不需要游标/运行记录）。"""
|
||
meta = self.get_metadata(task_code)
|
||
return meta is not None and not meta.requires_db_config
|
||
|
||
def get_all_task_codes(self) -> list[str]:
|
||
"""获取所有已注册的任务代码"""
|
||
return list(self._tasks.keys())
|
||
|
||
|
||
|
||
|
||
# 默认注册表
|
||
default_registry = TaskRegistry()
|
||
|
||
# ── ODS 层：通用 ODS 任务（由 ODS_TASK_CLASSES 动态生成）─────
|
||
for code, task_cls in ODS_TASK_CLASSES.items():
|
||
default_registry.register(code, task_cls, layer="ODS")
|
||
|
||
# ── DWD 层任务 ────────────────────────────────────────────────
|
||
default_registry.register("DWD_LOAD_FROM_ODS", DwdLoadTask, layer="DWD")
|
||
default_registry.register("DWD_QUALITY_CHECK", DwdQualityTask, requires_db_config=False, layer="DWD", task_type="verification")
|
||
|
||
# ── 工具类任务 ────────────────────────────────────────────────
|
||
default_registry.register("MANUAL_INGEST", ManualIngestTask, requires_db_config=False, task_type="utility")
|
||
default_registry.register("INIT_ODS_SCHEMA", InitOdsSchemaTask, requires_db_config=False, task_type="utility")
|
||
default_registry.register("INIT_DWD_SCHEMA", InitDwdSchemaTask, requires_db_config=False, task_type="utility")
|
||
default_registry.register("INIT_DWS_SCHEMA", InitDwsSchemaTask, requires_db_config=False, task_type="utility")
|
||
default_registry.register("ODS_JSON_ARCHIVE", OdsJsonArchiveTask, requires_db_config=False, task_type="utility")
|
||
default_registry.register("CHECK_CUTOFF", CheckCutoffTask, requires_db_config=False, task_type="utility")
|
||
default_registry.register("SEED_DWS_CONFIG", SeedDwsConfigTask, task_type="utility")
|
||
|
||
# ── 校验类任务 ────────────────────────────────────────────────
|
||
default_registry.register("DATA_INTEGRITY_CHECK", DataIntegrityTask, requires_db_config=False, task_type="verification")
|
||
|
||
# ── Core 层同步任务 ────────────────────────────────────────────
|
||
default_registry.register("CORE_DIM_SYNC", CoreDimSyncTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
|
||
|
||
# ── DWS 层业务任务 ────────────────────────────────────────────
|
||
default_registry.register("DWS_BUILD_ORDER_SUMMARY", DwsBuildOrderSummaryTask, requires_db_config=False, layer="DWS")
|
||
default_registry.register("DWS_ASSISTANT_DAILY", AssistantDailyTask, layer="DWS")
|
||
default_registry.register("DWS_ASSISTANT_ORDER_CONTRIBUTION", AssistantOrderContributionTask, layer="DWS")
|
||
# CHANGE [2026-07-17] intent: 为已知依赖关系添加 depends_on 声明（需求 8.1, 8.2）
|
||
default_registry.register("DWS_ASSISTANT_MONTHLY", AssistantMonthlyTask, layer="DWS", depends_on=["DWS_ASSISTANT_DAILY"])
|
||
default_registry.register("DWS_ASSISTANT_CUSTOMER", AssistantCustomerTask, layer="DWS")
|
||
default_registry.register("DWS_ASSISTANT_SALARY", AssistantSalaryTask, layer="DWS")
|
||
default_registry.register("DWS_ASSISTANT_FINANCE", AssistantFinanceTask, layer="DWS", depends_on=["DWS_ASSISTANT_SALARY"])
|
||
default_registry.register("DWS_MEMBER_CONSUMPTION", MemberConsumptionTask, layer="DWS")
|
||
default_registry.register("DWS_MEMBER_VISIT", MemberVisitTask, layer="DWS")
|
||
# CHANGE [2026-03-27] intent: 移除对 DWD_LOAD_FROM_ODS 的显式依赖，dwd_dws Flow 下该依赖不在批次内只产生无意义 warning
|
||
default_registry.register("DWS_ASSISTANT_PROJECT_TAG", AssistantProjectTagTask, layer="DWS")
|
||
default_registry.register("DWS_MEMBER_PROJECT_TAG", MemberProjectTagTask, layer="DWS")
|
||
default_registry.register("DWS_FINANCE_DAILY", FinanceDailyTask, layer="DWS")
|
||
default_registry.register("DWS_FINANCE_RECHARGE", FinanceRechargeTask, layer="DWS")
|
||
default_registry.register("DWS_FINANCE_INCOME_STRUCTURE", FinanceIncomeStructureTask, layer="DWS")
|
||
default_registry.register("DWS_FINANCE_DISCOUNT_DETAIL", FinanceDiscountDetailTask, layer="DWS")
|
||
default_registry.register("DWS_FINANCE_AREA_DAILY", FinanceAreaDailyTask, layer="DWS", depends_on=["DWS_FINANCE_DAILY"])
|
||
# CHANGE [2026-03-27] intent: 移除对 DWD_LOAD_FROM_ODS 的显式依赖，dwd_dws Flow 下该依赖不在批次内只产生无意义 warning
|
||
default_registry.register("DWS_GOODS_STOCK_DAILY", GoodsStockDailyTask, layer="DWS")
|
||
default_registry.register("DWS_GOODS_STOCK_WEEKLY", GoodsStockWeeklyTask, layer="DWS")
|
||
default_registry.register("DWS_GOODS_STOCK_MONTHLY", GoodsStockMonthlyTask, layer="DWS")
|
||
# CHANGE [2026-07-14] intent: 移除 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_FINANCE_DAILY / DWS_MV_REFRESH_ASSISTANT_DAILY，
|
||
# 替换为统一维护任务 DWS_MAINTENANCE（需求 4.5）
|
||
# depends_on: 所有其他 DWS 任务——MV 刷新和清理应在数据写入后执行
|
||
default_registry.register("DWS_MAINTENANCE", DwsMaintenanceTask, layer="DWS", depends_on=[
|
||
"DWS_ASSISTANT_DAILY", "DWS_ASSISTANT_ORDER_CONTRIBUTION",
|
||
"DWS_ASSISTANT_MONTHLY", "DWS_ASSISTANT_CUSTOMER",
|
||
"DWS_ASSISTANT_SALARY", "DWS_ASSISTANT_FINANCE",
|
||
"DWS_MEMBER_CONSUMPTION", "DWS_MEMBER_VISIT",
|
||
"DWS_ASSISTANT_PROJECT_TAG", "DWS_MEMBER_PROJECT_TAG",
|
||
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
|
||
"DWS_FINANCE_INCOME_STRUCTURE", "DWS_FINANCE_DISCOUNT_DETAIL",
|
||
"DWS_FINANCE_AREA_DAILY",
|
||
"DWS_BUILD_ORDER_SUMMARY",
|
||
"DWS_GOODS_STOCK_DAILY", "DWS_GOODS_STOCK_WEEKLY", "DWS_GOODS_STOCK_MONTHLY",
|
||
])
|
||
|
||
# ── INDEX 层：指数算法任务 ────────────────────────────────────
|
||
# CHANGE [2026-07-17] intent: 为指数任务添加 depends_on 声明（需求 8.1, 8.2）
|
||
default_registry.register("DWS_WINBACK_INDEX", WinbackIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_MEMBER_VISIT", "DWS_MEMBER_CONSUMPTION"])
|
||
default_registry.register("DWS_NEWCONV_INDEX", NewconvIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_MEMBER_VISIT", "DWS_MEMBER_CONSUMPTION"])
|
||
default_registry.register("DWS_ML_MANUAL_IMPORT", MlManualImportTask, requires_db_config=False, layer="INDEX")
|
||
default_registry.register("DWS_RELATION_INDEX", RelationIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_ASSISTANT_DAILY"])
|
||
default_registry.register("DWS_SPENDING_POWER_INDEX", SpendingPowerIndexTask, requires_db_config=False, layer="INDEX", depends_on=["DWS_MEMBER_CONSUMPTION"])
|
||
|
||
# ── 任务引擎编排 ─────────────────────────────────────────────
|
||
# CHANGE 2026-03-29 | DWS_TASK_ENGINE：DWS 指数计算完成后执行后端任务引擎
|
||
# depends_on: 所有指数任务——任务生成依赖 WBI/NCI/RS 指数数据
|
||
default_registry.register("DWS_TASK_ENGINE", DwsTaskEngineTask, requires_db_config=False, layer="INDEX", depends_on=[
|
||
"DWS_WINBACK_INDEX", "DWS_NEWCONV_INDEX", "DWS_RELATION_INDEX", "DWS_INDEX_BACKFILL",
|
||
])
|
||
|
||
# ── 回填 / 推演工具任务 ─────────────────────────────────────────
|
||
# CHANGE 2026-04-12 | 指数日快照回填工具任务
|
||
# layer="INDEX" 确保拓扑排序中 DWS_TASK_ENGINE 在其后执行（同层显式依赖优先）
|
||
# depends_on DWD_LOAD_FROM_ODS：回填读取 dwd.* 表，需要 DWD 数据已入库
|
||
default_registry.register("DWS_INDEX_BACKFILL", IndexBackfillTask, requires_db_config=False, layer="INDEX", task_type="utility", depends_on=["DWD_LOAD_FROM_ODS"])
|