230 lines
16 KiB
Python
230 lines
16 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""静态任务注册表
|
||
|
||
从 ETL orchestration/task_registry.py 提取的任务元数据硬编码副本。
|
||
后端不直接导入 ETL 代码，避免引入重量级依赖链。
|
||
|
||
业务域分组逻辑：按任务代码前缀 / 目标表语义归类，与 GUI 保持一致。
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from dataclasses import dataclass, field
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class TaskDefinition:
|
||
"""单个 ETL 任务的元数据"""
|
||
|
||
code: str
|
||
name: str
|
||
description: str
|
||
domain: str # 业务域：会员 / 结算 / 助教 / 商品 / 台桌 / 团购 / 库存 / 财务 / 指数 / 工具
|
||
layer: str # ODS / DWD / DWS / INDEX / UTILITY
|
||
requires_window: bool = True
|
||
is_ods: bool = False
|
||
is_dimension: bool = False
|
||
default_enabled: bool = True
|
||
is_common: bool = True # 常用任务标记，False 表示工具类/手动类任务
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class DwdTableDefinition:
|
||
"""DWD 表元数据"""
|
||
|
||
table_name: str # 完整表名（含 schema）
|
||
display_name: str
|
||
domain: str
|
||
ods_source: str # 对应的 ODS 源表
|
||
is_dimension: bool = False
|
||
|
||
|
||
# ── ODS 任务定义 ──────────────────────────────────────────────
|
||
|
||
ODS_TASKS: list[TaskDefinition] = [
|
||
TaskDefinition("ODS_ASSISTANT_ACCOUNT", "助教账号", "抽取助教账号主数据", "助教", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_ASSISTANT_LEDGER", "助教服务记录", "抽取助教服务流水", "助教", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_ASSISTANT_ABOLISH", "助教取消记录", "抽取助教取消/作废记录", "助教", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_SETTLEMENT_RECORDS", "结算记录", "抽取订单结算记录", "结算", "ODS", is_ods=True),
|
||
# CHANGE [2026-07-20] intent: 同步 ETL 侧移除——ODS_SETTLEMENT_TICKET 已在 Task 7.3 中彻底移除
|
||
TaskDefinition("ODS_TABLE_USE", "台费流水", "抽取台费使用流水", "台桌", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_TABLE_FEE_DISCOUNT", "台费折扣", "抽取台费折扣记录", "台桌", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_TABLES", "台桌主数据", "抽取门店台桌信息", "台桌", "ODS", is_ods=True, requires_window=False),
|
||
TaskDefinition("ODS_PAYMENT", "支付流水", "抽取支付交易记录", "结算", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_REFUND", "退款流水", "抽取退款交易记录", "结算", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_PLATFORM_COUPON", "平台券核销", "抽取平台优惠券核销记录", "团购", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_MEMBER", "会员主数据", "抽取会员档案", "会员", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_MEMBER_CARD", "会员储值卡", "抽取会员储值卡信息", "会员", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_MEMBER_BALANCE", "会员余额变动", "抽取会员余额变动记录", "会员", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_RECHARGE_SETTLE", "充值结算", "抽取充值结算记录", "会员", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_GROUP_PACKAGE", "团购套餐", "抽取团购套餐定义", "团购", "ODS", is_ods=True, requires_window=False),
|
||
TaskDefinition("ODS_GROUP_BUY_REDEMPTION", "团购核销", "抽取团购核销记录", "团购", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_INVENTORY_STOCK", "库存快照", "抽取商品库存汇总", "库存", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_INVENTORY_CHANGE", "库存变动", "抽取库存出入库记录", "库存", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_GOODS_CATEGORY", "商品分类", "抽取商品分类树", "商品", "ODS", is_ods=True, requires_window=False),
|
||
TaskDefinition("ODS_STORE_GOODS", "门店商品", "抽取门店商品主数据", "商品", "ODS", is_ods=True, requires_window=False),
|
||
TaskDefinition("ODS_STORE_GOODS_SALES", "商品销售", "抽取门店商品销售记录", "商品", "ODS", is_ods=True),
|
||
TaskDefinition("ODS_TENANT_GOODS", "租户商品", "抽取租户级商品主数据", "商品", "ODS", is_ods=True, requires_window=False),
|
||
]
|
||
|
||
# ── DWD 任务定义 ──────────────────────────────────────────────
|
||
|
||
DWD_TASKS: list[TaskDefinition] = [
|
||
TaskDefinition("DWD_LOAD_FROM_ODS", "DWD 装载", "从 ODS 装载至 DWD（维度 SCD2 + 事实增量）", "通用", "DWD", requires_window=False),
|
||
TaskDefinition("DWD_QUALITY_CHECK", "DWD 质量检查", "对 DWD 层数据执行质量校验", "通用", "DWD", requires_window=False, is_common=False),
|
||
]
|
||
|
||
# ── DWS 任务定义 ──────────────────────────────────────────────
|
||
|
||
DWS_TASKS: list[TaskDefinition] = [
|
||
TaskDefinition("DWS_BUILD_ORDER_SUMMARY", "订单汇总构建", "构建订单汇总宽表", "结算", "DWS"),
|
||
TaskDefinition("DWS_ASSISTANT_DAILY", "助教日报", "汇总助教每日业绩", "助教", "DWS"),
|
||
TaskDefinition("DWS_ASSISTANT_MONTHLY", "助教月报", "汇总助教月度业绩", "助教", "DWS"),
|
||
TaskDefinition("DWS_ASSISTANT_CUSTOMER", "助教客户分析", "汇总助教-客户关系", "助教", "DWS"),
|
||
TaskDefinition("DWS_ASSISTANT_SALARY", "助教工资计算", "计算助教工资", "助教", "DWS"),
|
||
TaskDefinition("DWS_ASSISTANT_FINANCE", "助教财务汇总", "汇总助教财务数据", "助教", "DWS"),
|
||
TaskDefinition("DWS_MEMBER_CONSUMPTION", "会员消费分析", "汇总会员消费数据", "会员", "DWS"),
|
||
TaskDefinition("DWS_MEMBER_VISIT", "会员到店分析", "汇总会员到店频次", "会员", "DWS"),
|
||
TaskDefinition("DWS_FINANCE_DAILY", "财务日报", "汇总每日财务数据", "财务", "DWS"),
|
||
TaskDefinition("DWS_FINANCE_RECHARGE", "充值汇总", "汇总充值数据", "财务", "DWS"),
|
||
TaskDefinition("DWS_FINANCE_INCOME_STRUCTURE", "收入结构", "分析收入结构", "财务", "DWS"),
|
||
TaskDefinition("DWS_FINANCE_DISCOUNT_DETAIL", "折扣明细", "汇总折扣明细", "财务", "DWS"),
|
||
# CHANGE [2026-02-19] intent: 同步 ETL 侧合并——原 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_* 已合并为 DWS_MAINTENANCE
|
||
TaskDefinition("DWS_MAINTENANCE", "DWS 维护", "刷新物化视图 + 清理过期留存数据", "通用", "DWS", requires_window=False, is_common=False),
|
||
# CHANGE [2026-07-20] intent: 注册 DWS 库存汇总任务（日/周/月），依赖 DWD goods_stock_summary 加载完成（需求 12.9）
|
||
TaskDefinition("DWS_GOODS_STOCK_DAILY", "库存日报", "按日粒度汇总商品库存数据", "库存", "DWS"),
|
||
TaskDefinition("DWS_GOODS_STOCK_WEEKLY", "库存周报", "按周粒度汇总商品库存数据", "库存", "DWS"),
|
||
TaskDefinition("DWS_GOODS_STOCK_MONTHLY", "库存月报", "按月粒度汇总商品库存数据", "库存", "DWS"),
|
||
]
|
||
|
||
# ── INDEX 任务定义 ────────────────────────────────────────────
|
||
|
||
INDEX_TASKS: list[TaskDefinition] = [
|
||
TaskDefinition("DWS_WINBACK_INDEX", "回流指数 (WBI)", "计算会员回流指数", "指数", "INDEX"),
|
||
TaskDefinition("DWS_NEWCONV_INDEX", "新客转化指数 (NCI)", "计算新客转化指数", "指数", "INDEX"),
|
||
TaskDefinition("DWS_ML_MANUAL_IMPORT", "手动导入 (ML)", "手动导入机器学习数据", "指数", "INDEX", requires_window=False, is_common=False),
|
||
# CHANGE [2026-02-19] intent: 补充说明 RelationIndexTask 产出 RS/OS/MS/ML 四个子指数
|
||
TaskDefinition("DWS_RELATION_INDEX", "关系指数 (RS)", "产出 RS/OS/MS/ML 四个子指数", "指数", "INDEX"),
|
||
]
|
||
|
||
# ── 工具类任务定义 ────────────────────────────────────────────
|
||
|
||
UTILITY_TASKS: list[TaskDefinition] = [
|
||
TaskDefinition("MANUAL_INGEST", "手动导入", "从本地 JSON 文件手动导入数据", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("INIT_ODS_SCHEMA", "初始化 ODS Schema", "创建 ODS 层表结构", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("INIT_DWD_SCHEMA", "初始化 DWD Schema", "创建 DWD 层表结构", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("INIT_DWS_SCHEMA", "初始化 DWS Schema", "创建 DWS 层表结构", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("ODS_JSON_ARCHIVE", "ODS JSON 归档", "归档 ODS 原始 JSON 文件", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("CHECK_CUTOFF", "游标检查", "检查各任务数据游标截止点", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("SEED_DWS_CONFIG", "DWS 配置种子", "初始化 DWS 配置数据", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
TaskDefinition("DATA_INTEGRITY_CHECK", "数据完整性校验", "校验跨层数据完整性", "工具", "UTILITY", requires_window=False, is_common=False),
|
||
]
|
||
|
||
# ── 全量任务列表 ──────────────────────────────────────────────
|
||
|
||
ALL_TASKS: list[TaskDefinition] = ODS_TASKS + DWD_TASKS + DWS_TASKS + INDEX_TASKS + UTILITY_TASKS
|
||
|
||
# 按 code 索引，便于快速查找
|
||
_TASK_BY_CODE: dict[str, TaskDefinition] = {t.code: t for t in ALL_TASKS}
|
||
|
||
|
||
def get_all_tasks() -> list[TaskDefinition]:
|
||
return ALL_TASKS
|
||
|
||
|
||
def get_task_by_code(code: str) -> TaskDefinition | None:
|
||
return _TASK_BY_CODE.get(code.upper())
|
||
|
||
|
||
def get_tasks_grouped_by_domain() -> dict[str, list[TaskDefinition]]:
|
||
"""按业务域分组返回任务列表"""
|
||
groups: dict[str, list[TaskDefinition]] = {}
|
||
for t in ALL_TASKS:
|
||
groups.setdefault(t.domain, []).append(t)
|
||
return groups
|
||
|
||
|
||
def get_tasks_by_layer(layer: str) -> list[TaskDefinition]:
|
||
"""获取指定层的所有任务"""
|
||
layer_upper = layer.upper()
|
||
return [t for t in ALL_TASKS if t.layer == layer_upper]
|
||
|
||
|
||
# ── Flow → 层映射 ────────────────────────────────────────────
|
||
# 每种 Flow 包含的层，用于前端按 Flow 过滤可选任务
|
||
|
||
FLOW_LAYER_MAP: dict[str, list[str]] = {
|
||
"api_ods": ["ODS"],
|
||
"api_ods_dwd": ["ODS", "DWD"],
|
||
"api_full": ["ODS", "DWD", "DWS", "INDEX"],
|
||
"ods_dwd": ["DWD"],
|
||
"dwd_dws": ["DWS"],
|
||
"dwd_dws_index": ["DWS", "INDEX"],
|
||
"dwd_index": ["INDEX"],
|
||
}
|
||
|
||
|
||
def get_compatible_tasks(flow_id: str) -> list[TaskDefinition]:
|
||
"""根据 Flow 包含的层，返回兼容的任务列表"""
|
||
layers = FLOW_LAYER_MAP.get(flow_id, [])
|
||
return [t for t in ALL_TASKS if t.layer in layers]
|
||
|
||
|
||
# ── DWD 表定义 ────────────────────────────────────────────────
|
||
|
||
DWD_TABLES: list[DwdTableDefinition] = [
|
||
# 维度表
|
||
DwdTableDefinition("dwd.dim_site", "门店维度", "台桌", "ods.table_fee_transactions", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_site_ex", "门店维度（扩展）", "台桌", "ods.table_fee_transactions", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_table", "台桌维度", "台桌", "ods.site_tables_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_table_ex", "台桌维度（扩展）", "台桌", "ods.site_tables_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_assistant", "助教维度", "助教", "ods.assistant_accounts_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_assistant_ex", "助教维度（扩展）", "助教", "ods.assistant_accounts_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_member", "会员维度", "会员", "ods.member_profiles", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_member_ex", "会员维度（扩展）", "会员", "ods.member_profiles", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_member_card_account", "会员储值卡维度", "会员", "ods.member_stored_value_cards", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_member_card_account_ex", "会员储值卡维度（扩展）", "会员", "ods.member_stored_value_cards", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_tenant_goods", "租户商品维度", "商品", "ods.tenant_goods_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_tenant_goods_ex", "租户商品维度（扩展）", "商品", "ods.tenant_goods_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_store_goods", "门店商品维度", "商品", "ods.store_goods_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_store_goods_ex", "门店商品维度（扩展）", "商品", "ods.store_goods_master", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_goods_category", "商品分类维度", "商品", "ods.stock_goods_category_tree", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_groupbuy_package", "团购套餐维度", "团购", "ods.group_buy_packages", is_dimension=True),
|
||
DwdTableDefinition("dwd.dim_groupbuy_package_ex", "团购套餐维度（扩展）", "团购", "ods.group_buy_packages", is_dimension=True),
|
||
# 事实表
|
||
DwdTableDefinition("dwd.dwd_settlement_head", "结算主表", "结算", "ods.settlement_records"),
|
||
DwdTableDefinition("dwd.dwd_settlement_head_ex", "结算主表（扩展）", "结算", "ods.settlement_records"),
|
||
DwdTableDefinition("dwd.dwd_table_fee_log", "台费流水", "台桌", "ods.table_fee_transactions"),
|
||
DwdTableDefinition("dwd.dwd_table_fee_log_ex", "台费流水（扩展）", "台桌", "ods.table_fee_transactions"),
|
||
DwdTableDefinition("dwd.dwd_table_fee_adjust", "台费折扣", "台桌", "ods.table_fee_discount_records"),
|
||
DwdTableDefinition("dwd.dwd_table_fee_adjust_ex", "台费折扣（扩展）", "台桌", "ods.table_fee_discount_records"),
|
||
DwdTableDefinition("dwd.dwd_store_goods_sale", "商品销售", "商品", "ods.store_goods_sales_records"),
|
||
DwdTableDefinition("dwd.dwd_store_goods_sale_ex", "商品销售（扩展）", "商品", "ods.store_goods_sales_records"),
|
||
DwdTableDefinition("dwd.dwd_assistant_service_log", "助教服务流水", "助教", "ods.assistant_service_records"),
|
||
DwdTableDefinition("dwd.dwd_assistant_service_log_ex", "助教服务流水（扩展）", "助教", "ods.assistant_service_records"),
|
||
DwdTableDefinition("dwd.dwd_assistant_trash_event", "助教取消事件", "助教", "ods.assistant_cancellation_records"),
|
||
DwdTableDefinition("dwd.dwd_assistant_trash_event_ex", "助教取消事件（扩展）", "助教", "ods.assistant_cancellation_records"),
|
||
DwdTableDefinition("dwd.dwd_member_balance_change", "会员余额变动", "会员", "ods.member_balance_changes"),
|
||
DwdTableDefinition("dwd.dwd_member_balance_change_ex", "会员余额变动（扩展）", "会员", "ods.member_balance_changes"),
|
||
DwdTableDefinition("dwd.dwd_groupbuy_redemption", "团购核销", "团购", "ods.group_buy_redemption_records"),
|
||
DwdTableDefinition("dwd.dwd_groupbuy_redemption_ex", "团购核销（扩展）", "团购", "ods.group_buy_redemption_records"),
|
||
DwdTableDefinition("dwd.dwd_platform_coupon_redemption", "平台券核销", "团购", "ods.platform_coupon_redemption_records"),
|
||
DwdTableDefinition("dwd.dwd_platform_coupon_redemption_ex", "平台券核销（扩展）", "团购", "ods.platform_coupon_redemption_records"),
|
||
DwdTableDefinition("dwd.dwd_recharge_order", "充值订单", "会员", "ods.recharge_settlements"),
|
||
DwdTableDefinition("dwd.dwd_recharge_order_ex", "充值订单（扩展）", "会员", "ods.recharge_settlements"),
|
||
DwdTableDefinition("dwd.dwd_payment", "支付流水", "结算", "ods.payment_transactions"),
|
||
DwdTableDefinition("dwd.dwd_refund", "退款流水", "结算", "ods.refund_transactions"),
|
||
DwdTableDefinition("dwd.dwd_refund_ex", "退款流水（扩展）", "结算", "ods.refund_transactions"),
|
||
# CHANGE [2026-07-20] intent: 同步 Task 6.1/6.2 新建的 DWD 库存表
|
||
DwdTableDefinition("dwd.dwd_goods_stock_summary", "库存汇总", "库存", "ods.goods_stock_summary"),
|
||
DwdTableDefinition("dwd.dwd_goods_stock_movement", "库存变动", "库存", "ods.goods_stock_movements"),
|
||
]
|
||
|
||
|
||
def get_dwd_tables_grouped_by_domain() -> dict[str, list[DwdTableDefinition]]:
|
||
"""按业务域分组返回 DWD 表定义"""
|
||
groups: dict[str, list[DwdTableDefinition]] = {}
|
||
for t in DWD_TABLES:
|
||
groups.setdefault(t.domain, []).append(t)
|
||
return groups
|