feat: P1-P3 全栈集成 — 数据库基础 + DWS 扩展 + 小程序鉴权 + 工程化体系
## P1 数据库基础 - zqyy_app: 创建 auth/biz schema、FDW 连接 etl_feiqiu - etl_feiqiu: 创建 app schema RLS 视图、商品库存预警表 - 清理 assistant_abolish 残留数据 ## P2 ETL/DWS 扩展 - 新增 DWS 助教订单贡献度表 (dws.assistant_order_contribution) - 新增 assistant_order_contribution_task 任务及 RLS 视图 - member_consumption 增加充值字段、assistant_daily 增加处罚字段 - 更新 ODS/DWD/DWS 任务文档及业务规则文档 - 更新 consistency_checker、flow_runner、task_registry 等核心模块 ## P3 小程序鉴权系统 - 新增 xcx_auth 路由/schema(微信登录 + JWT) - 新增 wechat/role/matching/application 服务层 - zqyy_app 鉴权表迁移 + 角色权限种子数据 - auth/dependencies.py 支持小程序 JWT 鉴权 ## 文档与审计 - 新增 DOCUMENTATION-MAP 文档导航 - 新增 7 份 BD_Manual 数据库变更文档 - 更新 DDL 基线快照(etl_feiqiu 6 schema + zqyy_app auth) - 新增全栈集成审计记录、部署检查清单更新 - 新增 BACKLOG 路线图、FDW→Core 迁移计划 ## Kiro 工程化 - 新增 5 个 Spec(P1/P2/P3/全栈集成/核心业务) - 新增审计自动化脚本(agent_on_stop/build_audit_context/compliance_prescan) - 新增 6 个 Hook(合规检查/会话日志/提交审计等) - 新增 doc-map steering 文件 ## 运维与测试 - 新增 ops 脚本:迁移验证/API 健康检查/ETL 监控/集成报告 - 新增属性测试:test_dws_contribution / test_auth_system - 清理过期 export 报告文件 - 更新 .gitignore 排除规则
This commit is contained in:
@@ -166,7 +166,11 @@ class FlowRunner:
|
||||
|
||||
timer.start_step("INCREMENT_ETL")
|
||||
if task_codes:
|
||||
results = self.task_executor.run_tasks(task_codes, data_source=data_source)
|
||||
# CHANGE [2026-02-24] intent: 对前端传入的 task_codes 也执行拓扑排序,
|
||||
# 避免 DWS 在 DWD 未完成时就开始计算(跨层依赖顺序缺失 bug)
|
||||
# prompt: "修复管理后台全选任务时不按层级顺序执行的问题"
|
||||
sorted_codes = topological_sort(task_codes, self.task_registry)
|
||||
results = self.task_executor.run_tasks(sorted_codes, data_source=data_source)
|
||||
else:
|
||||
auto_tasks = self._resolve_tasks(layers)
|
||||
results = self.task_executor.run_tasks(auto_tasks, data_source=data_source)
|
||||
|
||||
@@ -107,6 +107,11 @@ class TaskExecutor:
|
||||
results.append(result_entry)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.logger.error("任务 %s 失败: %s", task_code, exc, exc_info=True)
|
||||
# CHANGE 2026-02-24 | 任务失败后 rollback,防止 InFailedSqlTransaction 级联
|
||||
try:
|
||||
self.db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
results.append({
|
||||
"task_code": task_code,
|
||||
"status": "失败",
|
||||
|
||||
@@ -30,6 +30,7 @@ from tasks.utility.seed_dws_config_task import SeedDwsConfigTask
|
||||
# DWS 层任务导入
|
||||
from tasks.dws import (
|
||||
AssistantDailyTask,
|
||||
AssistantOrderContributionTask,
|
||||
AssistantMonthlyTask,
|
||||
AssistantCustomerTask,
|
||||
AssistantSalaryTask,
|
||||
@@ -147,6 +148,7 @@ default_registry.register("DATA_INTEGRITY_CHECK", DataIntegrityTask, requires_db
|
||||
# ── DWS 层业务任务 ────────────────────────────────────────────
|
||||
default_registry.register("DWS_BUILD_ORDER_SUMMARY", DwsBuildOrderSummaryTask, requires_db_config=False, layer="DWS")
|
||||
default_registry.register("DWS_ASSISTANT_DAILY", AssistantDailyTask, layer="DWS")
|
||||
default_registry.register("DWS_ASSISTANT_ORDER_CONTRIBUTION", AssistantOrderContributionTask, layer="DWS", depends_on=["DWD_LOAD_FROM_ODS"])
|
||||
# CHANGE [2026-07-17] intent: 为已知依赖关系添加 depends_on 声明(需求 8.1, 8.2)
|
||||
default_registry.register("DWS_ASSISTANT_MONTHLY", AssistantMonthlyTask, layer="DWS", depends_on=["DWS_ASSISTANT_DAILY"])
|
||||
default_registry.register("DWS_ASSISTANT_CUSTOMER", AssistantCustomerTask, layer="DWS")
|
||||
@@ -166,7 +168,8 @@ default_registry.register("DWS_GOODS_STOCK_MONTHLY", GoodsStockMonthlyTask, laye
|
||||
# 替换为统一维护任务 DWS_MAINTENANCE(需求 4.5)
|
||||
# depends_on: 所有其他 DWS 任务——MV 刷新和清理应在数据写入后执行
|
||||
default_registry.register("DWS_MAINTENANCE", DwsMaintenanceTask, layer="DWS", depends_on=[
|
||||
"DWS_ASSISTANT_DAILY", "DWS_ASSISTANT_MONTHLY", "DWS_ASSISTANT_CUSTOMER",
|
||||
"DWS_ASSISTANT_DAILY", "DWS_ASSISTANT_ORDER_CONTRIBUTION",
|
||||
"DWS_ASSISTANT_MONTHLY", "DWS_ASSISTANT_CUSTOMER",
|
||||
"DWS_ASSISTANT_SALARY", "DWS_ASSISTANT_FINANCE",
|
||||
"DWS_MEMBER_CONSUMPTION", "DWS_MEMBER_VISIT",
|
||||
"DWS_FINANCE_DAILY", "DWS_FINANCE_RECHARGE",
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
"""拓扑排序模块 — Kahn's algorithm
|
||||
|
||||
对任务列表按依赖关系执行拓扑排序:
|
||||
- 显式依赖:TaskMeta.depends_on 声明的任务间依赖
|
||||
- 隐含层级依赖:ODS → DWD → DWS → INDEX,同批任务中低层任务必须先于高层任务
|
||||
- 仅对当前执行列表内的任务排序
|
||||
- depends_on 中引用的任务不在列表内时记录警告
|
||||
- 检测循环依赖并抛出 ValueError
|
||||
@@ -11,10 +13,22 @@ import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 层级优先级:数值越小越先执行
|
||||
_LAYER_ORDER: dict[str, int] = {
|
||||
"ODS": 0,
|
||||
"DWD": 1,
|
||||
"DWS": 2,
|
||||
"INDEX": 3,
|
||||
}
|
||||
|
||||
|
||||
def topological_sort(task_codes: list[str], registry) -> list[str]:
|
||||
"""对任务列表执行拓扑排序(Kahn's algorithm)。
|
||||
|
||||
除了显式 depends_on 依赖外,还注入隐含的层级依赖:
|
||||
同批任务中,所有 ODS 任务排在 DWD 之前,DWD 排在 DWS 之前,
|
||||
DWS 排在 INDEX 之前。这确保跨层执行顺序正确。
|
||||
|
||||
Args:
|
||||
task_codes: 待排序的任务代码列表
|
||||
registry: TaskRegistry 实例,提供 get_metadata() 查询依赖
|
||||
@@ -29,9 +43,10 @@ def topological_sort(task_codes: list[str], registry) -> list[str]:
|
||||
return []
|
||||
|
||||
in_degree = {code: 0 for code in task_codes}
|
||||
graph = {code: [] for code in task_codes}
|
||||
graph: dict[str, list[str]] = {code: [] for code in task_codes}
|
||||
task_set = set(task_codes)
|
||||
|
||||
# 1. 显式依赖(depends_on)
|
||||
for code in task_codes:
|
||||
meta = registry.get_metadata(code)
|
||||
if meta and meta.depends_on:
|
||||
@@ -44,6 +59,31 @@ def topological_sort(task_codes: list[str], registry) -> list[str]:
|
||||
"任务 %s 依赖 %s,但后者不在当前执行列表中", code, dep
|
||||
)
|
||||
|
||||
# CHANGE [2026-02-24] intent: 注入隐含层级依赖,确保跨层执行顺序正确
|
||||
# assumptions: 层级顺序固定为 ODS→DWD→DWS→INDEX;同层任务无隐含互相依赖
|
||||
# prompt: "修复管理后台全选任务时不按层级顺序执行的问题"
|
||||
# 2. 隐含层级依赖:按层分组,相邻层之间建立边
|
||||
# 选择每层一个"代表节点"作为屏障,避免 O(n*m) 的全连接边
|
||||
layer_groups: dict[int, list[str]] = {}
|
||||
for code in task_codes:
|
||||
meta = registry.get_metadata(code)
|
||||
if meta and meta.layer:
|
||||
order = _LAYER_ORDER.get(meta.layer.upper())
|
||||
if order is not None:
|
||||
layer_groups.setdefault(order, []).append(code)
|
||||
|
||||
sorted_layers = sorted(layer_groups.keys())
|
||||
for i in range(len(sorted_layers) - 1):
|
||||
lower_layer = sorted_layers[i]
|
||||
higher_layer = sorted_layers[i + 1]
|
||||
# 高层的每个任务都依赖低层的所有任务
|
||||
for higher_code in layer_groups[higher_layer]:
|
||||
for lower_code in layer_groups[lower_layer]:
|
||||
# 避免重复添加已有的显式依赖边
|
||||
if higher_code not in graph[lower_code]:
|
||||
graph[lower_code].append(higher_code)
|
||||
in_degree[higher_code] += 1
|
||||
|
||||
queue = deque(code for code in task_codes if in_degree[code] == 0)
|
||||
result = []
|
||||
while queue:
|
||||
|
||||
Reference in New Issue
Block a user