feat: P1-P3 全栈集成 — 数据库基础 + DWS 扩展 + 小程序鉴权 + 工程化体系
## P1 数据库基础 - zqyy_app: 创建 auth/biz schema、FDW 连接 etl_feiqiu - etl_feiqiu: 创建 app schema RLS 视图、商品库存预警表 - 清理 assistant_abolish 残留数据 ## P2 ETL/DWS 扩展 - 新增 DWS 助教订单贡献度表 (dws.assistant_order_contribution) - 新增 assistant_order_contribution_task 任务及 RLS 视图 - member_consumption 增加充值字段、assistant_daily 增加处罚字段 - 更新 ODS/DWD/DWS 任务文档及业务规则文档 - 更新 consistency_checker、flow_runner、task_registry 等核心模块 ## P3 小程序鉴权系统 - 新增 xcx_auth 路由/schema(微信登录 + JWT) - 新增 wechat/role/matching/application 服务层 - zqyy_app 鉴权表迁移 + 角色权限种子数据 - auth/dependencies.py 支持小程序 JWT 鉴权 ## 文档与审计 - 新增 DOCUMENTATION-MAP 文档导航 - 新增 7 份 BD_Manual 数据库变更文档 - 更新 DDL 基线快照(etl_feiqiu 6 schema + zqyy_app auth) - 新增全栈集成审计记录、部署检查清单更新 - 新增 BACKLOG 路线图、FDW→Core 迁移计划 ## Kiro 工程化 - 新增 5 个 Spec(P1/P2/P3/全栈集成/核心业务) - 新增审计自动化脚本(agent_on_stop/build_audit_context/compliance_prescan) - 新增 6 个 Hook(合规检查/会话日志/提交审计等) - 新增 doc-map steering 文件 ## 运维与测试 - 新增 ops 脚本:迁移验证/API 健康检查/ETL 监控/集成报告 - 新增属性测试:test_dws_contribution / test_auth_system - 清理过期 export 报告文件 - 更新 .gitignore 排除规则
This commit is contained in:
@@ -161,6 +161,8 @@ class BaseOdsTask(BaseTask):
|
||||
segment_keys: set[tuple] = set()
|
||||
# CHANGE 2026-02-18 | 收集 WINDOW 模式下 API 返回数据的实际最早时间戳
|
||||
segment_earliest_time: datetime | None = None
|
||||
# CHANGE [2026-02-24] 收集 API 返回数据的实际最晚时间戳,用于 late-cutoff 保护
|
||||
segment_latest_time: datetime | None = None
|
||||
|
||||
self.logger.info(
|
||||
"%s: 开始执行(%s/%s),窗口[%s ~ %s]",
|
||||
@@ -197,6 +199,13 @@ class BaseOdsTask(BaseTask):
|
||||
if page_earliest is not None:
|
||||
if segment_earliest_time is None or page_earliest < segment_earliest_time:
|
||||
segment_earliest_time = page_earliest
|
||||
# CHANGE [2026-02-24] 收集实际最晚时间戳,用于 late-cutoff 保护
|
||||
page_latest = self._collect_latest_time(
|
||||
page_records, snapshot_time_column
|
||||
)
|
||||
if page_latest is not None:
|
||||
if segment_latest_time is None or page_latest > segment_latest_time:
|
||||
segment_latest_time = page_latest
|
||||
inserted, updated, skipped = self._insert_records_schema_aware(
|
||||
table=spec.table_name,
|
||||
records=page_records,
|
||||
@@ -229,13 +238,27 @@ class BaseOdsTask(BaseTask):
|
||||
spec.code, seg_start, segment_earliest_time,
|
||||
)
|
||||
effective_window_start = segment_earliest_time
|
||||
# CHANGE [2026-02-24] late-cutoff 保护:用 API 实际最晚时间戳收窄软删除范围
|
||||
# 防止 recent endpoint 数据保留期滚动导致窗口尾部数据消失时误标删除
|
||||
effective_window_end = seg_end
|
||||
if (
|
||||
snapshot_protect_early_cutoff
|
||||
and snapshot_mode == SnapshotMode.WINDOW
|
||||
and segment_latest_time is not None
|
||||
and segment_latest_time < seg_end
|
||||
):
|
||||
self.logger.info(
|
||||
"%s: late-cutoff 保护生效,软删除窗口终点从 %s 收窄至 %s",
|
||||
spec.code, seg_end, segment_latest_time,
|
||||
)
|
||||
effective_window_end = segment_latest_time
|
||||
deleted = self._mark_missing_as_deleted(
|
||||
table=spec.table_name,
|
||||
business_pk_cols=business_pk_cols,
|
||||
snapshot_mode=snapshot_mode,
|
||||
snapshot_time_column=snapshot_time_column,
|
||||
window_start=effective_window_start,
|
||||
window_end=seg_end,
|
||||
window_end=effective_window_end,
|
||||
key_values=segment_keys,
|
||||
allow_empty=snapshot_allow_empty,
|
||||
)
|
||||
@@ -548,7 +571,39 @@ class BaseOdsTask(BaseTask):
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
continue
|
||||
return earliest
|
||||
def _collect_latest_time(
|
||||
self, records: list, time_column: str
|
||||
) -> datetime | None:
|
||||
"""从一批 API 返回记录中提取 time_column 的最大值。
|
||||
|
||||
# CHANGE [2026-02-24] Prompt=诊断 2976396053006405 is_delete 误标
|
||||
# 用于 late-cutoff 保护:当 API recent endpoint 数据保留期滚动导致
|
||||
# 窗口尾部数据消失时,避免将尾部之后的数据误标为软删除。
|
||||
"""
|
||||
if not records or not time_column:
|
||||
return None
|
||||
latest: datetime | None = None
|
||||
for rec in records:
|
||||
if not isinstance(rec, dict):
|
||||
continue
|
||||
merged = self._merge_record_layers(rec)
|
||||
raw = self._get_value_case_insensitive(merged, time_column)
|
||||
if raw is None:
|
||||
continue
|
||||
try:
|
||||
if isinstance(raw, datetime):
|
||||
ts = raw
|
||||
elif isinstance(raw, str):
|
||||
ts = dtparser.parse(raw)
|
||||
else:
|
||||
continue
|
||||
if ts.tzinfo is None:
|
||||
ts = ts.replace(tzinfo=self.tz)
|
||||
if latest is None or ts > latest:
|
||||
latest = ts
|
||||
except (ValueError, TypeError, OverflowError):
|
||||
continue
|
||||
return latest
|
||||
|
||||
def _mark_missing_as_deleted(
|
||||
self,
|
||||
@@ -995,6 +1050,13 @@ class BaseOdsTask(BaseTask):
|
||||
updated += 1
|
||||
return inserted, updated
|
||||
|
||||
# goodsStockWarningInfo 嵌套字段 → ODS 扁平列名映射
|
||||
_STOCK_WARNING_FIELD_MAP: dict[str, str] = {
|
||||
"sales_day": "warning_sales_day",
|
||||
"warning_day_max": "warning_day_max",
|
||||
"warning_day_min": "warning_day_min",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _merge_record_layers(record: dict) -> dict:
|
||||
merged = record
|
||||
@@ -1005,6 +1067,13 @@ class BaseOdsTask(BaseTask):
|
||||
settle_inner = merged.get("settleList")
|
||||
if isinstance(settle_inner, dict):
|
||||
merged = {**settle_inner, **merged}
|
||||
# CHANGE 2026-02-24 | 扁平化 goodsStockWarningInfo 嵌套对象,
|
||||
# 将 sales_day/warning_day_max/warning_day_min 提升为顶层键
|
||||
warning_info = merged.get("goodsStockWarningInfo")
|
||||
if isinstance(warning_info, dict):
|
||||
for src_key, dst_key in BaseOdsTask._STOCK_WARNING_FIELD_MAP.items():
|
||||
if src_key in warning_info and dst_key not in merged:
|
||||
merged[dst_key] = warning_info[src_key]
|
||||
return merged
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user