root/Neo-ZQYY
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: 累积功能变更 — 聊天集成、租户管理、小程序更新、ETL 增强、迁移脚本

Browse Source 
包含多个会话的累积代码变更：
- backend: AI 聊天服务、触发器调度、认证增强、WebSocket、调度器最小间隔
- admin-web: ETL 状态页、任务管理、调度配置、登录优化
- miniprogram: 看板页面、聊天集成、UI 组件、导航更新
- etl: DWS 新任务（finance_area_daily/board_cache）、连接器增强
- tenant-admin: 项目初始化
- db: 19 个迁移脚本（etl_feiqiu 11 + zqyy_app 8）
- packages/shared: 枚举和工具函数更新
- tools: 数据库工具、报表生成、健康检查
- docs: PRD/架构/部署/合约文档更新

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Neo
2026-04-06 00:03:48 +08:00
parent 70324d8542
commit 6f8f12314f
515 changed files with 76604 additions and 7456 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apps/etl/connectors/feiqiu/tasks/dws/index/base_index_task.py
View File

@@ -83,7 +83,7 @@ class BaseIndexTask(BaseDwsTask):
self._index_params_cache_by_type: Dict[str, IndexParameters] = {}
# 默认参数
DEFAULT_LOOKBACK_DAYS = 60
DEFAULT_LOOKBACK_DAYS = 90
DEFAULT_PERCENTILE_LOWER = 5
DEFAULT_PERCENTILE_UPPER = 95
DEFAULT_EWMA_ALPHA = 0.2

9
apps/etl/connectors/feiqiu/tasks/dws/index/member_index_base.py
View File

@@ -86,7 +86,9 @@ class MemberIndexBaseTask(BaseIndexTask):
tenant_id = self._get_tenant_id()
params = self._load_params()
activities = self._build_member_activity(site_id, tenant_id, params)
# P19: 回测模式用 as_of_date 替代 datetime.now()
as_of = (context.as_of_date if context and context.as_of_date else None) or datetime.now(self.tz)
activities = self._build_member_activity(site_id, tenant_id, params, as_of=as_of)
if not activities:
self.logger.warning("No member activity data available; skip calculation")
return {'status': 'skipped', 'reason': 'no_data'}
@@ -402,9 +404,12 @@ class MemberIndexBaseTask(BaseIndexTask):
site_id: int,
tenant_id: int,
params: Dict[str, float],
*,
as_of: Optional[datetime] = None,
) -> Dict[int, MemberActivityData]:
"""构建会员活动特征"""
now = datetime.now(self.tz)
# P19: 回测模式用 as_of 替代 datetime.now()
now = as_of or datetime.now(self.tz)
base_date = now.date()
visit_lookback_days = int(params.get('visit_lookback_days', self.DEFAULT_VISIT_LOOKBACK_DAYS))

38
apps/etl/connectors/feiqiu/tasks/dws/index/newconv_index_task.py
View File

@@ -202,7 +202,9 @@ class NewconvIndexTask(MemberIndexBaseTask):
avg_raw=sum(all_raw) / len(all_raw)
)
inserted = self._save_newconv_data(newconv_list)
# P19: 回测模式传入 calc_time
calc_time = (context.as_of_date if context and context.as_of_date else None)
inserted = self._save_newconv_data(newconv_list, calc_time=calc_time)
self.logger.info("NCI calculation finished, inserted %d rows", inserted)
return {
@@ -286,21 +288,30 @@ class NewconvIndexTask(MemberIndexBaseTask):
if data.raw_score < 0:
data.raw_score = 0.0
def _save_newconv_data(self, data_list: List[MemberNewconvData]) -> int:
def _save_newconv_data(self, data_list: List[MemberNewconvData], *, calc_time=None) -> int:
"""保存 NCI 数据"""
if not data_list:
return 0
site_id = data_list[0].activity.site_id
# 按门店全量刷新，避免因分群变化导致过期数据残留。
delete_sql = """
DELETE FROM dws.dws_member_newconv_index
WHERE site_id = %s
"""
# P19: 回测模式按 calc_time 删除（保留其他快照），正常模式按 site_id 全量刷新
use_param_time = calc_time is not None
with self.db.conn.cursor() as cur:
cur.execute(delete_sql, (site_id,))
if use_param_time:
cur.execute(
"DELETE FROM dws.dws_member_newconv_index WHERE site_id = %s AND calc_time = %s",
(site_id, calc_time),
)
else:
cur.execute(
"DELETE FROM dws.dws_member_newconv_index WHERE site_id = %s",
(site_id,),
)
insert_sql = """
# P19: 回测模式传入 calc_time，正常模式用 NOW()
use_param_time = calc_time is not None
time_placeholder = "%s, %s, %s" if use_param_time else "NOW(), NOW(), NOW()"
insert_sql = f"""
INSERT INTO dws.dws_member_newconv_index (
site_id, tenant_id, member_id,
status, segment,
@@ -328,7 +339,7 @@ class NewconvIndexTask(MemberIndexBaseTask):
%s, %s, %s,
%s, %s, %s,
%s,
NOW(), NOW(), NOW()
{time_placeholder}
)
"""
@@ -336,7 +347,7 @@ class NewconvIndexTask(MemberIndexBaseTask):
with self.db.conn.cursor() as cur:
for data in data_list:
activity = data.activity
cur.execute(insert_sql, (
params = (
activity.site_id, activity.tenant_id, activity.member_id,
data.status, data.segment,
activity.member_create_time, activity.first_visit_time, activity.last_visit_time, activity.last_recharge_time,
@@ -349,7 +360,10 @@ class NewconvIndexTask(MemberIndexBaseTask):
data.raw_score_welcome, data.raw_score_convert, data.raw_score,
data.display_score_welcome, data.display_score_convert, data.display_score,
None,
))
)
if use_param_time:
params = params + (calc_time, calc_time, calc_time)
cur.execute(insert_sql, params)
inserted += cur.rowcount
self.db.conn.commit()

191
apps/etl/connectors/feiqiu/tasks/dws/index/relation_index_task.py
View File

@@ -78,7 +78,7 @@ class RelationIndexTask(BaseIndexTask):
INDEX_TYPE = "RS"
DEFAULT_PARAMS_RS: Dict[str, float] = {
"lookback_days": 60,
"lookback_days": 90,
"session_merge_hours": 4,
"incentive_weight": 1.5,
"halflife_session": 14.0,
@@ -93,15 +93,13 @@ class RelationIndexTask(BaseIndexTask):
"ewma_alpha": 0.2,
}
DEFAULT_PARAMS_OS: Dict[str, float] = {
"min_rs_raw_for_ownership": 0.05,
"min_total_rs_raw": 0.10,
"ownership_main_threshold": 0.60,
"ownership_comanage_threshold": 0.35,
"ownership_gap_threshold": 0.15,
"min_rs_for_label": 0.10,
"ownership_main_ratio": 0.70,
"ownership_comanage_ratio": 0.30,
"eps": 1e-6,
}
DEFAULT_PARAMS_MS: Dict[str, float] = {
"lookback_days": 60,
"lookback_days": 90,
"session_merge_hours": 4,
"incentive_weight": 1.5,
"halflife_short": 7.0,
@@ -115,7 +113,7 @@ class RelationIndexTask(BaseIndexTask):
}
# CHANGE 2026-02-13 | intent: ML 仅使用人工台账，移除 source_mode / recharge_attribute_hours
DEFAULT_PARAMS_ML: Dict[str, float] = {
"lookback_days": 60,
"lookback_days": 90,
"amount_base": 500.0,
"halflife_recharge": 21.0,
"percentile_lower": 5.0,
@@ -143,7 +141,8 @@ class RelationIndexTask(BaseIndexTask):
site_id = self._get_site_id(context)
tenant_id = self._get_tenant_id()
now = datetime.now(self.tz)
# P19: 回测模式用 as_of_date 替代 datetime.now()
now = (context.as_of_date if context and context.as_of_date else None) or datetime.now(self.tz)
params_rs = self._load_params("RS", self.DEFAULT_PARAMS_RS)
params_os = self._load_params("OS", self.DEFAULT_PARAMS_OS)
@@ -151,8 +150,8 @@ class RelationIndexTask(BaseIndexTask):
params_ml = self._load_params("ML", self.DEFAULT_PARAMS_ML)
service_lookback_days = max(
int(params_rs.get("lookback_days", 60)),
int(params_ms.get("lookback_days", 60)),
int(params_rs.get("lookback_days", 90)),
int(params_ms.get("lookback_days", 90)),
)
service_start = now - timedelta(days=service_lookback_days)
merge_hours = max(
@@ -181,7 +180,9 @@ class RelationIndexTask(BaseIndexTask):
self._apply_display_scores(pair_map, params_rs, params_ms, params_ml, site_id)
inserted = self._save_relation_rows(site_id, list(pair_map.values()))
# P19: 仅回测模式传 calc_time（按 calc_time 删除保留其他快照），正常模式传 None（按 site_id 全量刷新）
backtest_calc_time = now if (context and context.as_of_date) else None
inserted = self._save_relation_rows(site_id, list(pair_map.values()), calc_time=backtest_calc_time)
self.logger.info("关系指数计算完成，写入 %d 条记录", inserted)
return {
@@ -313,7 +314,7 @@ class RelationIndexTask(BaseIndexTask):
params: Dict[str, float],
now: datetime,
) -> None:
lookback_days = int(params.get("lookback_days", 60))
lookback_days = int(params.get("lookback_days", 90))
halflife_session = float(params.get("halflife_session", 14.0))
halflife_last = float(params.get("halflife_last", 10.0))
weight_f = float(params.get("weight_f", 1.0))
@@ -355,7 +356,7 @@ class RelationIndexTask(BaseIndexTask):
params: Dict[str, float],
now: datetime,
) -> None:
lookback_days = int(params.get("lookback_days", 60))
lookback_days = int(params.get("lookback_days", 90))
halflife_short = float(params.get("halflife_short", 7.0))
halflife_long = float(params.get("halflife_long", 30.0))
eps = float(params.get("eps", 1e-6))
@@ -382,7 +383,7 @@ class RelationIndexTask(BaseIndexTask):
site_id: int,
now: datetime,
) -> None:
lookback_days = int(params.get("lookback_days", 60))
lookback_days = int(params.get("lookback_days", 90))
amount_base = float(params.get("amount_base", 500.0))
halflife_recharge = float(params.get("halflife_recharge", 21.0))
start_time = now - timedelta(days=lookback_days)
@@ -439,68 +440,53 @@ class RelationIndexTask(BaseIndexTask):
pair_map: Dict[Tuple[int, int], RelationPairMetrics],
params: Dict[str, float],
) -> None:
min_rs = float(params.get("min_rs_raw_for_ownership", 0.05))
min_total = float(params.get("min_total_rs_raw", 0.10))
main_threshold = float(params.get("ownership_main_threshold", 0.60))
comanage_threshold = float(params.get("ownership_comanage_threshold", 0.35))
gap_threshold = float(params.get("ownership_gap_threshold", 0.15))
"""CHANGE 2026-03-31 | 新 OS 方案：基于第一名分值比例。
规则：
- 第一名一定为 MAIN（rs_raw ≥ min_rs_for_label）
- 在第一名分值的 main_ratio_threshold 以内都为 MAIN
- 在第一名分值的 comanage_ratio_threshold 以内都为 COMANAGE
- 其余为 POOL
- MAIN/COMANAGE 前提：rs_raw ≥ min_rs_for_label
"""
min_rs_for_label = float(params.get("min_rs_for_label", 0.1))
main_ratio = float(params.get("ownership_main_ratio", 0.70))
comanage_ratio = float(params.get("ownership_comanage_ratio", 0.30))
member_groups: Dict[int, List[RelationPairMetrics]] = {}
for metrics in pair_map.values():
member_groups.setdefault(metrics.member_id, []).append(metrics)
for _, rows in member_groups.items():
eligible = [row for row in rows if row.rs_raw >= min_rs]
sum_rs = sum(row.rs_raw for row in eligible)
if sum_rs < min_total:
for row in rows:
row.os_share = 0.0
row.os_label = "UNASSIGNED"
row.os_rank = None
continue
for row in rows:
if row.rs_raw >= min_rs:
row.os_share = row.rs_raw / sum_rs
else:
row.os_share = 0.0
sorted_eligible = sorted(
eligible,
# 按 rs_raw 降序排列
sorted_rows = sorted(
rows,
key=lambda item: (
-item.os_share,
-item.rs_raw,
item.days_since_last_session if item.days_since_last_session is not None else 10**9,
item.assistant_id,
),
)
for idx, row in enumerate(sorted_eligible, start=1):
row.os_rank = idx
top1 = sorted_eligible[0]
top2_share = sorted_eligible[1].os_share if len(sorted_eligible) > 1 else 0.0
gap = top1.os_share - top2_share
has_main = top1.os_share >= main_threshold and gap >= gap_threshold
top_rs = sorted_rows[0].rs_raw if sorted_rows else 0.0
if has_main:
for row in rows:
if row is top1:
row.os_label = "MAIN"
elif row.os_share >= comanage_threshold:
row.os_label = "COMANAGE"
else:
row.os_label = "POOL"
else:
for row in rows:
if row.os_share >= comanage_threshold and row.rs_raw >= min_rs:
row.os_label = "COMANAGE"
else:
row.os_label = "POOL"
for idx, row in enumerate(sorted_rows):
row.os_rank = idx + 1
# 计算份额（保留兼容性，用于前端展示）
sum_rs = sum(r.rs_raw for r in rows if r.rs_raw > 0)
row.os_share = row.rs_raw / sum_rs if sum_rs > 0 else 0.0
# 非 eligible 不赋 rank
for row in rows:
if row.rs_raw < min_rs:
row.os_rank = None
if top_rs <= 0 or row.rs_raw < min_rs_for_label:
row.os_label = "POOL"
continue
ratio = row.rs_raw / top_rs
if ratio >= main_ratio or row is sorted_rows[0]:
row.os_label = "MAIN"
elif ratio >= comanage_ratio:
row.os_label = "COMANAGE"
else:
row.os_label = "POOL"
def _apply_display_scores(
self,
@@ -599,18 +585,27 @@ class RelationIndexTask(BaseIndexTask):
return "asinh"
return "none"
def _save_relation_rows(self, site_id: int, rows: List[RelationPairMetrics]) -> int:
def _save_relation_rows(self, site_id: int, rows: List[RelationPairMetrics], *, calc_time: Optional[datetime] = None) -> int:
# P19: 回测模式传入 calc_time，正常模式用 NOW()
use_param_time = calc_time is not None
with self.db.conn.cursor() as cur:
cur.execute(
"DELETE FROM dws.dws_member_assistant_relation_index WHERE site_id = %s",
(site_id,),
)
# P19: 回测模式按 calc_time 删除（保留其他快照），正常模式按 site_id 全量刷新
if use_param_time:
cur.execute(
"DELETE FROM dws.dws_member_assistant_relation_index WHERE site_id = %s AND calc_time = %s",
(site_id, calc_time),
)
else:
cur.execute(
"DELETE FROM dws.dws_member_assistant_relation_index WHERE site_id = %s",
(site_id,),
)
if not rows:
self.db.conn.commit()
return 0
insert_sql = """
insert_sql = f"""
INSERT INTO dws.dws_member_assistant_relation_index (
site_id, tenant_id, member_id, assistant_id,
session_count, total_duration_minutes, basic_session_count, incentive_session_count,
@@ -628,41 +623,41 @@ class RelationIndexTask(BaseIndexTask):
%s, %s, %s,
%s, %s, %s, %s,
%s, %s, %s, %s,
NOW(), NOW(), NOW()
{('%s, %s, %s' if use_param_time else 'NOW(), NOW(), NOW()')}
)
"""
inserted = 0
for row in rows:
cur.execute(
insert_sql,
(
row.site_id,
row.tenant_id,
row.member_id,
row.assistant_id,
row.session_count,
row.total_duration_minutes,
row.basic_session_count,
row.incentive_session_count,
row.days_since_last_session,
row.rs_f,
row.rs_d,
row.rs_r,
row.rs_raw,
row.rs_display,
row.os_share,
row.os_label,
row.os_rank,
row.ms_f_short,
row.ms_f_long,
row.ms_raw,
row.ms_display,
row.ml_order_count,
row.ml_allocated_amount,
row.ml_raw,
row.ml_display,
),
params = (
row.site_id,
row.tenant_id,
row.member_id,
row.assistant_id,
row.session_count,
row.total_duration_minutes,
row.basic_session_count,
row.incentive_session_count,
row.days_since_last_session,
row.rs_f,
row.rs_d,
row.rs_r,
row.rs_raw,
row.rs_display,
row.os_share,
row.os_label,
row.os_rank,
row.ms_f_short,
row.ms_f_long,
row.ms_raw,
row.ms_display,
row.ml_order_count,
row.ml_allocated_amount,
row.ml_raw,
row.ml_display,
)
if use_param_time:
params = params + (calc_time, calc_time, calc_time)
cur.execute(insert_sql, params)
inserted += max(cur.rowcount, 0)
self.db.conn.commit()
return inserted

80
apps/etl/connectors/feiqiu/tasks/dws/index/spending_power_index_task.py
View File

@@ -173,13 +173,17 @@ class SpendingPowerIndexTask(BaseIndexTask):
# 1. 获取 site_id
site_id = self._get_site_id(context)
# P19: 回测模式用 as_of_date 替代 NOW()
from datetime import datetime as _dt
as_of = (context.as_of_date if context and context.as_of_date else None) or _dt.now(self.tz)
# 2. 加载参数（配置表 + 默认值合并）
db_params = self.load_index_parameters('SPI')
params = {**self.DEFAULT_PARAMS, **db_params}
# 3. 提取特征
features = self._extract_spending_features(site_id, params)
recharge_map = self._extract_recharge_features(site_id, params)
features = self._extract_spending_features(site_id, params, as_of=as_of)
recharge_map = self._extract_recharge_features(site_id, params, as_of=as_of)
# 合并充值特征
for mid, recharge_90 in recharge_map.items():
@@ -189,7 +193,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
# 批量计算日消费 EWMA 并合并
member_ids = list(features.keys())
ewma_map = self._compute_daily_spend_ewma_batch(site_id, member_ids, params)
ewma_map = self._compute_daily_spend_ewma_batch(site_id, member_ids, params, as_of=as_of)
for mid, ewma_val in ewma_map.items():
if mid in features:
features[mid].daily_spend_ewma_90 = ewma_val
@@ -279,7 +283,9 @@ class SpendingPowerIndexTask(BaseIndexTask):
feat.score_stability_display = stability_display_map.get(mid, 0.0)
# 8. delete-before-insert 持久化（Req 9.3）
records_inserted = self._save_spi_data(feat_list, site_id)
# P19: 仅回测模式传 calc_time，正常模式传 None（按 site_id 全量刷新）
backtest_calc_time = as_of if (context and context.as_of_date) else None
records_inserted = self._save_spi_data(feat_list, site_id, calc_time=backtest_calc_time)
# 9. 保存分位点历史（Req 9.5）——SPI 总分
raw_values = [f.raw_score for f in feat_list]
@@ -323,7 +329,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
# =========================================================================
def _extract_spending_features(
self, site_id: int, params: Dict[str, float]
self, site_id: int, params: Dict[str, float], *, as_of=None
) -> Dict[int, SPIMemberFeatures]:
"""从 dwd_settlement_head 提取消费特征，按 member_id 聚合。
@@ -339,8 +345,8 @@ class SpendingPowerIndexTask(BaseIndexTask):
cutoff = self.config.get("app.business_day_start_hour", 8)
biz_expr = biz_date_sql_expr("pay_time", cutoff)
# 单条 SQL 同时聚合 30 天和 90 天窗口，避免两次扫描
# INTERVAL 天数通过 f-string 内嵌（整数，安全）；site_id 走参数化
# P19: 回测模式用 as_of 参数替代 NOW()
# 时间基准用 %s 参数化，正常模式传 NOW() 等效的 as_of
sql = f"""
WITH consume_source AS (
SELECT
@@ -356,7 +362,8 @@ class SpendingPowerIndexTask(BaseIndexTask):
AND COALESCE(mca.is_delete, 0) = 0
WHERE s.site_id = %s
AND s.settle_type IN (1, 3)
AND s.pay_time >= NOW() - INTERVAL '{long_days} days'
AND s.pay_time >= %s - INTERVAL '{long_days} days'
AND s.pay_time < %s
)
SELECT
canonical_member_id AS member_id,
@@ -367,17 +374,17 @@ class SpendingPowerIndexTask(BaseIndexTask):
COUNT(DISTINCT EXTRACT(ISOYEAR FROM pay_time)::int * 100
+ EXTRACT(WEEK FROM pay_time)::int) AS active_weeks_90,
-- 30 天窗口（子集过滤）
SUM(CASE WHEN pay_time >= NOW() - INTERVAL '{short_days} days'
SUM(CASE WHEN pay_time >= %s - INTERVAL '{short_days} days'
THEN pay_amount ELSE 0 END) AS spend_30,
SUM(CASE WHEN pay_time >= NOW() - INTERVAL '{short_days} days'
SUM(CASE WHEN pay_time >= %s - INTERVAL '{short_days} days'
THEN 1 ELSE 0 END) AS orders_30,
COUNT(DISTINCT CASE WHEN pay_time >= NOW() - INTERVAL '{short_days} days'
COUNT(DISTINCT CASE WHEN pay_time >= %s - INTERVAL '{short_days} days'
THEN {biz_expr} END) AS visit_days_30
FROM consume_source
WHERE canonical_member_id > 0
GROUP BY canonical_member_id
"""
rows = self.db.query(sql, (site_id,))
rows = self.db.query(sql, (site_id, as_of, as_of, as_of, as_of, as_of))
result: Dict[int, SPIMemberFeatures] = {}
for row in (rows or []):
@@ -412,7 +419,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
def _extract_recharge_features(
self, site_id: int, params: Dict[str, float]
self, site_id: int, params: Dict[str, float], *, as_of=None
) -> Dict[int, float]:
"""从 dwd_recharge_order 提取充值特征，返回 {member_id: recharge_90}。
@@ -421,6 +428,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
"""
long_days = int(params.get('spend_window_long_days', 90))
# P19: 回测模式用 as_of 参数替代 NOW()
sql = f"""
WITH recharge_source AS (
SELECT
@@ -435,7 +443,8 @@ class SpendingPowerIndexTask(BaseIndexTask):
AND COALESCE(mca.is_delete, 0) = 0
WHERE r.site_id = %s
AND r.settle_type = 5
AND r.pay_time >= NOW() - INTERVAL '{long_days} days'
AND r.pay_time >= %s - INTERVAL '{long_days} days'
AND r.pay_time < %s
)
SELECT
canonical_member_id AS member_id,
@@ -444,7 +453,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
WHERE canonical_member_id > 0
GROUP BY canonical_member_id
"""
rows = self.db.query(sql, (site_id,))
rows = self.db.query(sql, (site_id, as_of, as_of))
result: Dict[int, float] = {}
for row in (rows or []):
@@ -513,7 +522,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
return ewma
def _compute_daily_spend_ewma_batch(
self, site_id: int, member_ids: List[int], params: Dict[str, float]
self, site_id: int, member_ids: List[int], params: Dict[str, float], *, as_of=None
) -> Dict[int, float]:
"""批量计算多个会员的日消费 EWMA，单次 SQL 查询避免 N+1。
@@ -528,6 +537,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
cutoff = self.config.get("app.business_day_start_hour", 8)
biz_expr_s = biz_date_sql_expr("s.pay_time", cutoff)
# P19: 回测模式用 as_of 参数替代 NOW()
sql = f"""
WITH consume_source AS (
SELECT
@@ -543,7 +553,8 @@ class SpendingPowerIndexTask(BaseIndexTask):
AND COALESCE(mca.is_delete, 0) = 0
WHERE s.site_id = %s
AND s.settle_type IN (1, 3)
AND s.pay_time >= NOW() - INTERVAL '{long_days} days'
AND s.pay_time >= %s - INTERVAL '{long_days} days'
AND s.pay_time < %s
)
SELECT canonical_member_id AS member_id,
pay_date,
@@ -553,7 +564,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
GROUP BY canonical_member_id, pay_date
ORDER BY canonical_member_id, pay_date
"""
rows = self.db.query(sql, (site_id,))
rows = self.db.query(sql, (site_id, as_of, as_of))
# 按 member_id 分组，逐组计算 EWMA
result: Dict[int, float] = {}
@@ -727,21 +738,31 @@ class SpendingPowerIndexTask(BaseIndexTask):
# =========================================================================
def _save_spi_data(
self, data_list: List[SPIMemberFeatures], site_id: int
self, data_list: List[SPIMemberFeatures], site_id: int, *, calc_time=None
) -> int:
"""delete-before-insert 写入 dws_member_spending_power_index"""
with self.db.conn.cursor() as cur:
# 先删除该门店旧记录（Req 9.3）
cur.execute(
"DELETE FROM dws.dws_member_spending_power_index WHERE site_id = %s",
(site_id,),
)
# P19: 回测模式按 calc_time 删除（保留其他快照），正常模式按 site_id 全量刷新
use_param_time = calc_time is not None
if use_param_time:
cur.execute(
"DELETE FROM dws.dws_member_spending_power_index WHERE site_id = %s AND calc_time = %s",
(site_id, calc_time),
)
else:
cur.execute(
"DELETE FROM dws.dws_member_spending_power_index WHERE site_id = %s",
(site_id,),
)
if not data_list:
self.db.conn.commit()
return 0
insert_sql = """
# P19: 回测模式传入 calc_time，正常模式用 NOW()
use_param_time = calc_time is not None
time_placeholder = "%s, %s, %s" if use_param_time else "NOW(), NOW(), NOW()"
insert_sql = f"""
INSERT INTO dws.dws_member_spending_power_index (
site_id, member_id,
spend_30, spend_90, recharge_90,
@@ -761,7 +782,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
%s, %s, %s,
%s, %s, %s,
%s, %s,
NOW(), NOW(), NOW()
{time_placeholder}
)
"""
inserted = 0
@@ -773,7 +794,7 @@ class SpendingPowerIndexTask(BaseIndexTask):
return max(lo, min(hi, v))
for f in data_list:
cur.execute(insert_sql, (
params_tuple = (
f.site_id, f.member_id,
f.spend_30, f.spend_90, f.recharge_90,
f.orders_30, f.orders_90,
@@ -787,7 +808,10 @@ class SpendingPowerIndexTask(BaseIndexTask):
_clamp(f.score_stability_display, 0, DISP_MAX),
_clamp(f.raw_score, -RAW_MAX, RAW_MAX),
_clamp(f.display_score, 0, DISP_MAX),
))
)
if use_param_time:
params_tuple = params_tuple + (calc_time, calc_time, calc_time)
cur.execute(insert_sql, params_tuple)
inserted += max(cur.rowcount, 0)
self.db.conn.commit()

39
apps/etl/connectors/feiqiu/tasks/dws/index/winback_index_task.py
View File

@@ -7,7 +7,7 @@ from __future__ import annotations
import math
from dataclasses import dataclass
from datetime import date, timedelta
from datetime import date, datetime, timedelta
from typing import Any, Dict, List, Optional, Tuple
from .member_index_base import MemberActivityData, MemberIndexBaseTask
@@ -178,7 +178,9 @@ class WinbackIndexTask(MemberIndexBaseTask):
avg_raw=sum(all_raw) / len(all_raw)
)
inserted = self._save_winback_data(winback_list)
# P19: 回测模式传入 calc_time
calc_time = (context.as_of_date if context and context.as_of_date else None)
inserted = self._save_winback_data(winback_list, calc_time=calc_time)
self.logger.info("WBI calculation finished, inserted %d rows", inserted)
return {
@@ -339,21 +341,29 @@ class WinbackIndexTask(MemberIndexBaseTask):
if data.raw_score < 0:
data.raw_score = 0.0
def _save_winback_data(self, data_list: List[MemberWinbackData]) -> int:
def _save_winback_data(self, data_list: List[MemberWinbackData], *, calc_time: Optional[datetime] = None) -> int:
"""保存 WBI 数据"""
if not data_list:
return 0
site_id = data_list[0].activity.site_id
# 按门店全量刷新，避免因分群变化导致过期数据残留。
delete_sql = """
DELETE FROM dws.dws_member_winback_index
WHERE site_id = %s
"""
# P19: 回测模式传入 calc_time，正常模式用 NOW()
use_param_time = calc_time is not None
# P19: 回测模式按 calc_time 删除（保留其他快照），正常模式按 site_id 全量刷新
with self.db.conn.cursor() as cur:
cur.execute(delete_sql, (site_id,))
if use_param_time:
cur.execute(
"DELETE FROM dws.dws_member_winback_index WHERE site_id = %s AND calc_time = %s",
(site_id, calc_time),
)
else:
cur.execute(
"DELETE FROM dws.dws_member_winback_index WHERE site_id = %s",
(site_id,),
)
insert_sql = """
time_placeholder = "%s, %s, %s" if use_param_time else "NOW(), NOW(), NOW()"
insert_sql = f"""
INSERT INTO dws.dws_member_winback_index (
site_id, tenant_id, member_id,
status, segment,
@@ -379,7 +389,7 @@ class WinbackIndexTask(MemberIndexBaseTask):
%s, %s,
%s, %s,
%s,
NOW(), NOW(), NOW()
{time_placeholder}
)
"""
@@ -387,7 +397,7 @@ class WinbackIndexTask(MemberIndexBaseTask):
with self.db.conn.cursor() as cur:
for data in data_list:
activity = data.activity
cur.execute(insert_sql, (
params = (
activity.site_id, activity.tenant_id, activity.member_id,
data.status, data.segment,
activity.member_create_time, activity.first_visit_time, activity.last_visit_time, activity.last_recharge_time,
@@ -399,7 +409,10 @@ class WinbackIndexTask(MemberIndexBaseTask):
data.ideal_interval_days, data.ideal_next_visit_date,
data.raw_score, data.display_score,
None,
))
)
if use_param_time:
params = params + (calc_time, calc_time, calc_time)
cur.execute(insert_sql, params)
inserted += cur.rowcount
self.db.conn.commit()