开发机迁移

This commit is contained in:
Neo
2026-04-10 06:24:13 +08:00
parent f65c1d038b
commit 79d3c2e97e
50 changed files with 1565 additions and 318 deletions

View File

@@ -31,6 +31,31 @@ from app.config import (
ETL_DB_USER,
)
# TCP keepalive 参数:防止长期运行的后台服务连接被网络设备/PostgreSQL 回收
_KEEPALIVE_KWARGS = {
"keepalives": 1,
"keepalives_idle": 60, # 空闲 60 秒后开始探测
"keepalives_interval": 10, # 每 10 秒探测一次
"keepalives_count": 3, # 连续 3 次失败判定断开
}
# 连接重试参数:应对 PostgreSQL 瞬时不可用Tailscale 网络抖动等)
_CONNECT_MAX_RETRIES = 3
_CONNECT_RETRY_DELAY = 1.0 # 秒
def _connect_with_retry(connect_fn, max_retries=_CONNECT_MAX_RETRIES):
"""带重试的数据库连接,应对服务端瞬时拒绝连接。"""
last_exc = None
for attempt in range(max_retries):
try:
return connect_fn()
except psycopg2.OperationalError as e:
last_exc = e
if attempt < max_retries - 1:
time.sleep(_CONNECT_RETRY_DELAY * (attempt + 1))
raise last_exc
def get_connection() -> PgConnection:
"""
@@ -49,13 +74,14 @@ def get_connection() -> PgConnection:
start = time.perf_counter() if should_trace else 0.0
conn = psycopg2.connect(
conn = _connect_with_retry(lambda: psycopg2.connect(
host=DB_HOST,
port=DB_PORT,
user=DB_USER,
password=DB_PASSWORD,
dbname=APP_DB_NAME,
)
**_KEEPALIVE_KWARGS,
))
if should_trace:
from datetime import datetime
@@ -86,13 +112,14 @@ def get_etl_global_readonly_connection() -> PgConnection:
用于系统管理后台等不需要门店隔离的场景(如 ETL 状态监控)。
"""
conn = psycopg2.connect(
conn = _connect_with_retry(lambda: psycopg2.connect(
host=ETL_DB_HOST,
port=ETL_DB_PORT,
user=ETL_DB_USER,
password=ETL_DB_PASSWORD,
dbname=ETL_DB_NAME,
)
**_KEEPALIVE_KWARGS,
))
try:
conn.autocommit = False
with conn.cursor() as cur:
@@ -121,13 +148,14 @@ def get_etl_readonly_connection(site_id: int | str) -> PgConnection:
finally:
conn.close()
"""
conn = psycopg2.connect(
conn = _connect_with_retry(lambda: psycopg2.connect(
host=ETL_DB_HOST,
port=ETL_DB_PORT,
user=ETL_DB_USER,
password=ETL_DB_PASSWORD,
dbname=ETL_DB_NAME,
)
**_KEEPALIVE_KWARGS,
))
try:
conn.autocommit = False
with conn.cursor() as cur:
@@ -142,3 +170,23 @@ def get_etl_readonly_connection(site_id: int | str) -> PgConnection:
conn.close()
raise
return conn
def get_etl_write_connection() -> PgConnection:
"""
获取 ETL 数据库etl_feiqiu的可写连接。
仅用于后端需要回写 ETL 汇总表的场景(如 task_generator 回写关系指数统计)。
不设置 RLS 隔离,调用方需在 SQL 中显式指定 site_id。
调用方负责关闭连接。
"""
conn = _connect_with_retry(lambda: psycopg2.connect(
host=ETL_DB_HOST,
port=ETL_DB_PORT,
user=ETL_DB_USER,
password=ETL_DB_PASSWORD,
dbname=ETL_DB_NAME,
**_KEEPALIVE_KWARGS,
))
conn.autocommit = False
return conn

View File

@@ -39,6 +39,61 @@ class RunJobByNameResponse(BaseModel):
job_name: str
class EtlCompletedRequest(BaseModel):
"""ETL 完成通知请求体。"""
pipeline: str = Field(default="api_full", description="完成的 pipeline 名称")
site_id: int | None = Field(default=None, description="门店 ID可选")
class EtlCompletedResponse(BaseModel):
"""ETL 完成编排结果。"""
success: bool
recall_result: dict | None = None
task_gen_result: dict | None = None
message: str = ""
@router.post("/etl-completed", response_model=EtlCompletedResponse)
async def etl_completed_endpoint(
body: EtlCompletedRequest,
_token: str = Depends(verify_internal_token),
) -> EtlCompletedResponse:
"""ETL pipeline 完成后的统一编排入口。
CHANGE 2026-04-07 | Fix-12ETL 完成后自动触发。
编排顺序recall_detector.run() → task_generator.run()
"""
from app.services import recall_detector, task_generator
recall_result = None
task_gen_result = None
errors = []
# Step 1: 先检查召回完成(含回溯)
try:
recall_result = recall_detector.run()
logger.info("ETL 编排 Step1 recall_detector 完成: %s", recall_result)
except Exception:
logger.exception("ETL 编排 Step1 recall_detector 失败")
errors.append("recall_detector failed")
# Step 2: 再生成新任务
try:
task_gen_result = task_generator.run()
logger.info("ETL 编排 Step2 task_generator 完成: %s", task_gen_result)
except Exception:
logger.exception("ETL 编排 Step2 task_generator 失败")
errors.append("task_generator failed")
success = len(errors) == 0
return EtlCompletedResponse(
success=success,
recall_result=recall_result,
task_gen_result=task_gen_result,
message="; ".join(errors) if errors else "ok",
)
@router.post("/run-job", response_model=RunJobByNameResponse)
async def run_job_by_name_endpoint(
body: RunJobByNameRequest,

View File

@@ -12,6 +12,7 @@
- POST /api/xcx/tasks/{id}/restore — 恢复任务
所有端点均需 JWTapproved 状态)。
回访任务通过提交备注自动完成note_service不提供手动完成接口。
"""
from __future__ import annotations
@@ -20,6 +21,7 @@ from fastapi import APIRouter, Depends, Query
from app.auth.dependencies import CurrentUser
from app.middleware.permission import require_approved, require_permission
from app.schemas.xcx_tasks import (
AbandonRequest,
TaskDetailResponse,

View File

@@ -375,9 +375,9 @@ async def get_coach_board(
"task_total": task_recall + task_callback,
})
# 7. 排序
# 7. 排序id 作 tiebreaker 保证分页稳定)
sort_key, sort_desc = _SORT_KEY_MAP.get(sort, ("perf_hours", True))
items.sort(key=lambda x: x.get(sort_key, 0), reverse=sort_desc)
items.sort(key=lambda x: (x.get(sort_key, 0), x.get("id", 0)), reverse=sort_desc)
# 移除内部排序字段
for item in items:
@@ -406,7 +406,9 @@ def _query_coach_tasks(
"""
查询助教任务完成数BOARD-1 task 维度)。
来源: biz.coach_tasks按 task_type 分类统计 recall/callback。
CHANGE 2026-04-08 | Fix-13 改造:
- recall: 广义召回数(从 biz.recall_events 统计,按天去重,不重复叠加)
- callback: 回访完成数(从 biz.coach_tasks 统计status='completed'
"""
if not assistant_ids:
return {}
@@ -414,26 +416,42 @@ def _query_coach_tasks(
result: dict[int, dict] = {}
try:
with conn.cursor() as cur:
# 广义召回数:从 recall_events 统计(天然去重)
cur.execute(
"""
SELECT assistant_id,
COUNT(*) FILTER (WHERE task_type IN ('high_priority_recall', 'priority_recall')) AS recall_count,
COUNT(*) FILTER (WHERE task_type = 'relationship_building') AS callback_count
FROM biz.coach_tasks
SELECT assistant_id, COUNT(*) AS recall_count
FROM biz.recall_events
WHERE assistant_id = ANY(%s)
AND site_id = %s
AND completed_at >= %s::date
AND completed_at <= %s::date
AND status = 'completed'
AND pay_time >= %s::date
AND pay_time < (%s::date + INTERVAL '1 day')
GROUP BY assistant_id
""",
(assistant_ids, site_id, start_date, end_date),
)
for row in cur.fetchall():
result[row[0]] = {
"recall": row[1] or 0,
"callback": row[2] or 0,
}
result.setdefault(row[0], {"recall": 0, "callback": 0})
result[row[0]]["recall"] = row[1] or 0
# 回访完成数:从 coach_tasks 统计
cur.execute(
"""
SELECT assistant_id, COUNT(*) AS callback_count
FROM biz.coach_tasks
WHERE assistant_id = ANY(%s)
AND site_id = %s
AND completed_at >= %s::date
AND completed_at < (%s::date + INTERVAL '1 day')::timestamptz
AND status = 'completed'
AND task_type = 'follow_up_visit'
GROUP BY assistant_id
""",
(assistant_ids, site_id, start_date, end_date),
)
for row in cur.fetchall():
result.setdefault(row[0], {"recall": 0, "callback": 0})
result[row[0]]["callback"] = row[1] or 0
conn.commit()
except Exception:
logger.warning("BOARD-1 任务查询失败,降级为空", exc_info=True)
@@ -762,11 +780,16 @@ def _build_overview(
) -> dict:
"""经营一览板块。
CHANGE 2026-04-01 | board-finance-dws-area-refactor 9.1 |
改为从 dws_finance_area_daily 按 area_code 查询(通过 get_finance_overview_area
CHANGE 2026-04-08 | P0 修复发生额失真 |
area=all 时回退到 dws_finance_daily_summaryget_finance_overview
仅 area≠all 时走 dws_finance_area_dailyget_finance_overview_area
原因area_daily 的 all 行只聚合有桌台映射的结算单,漏算无桌台单据约 12%
"""
try:
data = fdw_queries.get_finance_overview_area(conn, site_id, start, end, area)
if area == "all":
data = fdw_queries.get_finance_overview(conn, site_id, start, end)
else:
data = fdw_queries.get_finance_overview_area(conn, site_id, start, end, area)
except Exception:
logger.warning("overview 查询失败,降级为空", exc_info=True)
return _empty_overview()
@@ -775,7 +798,10 @@ def _build_overview(
if compare == 1 and prev_start and prev_end:
try:
prev = fdw_queries.get_finance_overview_area(conn, site_id, prev_start, prev_end, area)
if area == "all":
prev = fdw_queries.get_finance_overview(conn, site_id, prev_start, prev_end)
else:
prev = fdw_queries.get_finance_overview_area(conn, site_id, prev_start, prev_end, area)
_attach_compare(result, data, prev, [
"occurrence", "discount", "discount_rate", "confirmed_revenue",
"cash_in", "cash_out", "cash_balance", "balance_rate",

View File

@@ -937,18 +937,18 @@ def get_consumption_60d(
"""
查询客户近 60 天消费金额。
来源: app.v_dwd_assistant_service_log
⚠️ DWD-DOC 规则 1: 使用 ledger_amountitems_sum 口径)
⚠️ 废单排除: is_delete = 0
来源: app.v_dws_member_consumption_summaryDWS 预聚合表)
与 board-customer spend60 维度统一口径items_sum60天窗口日粒度
取最新 stat_date 的快照行
"""
with _fdw_context(conn, site_id, etl_conn=etl_conn) as cur:
cur.execute(
"""
SELECT COALESCE(SUM(ledger_amount), 0)
FROM app.v_dwd_assistant_service_log
WHERE tenant_member_id = %s
AND is_delete = 0
AND create_time >= (CURRENT_DATE - INTERVAL '60 days')::timestamptz
SELECT consume_amount_60d
FROM app.v_dws_member_consumption_summary
WHERE member_id = %s
ORDER BY stat_date DESC
LIMIT 1
""",
(member_id,),
)
@@ -1729,6 +1729,8 @@ def get_coach_sv_data(
result: dict[int, dict] = {}
with _fdw_context(conn, site_id) as cur:
# CHANGE 2026-04-07 | Fix-6sv_consume 改为从结算表按 start_date/end_date 过滤,
# 使其随时间筛选联动,而非固定 60 天窗口。
cur.execute(
"""
WITH coach_members AS (
@@ -1737,11 +1739,21 @@ def get_coach_sv_data(
FROM app.v_dws_member_assistant_relation_index ri
WHERE ri.assistant_id = ANY(%s)
AND ri.session_count > 0
),
period_consume AS (
SELECT sh.member_id,
COALESCE(SUM(sh.items_sum), 0) AS consume_amount
FROM app.v_dwd_settlement_head sh
WHERE sh.member_id = ANY(SELECT member_id FROM coach_members)
AND sh.settle_type IN (1, 3)
AND sh.pay_time::date >= %s::date
AND sh.pay_time::date <= %s::date
GROUP BY sh.member_id
)
SELECT cm.assistant_id,
COALESCE(SUM(ca_agg.balance), 0) AS sv_amount,
COUNT(DISTINCT CASE WHEN ca_agg.balance > 0 THEN cm.member_id END) AS sv_customer_count,
COALESCE(SUM(cs.consume_amount_60d), 0) AS sv_consume
COALESCE(SUM(pc.consume_amount), 0) AS sv_consume
FROM coach_members cm
LEFT JOIN (
SELECT tenant_member_id, SUM(balance) AS balance
@@ -1749,11 +1761,11 @@ def get_coach_sv_data(
WHERE scd2_is_current = 1
GROUP BY tenant_member_id
) ca_agg ON cm.member_id = ca_agg.tenant_member_id
LEFT JOIN app.v_dws_member_consumption_summary cs
ON cm.member_id = cs.member_id
LEFT JOIN period_consume pc
ON cm.member_id = pc.member_id
GROUP BY cm.assistant_id
""",
(assistant_ids,),
(assistant_ids, start_date, end_date),
)
for row in cur.fetchall():
result[row[0]] = {
@@ -1769,19 +1781,20 @@ def get_coach_sv_data(
# ---------------------------------------------------------------------------
def _project_filter_clause(project: str) -> tuple[str, tuple]:
def _project_filter_clause(project: str, member_col: str = "member_id") -> tuple[str, tuple]:
"""
生成项目筛选 SQL 片段(用于 BOARD-2 会员维度查询)。
CHANGE 2026-03-20 | R3 修复project 参数直接接收 category_code
BILLIARD/SNOOKER/MAHJONG/KTV/ALL去掉 chinese→BILLIARD 映射层。
CHANGE 2026-04-07 | Fix-1member_col 参数化,修复 6 个维度别名不匹配导致 SQL 500。
返回 (sql_fragment, params)sql_fragment 以 AND 开头,可直接拼入 WHERE 子句。
"""
_valid_categories = {"BILLIARD", "SNOOKER", "MAHJONG", "KTV"}
if project == "ALL" or project not in _valid_categories:
return "", ()
clause = """
AND vd.member_id IN (
clause = f"""
AND {member_col} IN (
SELECT mpt.member_id
FROM app.v_dws_member_project_tag mpt
WHERE mpt.category_code = %s AND mpt.is_tagged = true
@@ -1802,13 +1815,13 @@ def get_customer_board_recall(
ideal_days → ideal_interval_days, wbi_score → display_score,
elapsed_days → CURRENT_DATE - last_visit_time::date (计算列),
overdue_days → elapsed_days - ideal_interval_days (计算列),
visits_30d 不存在(有 visits_14d/visits_60d),用 visits_14d 近似,
CHANGE 2026-04-07 | Fix-3visits_30d 新增字段,替代 visits_14d 近似,
balance_amount → balance (v_dim_member_card_account)
⚠️ DQ-6: 客户姓名通过 member_id JOIN v_dim_member。
⚠️ DQ-7: 余额通过 JOIN v_dim_member_card_account。
按 display_score 降序LIMIT/OFFSET 分页。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "wi.member_id")
with _fdw_context(conn, site_id) as cur:
# 总数
@@ -1817,6 +1830,7 @@ def get_customer_board_recall(
SELECT COUNT(*)
FROM app.v_dws_member_winback_index wi
WHERE 1=1 {proj_clause}
""",
proj_params,
)
@@ -1831,7 +1845,7 @@ def get_customer_board_recall(
wi.ideal_interval_days,
CURRENT_DATE - wi.last_visit_time::date AS elapsed_days,
(CURRENT_DATE - wi.last_visit_time::date) - COALESCE(wi.ideal_interval_days, 0) AS overdue_days,
wi.visits_14d,
wi.visits_30d,
wi.display_score,
COALESCE(ca.balance, 0) AS balance
FROM app.v_dws_member_winback_index wi
@@ -1844,7 +1858,7 @@ def get_customer_board_recall(
GROUP BY tenant_member_id
) ca ON wi.member_id = ca.tenant_member_id
WHERE 1=1 {proj_clause}
ORDER BY wi.display_score DESC
ORDER BY wi.display_score DESC, wi.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -1870,16 +1884,18 @@ def _derive_potential_tags(
level_score: float | None,
speed_score: float | None,
stability_score: float | None,
) -> list[str]:
"""从三维分数派生潜力标签display_score 为 0-100 区间)。"""
tags = []
threshold = 60.0
) -> list[dict]:
"""从三维分数派生潜力标签display_score 为 0-10 区间)。
返回 [{text, theme}] 格式,与前端 potentialTags 类型一致。
"""
tags: list[dict] = []
threshold = 6.0
if level_score is not None and float(level_score) >= threshold:
tags.append("high_level")
tags.append({"text": "高消费力", "theme": "success"})
if speed_score is not None and float(speed_score) >= threshold:
tags.append("fast_growth")
tags.append({"text": "快增长", "theme": "warning"})
if stability_score is not None and float(stability_score) >= threshold:
tags.append("stable")
tags.append({"text": "稳定", "theme": "primary"})
return tags
@@ -1934,7 +1950,7 @@ def get_customer_board_potential(
) ca_agg ON spi.member_id = ca_agg.tenant_member_id
WHERE 1=1
{f"AND spi.member_id IN (SELECT member_id FROM app.v_dws_member_project_tag WHERE category_code = %s AND is_tagged = true)" if proj_params else ""}
ORDER BY spi.display_score DESC NULLS LAST, COALESCE(ca_agg.balance, 0) DESC
ORDER BY spi.display_score DESC NULLS LAST, COALESCE(ca_agg.balance, 0) DESC, spi.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -1969,7 +1985,7 @@ def get_customer_board_balance(
⚠️ DQ-7: 余额通过 tenant_member_id JOIN取 scd2_is_current=1。
按 balance 降序。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "ca.tenant_member_id")
with _fdw_context(conn, site_id) as cur:
# CHANGE 2026-03-28 | 修复客户重复dim_member_card_account 同一 member 有多条记录,
@@ -2009,9 +2025,14 @@ def get_customer_board_balance(
) ca_agg
LEFT JOIN app.v_dim_member dm
ON ca_agg.member_id = dm.member_id AND dm.scd2_is_current = 1
LEFT JOIN app.v_dws_member_consumption_summary vd
ON ca_agg.member_id = vd.member_id
ORDER BY ca_agg.balance DESC
LEFT JOIN LATERAL (
SELECT cs.days_since_last, cs.consume_amount_60d
FROM app.v_dws_member_consumption_summary cs
WHERE cs.member_id = ca_agg.member_id
ORDER BY cs.stat_date DESC
LIMIT 1
) vd ON true
ORDER BY ca_agg.balance DESC, ca_agg.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -2026,9 +2047,10 @@ def get_customer_board_balance(
"last_visit": f"{row[3]}天前" if row[3] is not None else "--",
"last_visit_date": row[3],
"ideal_days": None, # balance 维度无 ideal_days由 board_service 补充
"monthly_consume": float(row[4]) if row[4] is not None else 0.0,
# CHANGE 2026-04-07 | Fix-4consume_amount_60d 是 60 天总额,月均 = /2
"monthly_consume": float(row[4]) / 2 if row[4] is not None else 0.0,
"available_months": (
f"{float(row[2]) / float(row[4]):.1f}个月"
f"{2 * float(row[2]) / float(row[4]):.1f}个月"
if row[2] and row[4] and float(row[4]) > 0
else "--"
),
@@ -2050,7 +2072,7 @@ def get_customer_board_recharge(
balance_amount → balance (v_dim_member_card_account)
按 last_recharge_date (MAX(pay_time::date)) 降序。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "ro.member_id")
with _fdw_context(conn, site_id) as cur:
cur.execute(
@@ -2084,11 +2106,16 @@ def get_customer_board_recharge(
WHERE scd2_is_current = 1
GROUP BY tenant_member_id
) ca_agg ON ro.member_id = ca_agg.tenant_member_id
LEFT JOIN app.v_dws_member_consumption_summary cs
ON ro.member_id = cs.member_id
LEFT JOIN LATERAL (
SELECT cs2.days_since_last
FROM app.v_dws_member_consumption_summary cs2
WHERE cs2.member_id = ro.member_id
ORDER BY cs2.stat_date DESC
LIMIT 1
) cs ON true
WHERE 1=1 {proj_clause}
GROUP BY ro.member_id, dm.nickname, ca_agg.balance, cs.days_since_last
ORDER BY MAX(ro.pay_time::date) DESC
ORDER BY MAX(ro.pay_time::date) DESC, ro.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -2121,7 +2148,7 @@ def get_customer_board_recent(
不再硬编码为 0。来源: v_dws_member_visit_detail + v_dim_member + v_dws_member_winback_index。
按 last_visit_date 降序。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "vd.member_id")
with _fdw_context(conn, site_id) as cur:
cur.execute(
@@ -2161,7 +2188,7 @@ def get_customer_board_recent(
ON ma.member_id = dm.member_id AND dm.scd2_is_current = 1
LEFT JOIN app.v_dws_member_winback_index wi
ON ma.member_id = wi.member_id
ORDER BY ma.last_visit_date DESC
ORDER BY ma.last_visit_date DESC, ma.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -2199,15 +2226,21 @@ def get_customer_board_spend60(
high_spend_tag/avg_spend 不存在,用 avg_ticket_amount 替代 avg_spend
high_spend_tag 通过阈值计算。
按 consume_amount_60d 降序。
CHANGE 2026-04-08 | Fixconsumption_summary 按 stat_date 有多行快照,
用 DISTINCT ON 取最新快照避免同一客户出现多次。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "cs.member_id")
with _fdw_context(conn, site_id) as cur:
cur.execute(
f"""
SELECT COUNT(*)
FROM app.v_dws_member_consumption_summary cs
WHERE 1=1 {proj_clause}
FROM (
SELECT DISTINCT ON (cs.member_id) cs.member_id
FROM app.v_dws_member_consumption_summary cs
WHERE 1=1 {proj_clause}
ORDER BY cs.member_id, cs.stat_date DESC
) sub
""",
proj_params,
)
@@ -2216,16 +2249,23 @@ def get_customer_board_spend60(
offset = (page - 1) * page_size
cur.execute(
f"""
WITH latest_cs AS (
SELECT DISTINCT ON (cs.member_id)
cs.member_id, cs.consume_amount_60d,
cs.visit_count_60d, cs.avg_ticket_amount
FROM app.v_dws_member_consumption_summary cs
WHERE 1=1 {proj_clause}
ORDER BY cs.member_id, cs.stat_date DESC
)
SELECT cs.member_id,
dm.nickname,
cs.consume_amount_60d,
cs.visit_count_60d,
cs.avg_ticket_amount
FROM app.v_dws_member_consumption_summary cs
FROM latest_cs cs
LEFT JOIN app.v_dim_member dm
ON cs.member_id = dm.member_id AND dm.scd2_is_current = 1
WHERE 1=1 {proj_clause}
ORDER BY cs.consume_amount_60d DESC
ORDER BY cs.consume_amount_60d DESC, cs.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -2256,15 +2296,20 @@ def get_customer_board_freq60(
CHANGE 2026-03-20 | 修正列名items_sum_60d → consume_amount_60d,
avg_interval_days 不存在,用 60/visit_count_60d 近似计算。
按 visit_count_60d 降序。
CHANGE 2026-04-08 | Fix同 spend60DISTINCT ON 取最新快照。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "cs.member_id")
with _fdw_context(conn, site_id) as cur:
cur.execute(
f"""
SELECT COUNT(*)
FROM app.v_dws_member_consumption_summary cs
WHERE 1=1 {proj_clause}
FROM (
SELECT DISTINCT ON (cs.member_id) cs.member_id
FROM app.v_dws_member_consumption_summary cs
WHERE 1=1 {proj_clause}
ORDER BY cs.member_id, cs.stat_date DESC
) sub
""",
proj_params,
)
@@ -2273,15 +2318,21 @@ def get_customer_board_freq60(
offset = (page - 1) * page_size
cur.execute(
f"""
WITH latest_cs AS (
SELECT DISTINCT ON (cs.member_id)
cs.member_id, cs.visit_count_60d, cs.consume_amount_60d
FROM app.v_dws_member_consumption_summary cs
WHERE 1=1 {proj_clause}
ORDER BY cs.member_id, cs.stat_date DESC
)
SELECT cs.member_id,
dm.nickname,
cs.visit_count_60d,
cs.consume_amount_60d
FROM app.v_dws_member_consumption_summary cs
FROM latest_cs cs
LEFT JOIN app.v_dim_member dm
ON cs.member_id = dm.member_id AND dm.scd2_is_current = 1
WHERE 1=1 {proj_clause}
ORDER BY cs.visit_count_60d DESC
ORDER BY cs.visit_count_60d DESC, cs.member_id
LIMIT %s OFFSET %s
""",
(*proj_params, page_size, offset),
@@ -2316,20 +2367,21 @@ def _get_weekly_visits_batch(cur: Any, member_ids: list[int]) -> dict[int, list[
"""
批量查询客户最近 8 周的到店次数(用于 freq60 维度柱状图)。
来源: app.v_dwd_assistant_service_log,按 ISO 周分组。
CHANGE 2026-04-07 | Fix-5数据源从 v_dwd_assistant_service_log 改为
v_dwd_settlement_headsettle_type IN (1,3)),与汇总维度口径一致。
返回 {member_id: [{val: int, pct: int}, ...]},固定 8 个元素。
"""
cur.execute(
"""
WITH weekly AS (
SELECT tenant_member_id AS member_id,
DATE_TRUNC('week', create_time::date) AS week_start,
COUNT(*) AS cnt
FROM app.v_dwd_assistant_service_log
WHERE tenant_member_id = ANY(%s)
AND is_delete = 0
AND create_time >= CURRENT_DATE - INTERVAL '56 days'
GROUP BY tenant_member_id, DATE_TRUNC('week', create_time::date)
SELECT member_id,
DATE_TRUNC('week', pay_time::date) AS week_start,
COUNT(DISTINCT pay_time::date) AS cnt
FROM app.v_dwd_settlement_head
WHERE member_id = ANY(%s)
AND settle_type IN (1, 3)
AND pay_time >= CURRENT_DATE - INTERVAL '56 days'
GROUP BY member_id, DATE_TRUNC('week', pay_time::date)
)
SELECT member_id, week_start, cnt
FROM weekly
@@ -2379,7 +2431,7 @@ def get_customer_board_loyal(
来源: app.v_dws_member_assistant_relation_index。
按 max_rs最高亲密度降序。
"""
proj_clause, proj_params = _project_filter_clause(project)
proj_clause, proj_params = _project_filter_clause(project, "ri.member_id")
with _fdw_context(conn, site_id) as cur:
cur.execute(
@@ -2403,7 +2455,7 @@ def get_customer_board_loyal(
FROM app.v_dws_member_assistant_relation_index ri
WHERE 1=1 {proj_clause}
GROUP BY ri.member_id
ORDER BY MAX(ri.rs_display) DESC
ORDER BY MAX(ri.rs_display) DESC, ri.member_id
LIMIT %s OFFSET %s
)
SELECT mt.member_id,
@@ -2417,7 +2469,7 @@ def get_customer_board_loyal(
ON mt.member_id = dm.member_id AND dm.scd2_is_current = 1
LEFT JOIN app.v_dim_assistant da
ON mt.top_assistant_id = da.assistant_id AND da.scd2_is_current = 1
ORDER BY mt.max_rs DESC
ORDER BY mt.max_rs DESC, mt.member_id
""",
(*proj_params, page_size, offset),
)

View File

@@ -1,20 +1,22 @@
# AI_CHANGELOG
# - 2026-03-20 | Prompt: H2 FDW→直连ETL统一改造 | _process_site() 中 fdw_etl.v_dwd_assistant_service_log
# 改为直连 ETL 库查询 app.v_dwd_assistant_service_log。使用 fdw_queries._fdw_context()。
# -*- coding: utf-8 -*-
"""
召回完成检测器Recall Completion Detector
ETL 数据更新后,直连 ETL 库读取助教服务记录,
匹配活跃任务标记 completed记录 completed_at 和 completed_task_type 快照,
触发 recall_completed 事件通知备注回溯重分类器
ETL 数据更新后,扫描所有 MAIN 关系对的结算记录,
记录广义召回事件recall_events匹配活跃任务标记 completed
对所有到店的 MAIN 关联客户生成回访任务follow_up_visit
由 trigger_jobs 中的 recall_completion_check 配置驱动event: etl_data_updated
CHANGE 2026-04-08 | Fix-13 改造:
- 扫描范围从"有 active 任务的客户"扩大为"所有 os_label='MAIN' 的关联客户"
- 新增 recall_events 事件表记录广义召回(按天去重)
- 无 active 任务的客户到店也生成 follow_up_visit
"""
import json
import logging
from datetime import timedelta
from app.trace.decorators import trace_service
@@ -63,69 +65,37 @@ def run(payload: dict | None = None, job_id: int | None = None) -> dict:
"""
召回完成检测主流程。
1. 从 trigger_jobs 读取 last_run_at 作为增量过滤基准
2. 获取所有 distinct site_id从 active 任务中)
3. 对每个 site_idSET LOCAL app.current_site_id 后
通过 FDW 读取 v_dwd_assistant_service_log 中 service_time > last_run_at 的新增服务记录
4. 对每条服务记录,查找 biz.coach_tasks 中匹配的
(site_id, assistant_id, member_id) 且 status='active' 的任务
5. 将匹配任务标记为 completed
- status = 'completed'
- completed_at = 服务时间
- completed_task_type = 当前 task_type快照
6. 记录 coach_task_history
7. 触发 fire_event('recall_completed', {site_id, assistant_id, member_id, service_time})
参数:
payload: 事件载荷event 触发时由 trigger_scheduler 传入)
job_id: 触发器 job ID由 trigger_scheduler 传入),用于在最终事务中
更新 last_run_at保证 handler 数据变更与 last_run_at 原子提交
返回: {"completed_count": int}
CHANGE 2026-04-08 | Fix-13 改造:扫描所有 MAIN 关系对。
1. 从 biz.sites 获取所有活跃门店
2. 对每个 site_id通过 _fdw_context 扫描 MAIN 关系对的结算记录
3. 有结算 → 写 recall_events + 完成任务(如有)+ 生成回访
"""
completed_count = 0
event_count = 0
conn = _get_connection()
try:
# ── 1. 读取 last_run_at ──
# ── 1. 从业务库获取所有活跃门店 ──
with conn.cursor() as cur:
cur.execute(
"""
SELECT last_run_at
FROM biz.trigger_jobs
WHERE job_name = 'recall_completion_check'
"""
)
row = cur.fetchone()
last_run_at = row[0] if row else None
conn.commit()
# ── 2. 获取所有有 active 任务的 distinct site_id ──
with conn.cursor() as cur:
cur.execute(
"""
SELECT DISTINCT site_id
FROM biz.coach_tasks
WHERE status = 'active'
"""
"SELECT site_id FROM biz.sites WHERE is_active = true"
)
site_ids = [r[0] for r in cur.fetchall()]
conn.commit()
# ── 3. 逐 site_id 读取新增服务记录 ──
# ── 2. 逐 site_id 处理 ──
for site_id in site_ids:
try:
count = _process_site(conn, site_id, last_run_at)
completed_count += count
result = _process_site(conn, site_id)
completed_count += result["completed"]
event_count += result["events"]
except Exception:
logger.exception(
"处理门店召回检测失败: site_id=%s", site_id
)
conn.rollback()
# ── 事务安全T5handler 成功后更新 last_run_at ──
# job_id 由 trigger_scheduler 传入,在 handler 最终事务中更新
# handler 异常时此处不会执行异常向上传播last_run_at 不变
# ── 更新 last_run_at(兼容 trigger_scheduler 调度记录) ──
if job_id is not None:
from app.services.trigger_scheduler import update_job_last_run_at
@@ -137,145 +107,196 @@ def run(payload: dict | None = None, job_id: int | None = None) -> dict:
finally:
conn.close()
logger.info("召回完成检测完成: completed_count=%d", completed_count)
return {"completed_count": completed_count}
logger.info(
"召回完成检测完成: completed_count=%d, event_count=%d",
completed_count, event_count,
)
return {"completed_count": completed_count, "event_count": event_count}
def _process_site(conn, site_id: int, last_run_at) -> int:
def _process_site(conn, site_id: int) -> dict:
"""
处理单个门店的召回完成检测。
处理单个门店的广义召回检测。
直连 ETL 库读取新增服务记录,匹配 active 任务并标记 completed。
返回本门店完成的任务数。
CHANGE 2026-04-08 | Fix-13 改造:
1. 从 ETL 查询所有 os_label='MAIN' 的 (assistant_id, member_id) 对
2. 批量查询这些客户的最新结算记录
3. 对每个有新结算的关系对:写 recall_events + 完成任务 + 生成回访
"""
completed = 0
events = 0
# CHANGE 2026-03-20 | H2 FDW→直连ETL | fdw_etl.v_dwd_assistant_service_log → app.v_dwd_assistant_service_log
# intent: 修复 RLS 门店隔离失效postgres_fdw 不传递 GUC 参数)
# assumptions: _fdw_context 内部管理 ETL 连接conn 仅用于后续业务库操作
from app.services.fdw_queries import _fdw_context
# ── 1. 获取本门店所有 MAIN 关系对 ──
with _fdw_context(conn, site_id) as cur:
if last_run_at is not None:
# 列名映射: FDW 外部表 assistant_id/member_id/service_time
# → RLS 视图 site_assistant_id/tenant_member_id/create_time
cur.execute(
"""
SELECT DISTINCT site_assistant_id, tenant_member_id, create_time
FROM app.v_dwd_assistant_service_log
WHERE create_time > %s
ORDER BY create_time ASC
""",
(last_run_at,),
)
else:
# 首次运行,读取所有服务记录
cur.execute(
"""
SELECT DISTINCT site_assistant_id, tenant_member_id, create_time
FROM app.v_dwd_assistant_service_log
ORDER BY create_time ASC
"""
)
service_records = cur.fetchall()
cur.execute(
"""
SELECT assistant_id, member_id
FROM app.v_dws_member_assistant_relation_index
WHERE os_label = 'MAIN'
"""
)
main_pairs = [(r[0], r[1]) for r in cur.fetchall()]
# ── 4-7. 逐条服务记录匹配并处理 ──
for assistant_id, member_id, service_time in service_records:
# 散客过滤member_id ≤ 0 不参与任务系统)
if member_id is None or member_id <= 0:
continue
try:
count = _process_service_record(
conn, site_id, assistant_id, member_id, service_time
if not main_pairs:
return {"completed": 0, "events": 0}
# ── 2. 批量查询这些客户的最新结算时间 ──
member_ids = list({mid for _, mid in main_pairs})
settlement_map: dict[tuple[int, int], object] = {} # (assistant_id, member_id) → latest_pay_time
with _fdw_context(conn, site_id) as cur:
cur.execute(
"""
SELECT sl.site_assistant_id AS assistant_id,
sh.member_id,
MAX(sh.pay_time) AS latest_pay_time
FROM app.v_dwd_settlement_head sh
JOIN app.v_dwd_assistant_service_log sl
ON sl.order_settle_id = sh.order_settle_id
AND sl.is_delete = 0
WHERE sh.member_id = ANY(%s)
AND sh.settle_type IN (1, 3)
GROUP BY sl.site_assistant_id, sh.member_id
""",
(member_ids,),
)
for row in cur.fetchall():
settlement_map[(row[0], row[1])] = row[2]
# ── 3. 获取本门店所有 active 的召回/回访任务(用于匹配) ──
active_tasks_map: dict[tuple[int, int], list] = {} # (assistant_id, member_id) → [(id, task_type, created_at)]
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, assistant_id, member_id, task_type, created_at
FROM biz.coach_tasks
WHERE site_id = %s
AND status = 'active'
AND task_type IN ('high_priority_recall', 'priority_recall', 'follow_up_visit')
""",
(site_id,),
)
for row in cur.fetchall():
key = (row[1], row[2])
active_tasks_map.setdefault(key, []).append(
{"id": row[0], "task_type": row[3], "created_at": row[4]}
)
completed += count
conn.commit()
# ── 4. 逐关系对处理 ──
for assistant_id, member_id in main_pairs:
latest_pay = settlement_map.get((assistant_id, member_id))
if latest_pay is None:
continue
active_tasks = active_tasks_map.get((assistant_id, member_id), [])
try:
result = _process_pair(
conn, site_id, assistant_id, member_id,
latest_pay, active_tasks,
)
completed += result["completed"]
events += result["events"]
except Exception:
logger.exception(
"处理服务记录失败: site_id=%s, assistant_id=%s, member_id=%s",
site_id,
assistant_id,
member_id,
"处理关系对失败: site_id=%s, assistant_id=%s, member_id=%s",
site_id, assistant_id, member_id,
)
conn.rollback()
return completed
return {"completed": completed, "events": events}
def _process_service_record(
def _process_pair(
conn,
site_id: int,
assistant_id: int,
member_id: int,
service_time,
) -> int:
latest_pay_time,
active_tasks: list[dict],
) -> dict:
"""
处理单条服务记录:匹配 active 任务并标记 completed + 生成回访任务
处理单个 MAIN 关系对的召回检测
CHANGE 2026-03-30 | 回访任务直接在此生成(不再依赖 note_reclassifier 事件链)。
规则:
- 有 active 召回任务 → 标记 completed然后生成回访任务
- 有 active 回访任务 → 关闭旧回访,生成新回访(重置 48h 倒计时
- 无任何 active 召回/回访 → 直接生成回访任务
CHANGE 2026-04-08 | Fix-13 改造:
- 写 recall_eventsON CONFLICT DO NOTHING 按天去重)
- 有 active 召回任务且 pay_time > created_at → 完成任务
- 关闭旧回访 → 新建回访48h
- 无 active 任务也生成回访
每条服务记录独立事务,失败不影响其他。
返回本次完成的任务数。
返回: {"completed": int, "events": int}
"""
completed = 0
events = 0
with conn.cursor() as cur:
cur.execute("BEGIN")
# ── 1. 查找匹配的 active 召回类任务 ──
# ── 1. 写 recall_events按天去重 ──
# 先查是否有匹配的召回任务(用于填充 task_id/task_type
recall_tasks = [
t for t in active_tasks
if t["task_type"] in ("high_priority_recall", "priority_recall")
and latest_pay_time > t["created_at"]
]
event_task_id = recall_tasks[0]["id"] if recall_tasks else None
event_task_type = recall_tasks[0]["task_type"] if recall_tasks else None
cur.execute(
"""
SELECT id, task_type
FROM biz.coach_tasks
WHERE site_id = %s
AND assistant_id = %s
AND member_id = %s
AND status = 'active'
AND task_type IN ('high_priority_recall', 'priority_recall')
INSERT INTO biz.recall_events
(site_id, assistant_id, member_id, pay_time, task_id, task_type)
VALUES (%s, %s, %s, %s, %s, %s)
ON CONFLICT (site_id, assistant_id, member_id, (date_trunc('day', pay_time AT TIME ZONE 'Asia/Shanghai')))
DO NOTHING
RETURNING id
""",
(site_id, assistant_id, member_id),
(site_id, assistant_id, member_id, latest_pay_time,
event_task_id, event_task_type),
)
active_recall_tasks = cur.fetchall()
inserted = cur.fetchone()
if inserted is None:
# 今天已记录过,跳过后续处理(避免重复生成回访)
conn.commit()
return {"completed": 0, "events": 0}
has_active_recall = len(active_recall_tasks) > 0
events = 1
# 将所有匹配的 active 召回任务标记为 completed
for task_id, task_type in active_recall_tasks:
# ── 2. 完成匹配的召回任务 ──
has_active_recall = len(recall_tasks) > 0
for task in recall_tasks:
cur.execute(
"""
UPDATE biz.coach_tasks
SET status = 'completed',
completed_at = %s,
completed_task_type = %s,
completion_type = 'auto',
updated_at = NOW()
WHERE id = %s AND status = 'active'
""",
(service_time, task_type, task_id),
(latest_pay_time, task["task_type"], task["id"]),
)
_insert_history(
cur,
task_id,
task["id"],
action="completed",
old_status="active",
new_status="completed",
old_task_type=task_type,
new_task_type=task_type,
old_task_type=task["task_type"],
new_task_type=task["task_type"],
detail={
"service_time": str(service_time),
"completed_task_type": task_type,
"service_time": str(latest_pay_time),
"completed_task_type": task["task_type"],
},
)
completed += 1
# ── 2. 生成回访任务CHANGE 2026-03-30 ──
# 如果还有 active 召回任务(其他助教的),不生成回访
# 注意:上面已经把当前助教的召回任务标记为 completed 了
# 这里检查的是当前助教-客户对是否还有未完成的召回任务(不应该有了)
# 关闭已有的 active 回访任务
# ── 3. 关闭已有的 active 回访任务 ──
cur.execute(
"""
SELECT id FROM biz.coach_tasks
@@ -299,16 +320,19 @@ def _process_service_record(
action="superseded_by_new_visit",
old_status="active", new_status="inactive",
old_task_type="follow_up_visit", new_task_type="follow_up_visit",
detail={"reason": "new_service_record", "service_time": str(service_time)},
detail={"reason": "new_service_record", "service_time": str(latest_pay_time)},
)
# 创建新的回访任务48h 过期)
from datetime import timedelta
expires_at = service_time + timedelta(hours=48) if hasattr(service_time, '__add__') else None
# ── 4. 创建新的回访任务48h 过期) ──
expires_at = (
latest_pay_time + timedelta(hours=48)
if hasattr(latest_pay_time, '__add__') else None
)
cur.execute(
"""
INSERT INTO biz.coach_tasks
(site_id, assistant_id, member_id, task_type, status, expires_at, created_at, updated_at)
(site_id, assistant_id, member_id, task_type, status,
expires_at, created_at, updated_at)
VALUES (%s, %s, %s, 'follow_up_visit', 'active', %s, NOW(), NOW())
RETURNING id
""",
@@ -322,14 +346,14 @@ def _process_service_record(
new_task_type="follow_up_visit",
detail={
"reason": "service_record_detected",
"service_time": str(service_time),
"service_time": str(latest_pay_time),
"had_recall": has_active_recall,
},
)
conn.commit()
# ── 3. 触发 recall_completed 事件(仅当有召回任务被完成时) ──
# ── 5. 触发 recall_completed 事件(仅当有召回任务被完成时) ──
if has_active_recall:
try:
from app.services.trigger_scheduler import fire_event
@@ -339,7 +363,7 @@ def _process_service_record(
"site_id": site_id,
"assistant_id": assistant_id,
"member_id": member_id,
"service_time": str(service_time),
"service_time": str(latest_pay_time),
},
)
except Exception:
@@ -348,4 +372,4 @@ def _process_service_record(
site_id, assistant_id, member_id,
)
return completed
return {"completed": completed, "events": events}

View File

@@ -998,8 +998,8 @@ def _update_task_stats(conn, site_id: int) -> None:
# ── C: 历史总计(写入 ETL 关系指数表) ──
try:
from app.database import get_etl_readonly_connection
etl_conn = get_etl_readonly_connection(site_id)
from app.database import get_etl_write_connection
etl_conn = get_etl_write_connection()
try:
# 先从业务库聚合历史总计
with conn.cursor() as cur:

View File

@@ -95,6 +95,7 @@ DWS_TASKS: list[TaskDefinition] = [
TaskDefinition("DWS_FINANCE_RECHARGE", "充值汇总", "汇总充值数据", "财务", "DWS"),
TaskDefinition("DWS_FINANCE_INCOME_STRUCTURE", "收入结构", "分析收入结构", "财务", "DWS"),
TaskDefinition("DWS_FINANCE_DISCOUNT_DETAIL", "折扣明细", "汇总折扣明细", "财务", "DWS"),
TaskDefinition("DWS_FINANCE_AREA_DAILY", "区域财务日报", "按区域汇总每日财务数据", "财务", "DWS"),
# CHANGE [2026-02-19] intent: 同步 ETL 侧合并——原 DWS_RETENTION_CLEANUP / DWS_MV_REFRESH_* 已合并为 DWS_MAINTENANCE
TaskDefinition("DWS_MAINTENANCE", "DWS 维护", "刷新物化视图 + 清理过期留存数据", "通用", "DWS", requires_window=False, is_common=False),
# CHANGE [2026-03-27] intent: 注册 DWS 库存汇总任务(日/周/月)(需求 12.9