初始提交：飞球 ETL 系统全量代码

2026-02-13 08:05:34 +08:00
commit 3c51f5485d
441 changed files with 117631 additions and 0 deletions
--- a/tasks/dws/finance_recharge_task.py
+++ b/tasks/dws/finance_recharge_task.py
@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+"""
+充值统计任务
+
+功能说明：
+    以"日期"为粒度，统计充值数据
+    
+数据来源：
+    - dwd_recharge_order: 充值订单
+    - dim_member_card_account: 会员卡账户（余额快照）
+
+目标表：
+    billiards_dws.dws_finance_recharge_summary
+
+更新策略：
+    - 更新频率：每日更新
+    - 幂等方式：delete-before-insert（按日期）
+    
+业务规则：
+    - 首充/续充：通过 is_first 字段区分
+    - 现金/赠送：通过 pay_money/gift_money 区分
+    - 卡余额：区分储值卡和赠送卡
+
+作者：ETL团队
+创建日期：2026-02-01
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from typing import Any, Dict, List, Optional, Tuple
+
+from .base_dws_task import BaseDwsTask, TaskContext
+
+
+class FinanceRechargeTask(BaseDwsTask):
+    """
+    充值统计任务
+    """
+    
+    def get_task_code(self) -> str:
+        return "DWS_FINANCE_RECHARGE"
+    
+    def get_target_table(self) -> str:
+        return "dws_finance_recharge_summary"
+    
+    def get_primary_keys(self) -> List[str]:
+        return ["site_id", "stat_date"]
+    
+    def extract(self, context: TaskContext) -> Dict[str, Any]:
+        start_date = context.window_start.date() if hasattr(context.window_start, 'date') else context.window_start
+        end_date = context.window_end.date() if hasattr(context.window_end, 'date') else context.window_end
+        site_id = context.store_id
+        
+        recharge_summary = self._extract_recharge_summary(site_id, start_date, end_date)
+        card_balances = self._extract_card_balances(site_id, end_date)
+        
+        return {
+            'recharge_summary': recharge_summary,
+            'card_balances': card_balances,
+            'start_date': start_date,
+            'end_date': end_date,
+            'site_id': site_id
+        }
+    
+    def transform(self, extracted: Dict[str, Any], context: TaskContext) -> List[Dict[str, Any]]:
+        recharge_summary = extracted['recharge_summary']
+        card_balances = extracted['card_balances']
+        site_id = extracted['site_id']
+        
+        results = []
+        for recharge in recharge_summary:
+            stat_date = recharge.get('stat_date')
+            
+            # 仅有当前快照时，统一写入（避免窗口内其他日期为0）
+            balance = card_balances
+            
+            record = {
+                'site_id': site_id,
+                'tenant_id': self.config.get("app.tenant_id", site_id),
+                'stat_date': stat_date,
+                'recharge_count': self.safe_int(recharge.get('recharge_count', 0)),
+                'recharge_total': self.safe_decimal(recharge.get('recharge_total', 0)),
+                'recharge_cash': self.safe_decimal(recharge.get('recharge_cash', 0)),
+                'recharge_gift': self.safe_decimal(recharge.get('recharge_gift', 0)),
+                'first_recharge_count': self.safe_int(recharge.get('first_recharge_count', 0)),
+                'first_recharge_cash': self.safe_decimal(recharge.get('first_recharge_cash', 0)),
+                'first_recharge_gift': self.safe_decimal(recharge.get('first_recharge_gift', 0)),
+                'first_recharge_total': self.safe_decimal(recharge.get('first_recharge_total', 0)),
+                'renewal_count': self.safe_int(recharge.get('renewal_count', 0)),
+                'renewal_cash': self.safe_decimal(recharge.get('renewal_cash', 0)),
+                'renewal_gift': self.safe_decimal(recharge.get('renewal_gift', 0)),
+                'renewal_total': self.safe_decimal(recharge.get('renewal_total', 0)),
+                'recharge_member_count': self.safe_int(recharge.get('recharge_member_count', 0)),
+                'new_member_count': self.safe_int(recharge.get('new_member_count', 0)),
+                'total_card_balance': self.safe_decimal(balance.get('total_balance', 0)),
+                'cash_card_balance': self.safe_decimal(balance.get('cash_balance', 0)),
+                'gift_card_balance': self.safe_decimal(balance.get('gift_balance', 0)),
+            }
+            results.append(record)
+        
+        return results
+    
+    def load(self, transformed: List[Dict[str, Any]], context: TaskContext) -> Dict:
+        if not transformed:
+            return {"counts": {"fetched": 0, "inserted": 0, "updated": 0, "skipped": 0, "errors": 0}}
+        
+        deleted = self.delete_existing_data(context, date_col="stat_date")
+        inserted = self.bulk_insert(transformed)
+        
+        return {
+            "counts": {"fetched": len(transformed), "inserted": inserted, "updated": 0, "skipped": 0, "errors": 0},
+            "extra": {"deleted": deleted}
+        }
+    
+    def _extract_recharge_summary(self, site_id: int, start_date: date, end_date: date) -> List[Dict[str, Any]]:
+        sql = """
+            SELECT 
+                DATE(pay_time) AS stat_date,
+                COUNT(*) AS recharge_count,
+                SUM(pay_money + gift_money) AS recharge_total,
+                SUM(pay_money) AS recharge_cash,
+                SUM(gift_money) AS recharge_gift,
+                COUNT(CASE WHEN is_first = 1 THEN 1 END) AS first_recharge_count,
+                SUM(CASE WHEN is_first = 1 THEN pay_money ELSE 0 END) AS first_recharge_cash,
+                SUM(CASE WHEN is_first = 1 THEN gift_money ELSE 0 END) AS first_recharge_gift,
+                SUM(CASE WHEN is_first = 1 THEN pay_money + gift_money ELSE 0 END) AS first_recharge_total,
+                COUNT(CASE WHEN is_first != 1 OR is_first IS NULL THEN 1 END) AS renewal_count,
+                SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN pay_money ELSE 0 END) AS renewal_cash,
+                SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN gift_money ELSE 0 END) AS renewal_gift,
+                SUM(CASE WHEN is_first != 1 OR is_first IS NULL THEN pay_money + gift_money ELSE 0 END) AS renewal_total,
+                COUNT(DISTINCT member_id) AS recharge_member_count,
+                COUNT(DISTINCT CASE WHEN is_first = 1 THEN member_id END) AS new_member_count
+            FROM billiards_dwd.dwd_recharge_order
+            WHERE site_id = %s AND DATE(pay_time) >= %s AND DATE(pay_time) <= %s
+            GROUP BY DATE(pay_time)
+        """
+        rows = self.db.query(sql, (site_id, start_date, end_date))
+        return [dict(row) for row in rows] if rows else []
+    
+    def _extract_card_balances(self, site_id: int, stat_date: date) -> Dict[str, Decimal]:
+        CASH_CARD_TYPE_ID = 2793249295533893
+        GIFT_CARD_TYPE_IDS = [2791990152417157, 2793266846533445, 2794699703437125]
+        
+        sql = """
+            SELECT card_type_id, SUM(balance) AS total_balance
+            FROM billiards_dwd.dim_member_card_account
+            WHERE site_id = %s AND scd2_is_current = 1
+              AND COALESCE(is_delete, 0) = 0
+            GROUP BY card_type_id
+        """
+        rows = self.db.query(sql, (site_id,))
+        
+        cash_balance = Decimal('0')
+        gift_balance = Decimal('0')
+        
+        for row in (rows or []):
+            card_type_id = row['card_type_id']
+            balance = self.safe_decimal(row['total_balance'])
+            if card_type_id == CASH_CARD_TYPE_ID:
+                cash_balance += balance
+            elif card_type_id in GIFT_CARD_TYPE_IDS:
+                gift_balance += balance
+        
+        return {
+            'cash_balance': cash_balance,
+            'gift_balance': gift_balance,
+            'total_balance': cash_balance + gift_balance
+        }
+
+
+__all__ = ['FinanceRechargeTask']