微信小程序页面迁移校验之前 P5任务处理之前

apps/etl/connectors/feiqiu/config/defaults.py

@@ -4,6 +4,7 @@
 DEFAULTS = {
     "app": {
         "timezone": "Asia/Shanghai",
+        "business_day_start_hour": 8,
         "store_id": "",
         # CHANGE 2026-02-15 | 对齐新库 etl_feiqiu 六层架构
         "schema_oltp": "ods",
@@ -52,7 +53,6 @@ DEFAULTS = {
             "INVENTORY_CHANGE",
             "TOPUPS",
             "TABLE_DISCOUNT",
-            "ASSISTANT_ABOLISH",
             "LEDGER",
         ],
         "dws_tasks": [],
@@ -178,5 +178,4 @@ TASK_TABLES = "TABLES"
 TASK_PACKAGES_DEF = "PACKAGES_DEF"
 TASK_TOPUPS = "TOPUPS"
 TASK_TABLE_DISCOUNT = "TABLE_DISCOUNT"
-TASK_ASSISTANT_ABOLISH = "ASSISTANT_ABOLISH"
 TASK_LEDGER = "LEDGER"

apps/etl/connectors/feiqiu/config/env_parser.py

@@ -7,6 +7,7 @@ from copy import deepcopy
 
 ENV_MAP = {
     "TIMEZONE": ("app.timezone",),
+    "BUSINESS_DAY_START_HOUR": ("app.business_day_start_hour",),
     "STORE_ID": ("app.store_id",),
     "SCHEMA_OLTP": ("app.schema_oltp",),
     "SCHEMA_ETL": ("app.schema_etl",),
@@ -114,6 +115,9 @@ ENV_MAP = {
     "DATA_SOURCE": ("run.data_source",),
     # API 额外请求头（JSON 对象格式）
     "API_HEADERS_EXTRA": ("api.headers_extra",),
+    # Pipeline 管道限流参数
+    "PIPELINE_RATE_MIN": ("pipeline.rate_min",),
+    "PIPELINE_RATE_MAX": ("pipeline.rate_max",),
 }
 
 

apps/etl/connectors/feiqiu/config/pipeline_config.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""统一管道配置数据类。
+
+支持全局默认值 + 任务级覆盖的三级回退：
+  pipeline.<task_code>.* → pipeline.* → 硬编码默认值
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .settings import AppConfig
+
+
+@dataclass(frozen=True)
+class PipelineConfig:
+    """统一管道配置，支持全局默认 + 任务级覆盖。"""
+
+    workers: int = 2                        # ProcessingPool 工作线程数
+    queue_size: int = 100                   # 处理队列容量
+    batch_size: int = 100                   # WriteWorker 批量写入阈值
+    batch_timeout: float = 5.0              # WriteWorker 等待超时（秒）
+    rate_min: float = 0.1                   # RateLimiter 最小间隔（秒）
+    rate_max: float = 2.0                   # RateLimiter 最大间隔（秒）
+    max_consecutive_failures: int = 10      # 连续失败中断阈值
+
+    def __post_init__(self) -> None:
+        if self.workers < 1:
+            raise ValueError(f"workers 必须 >= 1，当前值: {self.workers}")
+        if self.queue_size < 1:
+            raise ValueError(f"queue_size 必须 >= 1，当前值: {self.queue_size}")
+        if self.batch_size < 1:
+            raise ValueError(f"batch_size 必须 >= 1，当前值: {self.batch_size}")
+        if self.rate_min > self.rate_max:
+            raise ValueError(
+                f"rate_min({self.rate_min}) 不能大于 rate_max({self.rate_max})"
+            )
+
+    @classmethod
+    def from_app_config(
+        cls,
+        config: AppConfig,
+        task_code: str | None = None,
+    ) -> PipelineConfig:
+        """从 AppConfig 加载，支持 pipeline.<task_code>.* 任务级覆盖。
+
+        回退优先级：
+          1. pipeline.<task_code_lower>.<key>  （任务级，仅 task_code 非空时查找）
+          2. pipeline.<key>                    （全局级）
+          3. 字段硬编码默认值
+        """
+
+        def _get(key: str, default):  # noqa: ANN001
+            # 任务级覆盖
+            if task_code:
+                val = config.get(f"pipeline.{task_code.lower()}.{key}")
+                if val is not None:
+                    return type(default)(val)
+            # 全局级
+            val = config.get(f"pipeline.{key}")
+            if val is not None:
+                return type(default)(val)
+            # 硬编码默认值
+            return default
+
+        return cls(
+            workers=_get("workers", 2),
+            queue_size=_get("queue_size", 100),
+            batch_size=_get("batch_size", 100),
+            batch_timeout=_get("batch_timeout", 5.0),
+            rate_min=_get("rate_min", 5.0),
+            rate_max=_get("rate_max", 20.0),
+            max_consecutive_failures=_get("max_consecutive_failures", 10),
+        )

apps/etl/connectors/feiqiu/config/settings.py

@@ -111,6 +111,12 @@ class AppConfig:
             missing.append("app.store_id")
         if missing:
             raise SystemExit("缺少必需配置: " + ", ".join(missing))
+
+        # business_day_start_hour 范围校验（0–23 整数）
+        hour = cfg["app"].get("business_day_start_hour", 8)
+        if not isinstance(hour, int) or not (0 <= hour <= 23):
+            raise SystemExit("app.business_day_start_hour 必须为 0–23 的整数")
+
     
     def get(self, key: str, default=None):
         """获取配置值（支持点号路径）"""