初始提交：飞球 ETL 系统全量代码

2026-02-13 08:05:34 +08:00
commit 3c51f5485d
441 changed files with 117631 additions and 0 deletions
--- a/config/defaults.py
+++ b/config/defaults.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+"""配置默认值定义"""
+
+DEFAULTS = {
+    "app": {
+        "timezone": "Asia/Shanghai",
+        "store_id": "",
+        "schema_oltp": "billiards",
+        "schema_etl": "etl_admin",
+    },
+    "db": {
+        "dsn": "",
+        "host": "",
+        "port": "",
+        "name": "",
+        "user": "",
+        "password": "",
+        "connect_timeout_sec": 20,
+        "batch_size": 1000,
+        "session": {
+            "timezone": "Asia/Shanghai",
+            "statement_timeout_ms": 30000,
+            "lock_timeout_ms": 5000,
+            "idle_in_tx_timeout_ms": 600000,
+        },
+    },
+    "api": {
+        "base_url": "https://pc.ficoo.vip/apiprod/admin/v1",
+        "token": None,
+        "timeout_sec": 20,
+        "page_size": 200,
+        "params": {},
+        "retries": {
+            "max_attempts": 3,
+            "backoff_sec": [1, 2, 4],
+        },
+        "headers_extra": {},
+    },
+    "run": {
+        "data_source": "hybrid",
+        "tasks": [
+            "PRODUCTS",
+            "TABLES",
+            "MEMBERS",
+            "ASSISTANTS",
+            "PACKAGES_DEF",
+            "ORDERS",
+            "PAYMENTS",
+            "REFUNDS",
+            "COUPON_USAGE",
+            "INVENTORY_CHANGE",
+            "TOPUPS",
+            "TABLE_DISCOUNT",
+            "ASSISTANT_ABOLISH",
+            "LEDGER",
+        ],
+        "dws_tasks": [],
+        "index_tasks": [],
+        "index_lookback_days": 60,
+        "window_minutes": {
+            "default_busy": 30,
+            "default_idle": 180,
+        },
+        "overlap_seconds": 600,
+        "snapshot_missing_delete": True,
+        "snapshot_allow_empty_delete": False,
+        "window_split": {
+            "unit": "day",
+            "days": 10,
+            "compensation_hours": 2,
+        },
+        "idle_window": {
+            "start": "04:00",
+            "end": "16:00",
+        },
+        "allow_empty_result_advance": True,
+    },
+    "io": {
+        "export_root": "export/JSON",
+        "log_root": "export/LOG",
+        "fetch_root": "export/JSON",
+        "ingest_source_dir": "",
+        "manifest_name": "manifest.json",
+        "ingest_report_name": "ingest_report.json",
+        "write_pretty_json": True,
+        "max_file_bytes": 50 * 1024 * 1024,
+    },
+    "pipeline": {
+        # 运行流程：FETCH_ONLY（仅在线抓取落盘）、INGEST_ONLY（本地清洗入库）、FULL（抓取 + 清洗入库）
+        "flow": "FULL",
+        # 在线抓取 JSON 输出根目录（按任务、run_id 与时间自动创建子目录）
+        "fetch_root": "export/JSON",
+        # 本地清洗入库时的 JSON 输入目录（为空则默认使用本次抓取目录）
+        "ingest_source_dir": "",
+    },
+    "clean": {
+        "log_unknown_fields": True,
+        "unknown_fields_limit": 50,
+        "hash_key": {
+            "algo": "sha1",
+            "salt": "",
+        },
+        "strict_numeric": True,
+        "round_money_scale": 2,
+    },
+    "security": {
+        "redact_in_logs": True,
+        "redact_keys": ["token", "password", "Authorization"],
+        "echo_token_in_logs": False,
+    },
+    "ods": {
+        # ODS 离线重建/回放相关（仅开发/运维使用）
+        "json_doc_dir": "export/test-json-doc",
+        "include_files": "",
+        "drop_schema_first": True,
+    },
+    "integrity": {
+        "mode": "history",
+        "history_start": "2025-07-01",
+        "history_end": "",
+        "include_dimensions": True,
+        "auto_check": False,
+        "auto_backfill": False,
+        "compare_content": True,
+        "content_sample_limit": 50,
+        "backfill_mismatch": True,
+        "recheck_after_backfill": True,
+        "ods_task_codes": "",
+        "force_monthly_split": True,
+    },
+    "verification": {
+        "skip_ods_when_fetch_before_verify": True,
+        "ods_use_local_json": True,
+    },
+    "dws": {
+        "monthly": {
+            "allow_history": False,
+            "prev_month_grace_days": 5,
+            "history_months": 0,
+            "new_hire_cap_effective_from": "2026-03-01",
+            "new_hire_cap_day": 25,
+            "new_hire_max_tier_level": 2,
+        },
+        "salary": {
+            "run_days": 5,
+            "allow_out_of_cycle": False,
+            "room_course_price": 138,
+        },
+    },
+    "dwd": {
+        "fact_upsert": True,
+        # 事实表补齐 UPSERT 批量参数（可按锁冲突情况调优）
+        "fact_upsert_batch_size": 1000,
+        "fact_upsert_min_batch_size": 100,
+        "fact_upsert_max_retries": 2,
+        "fact_upsert_retry_backoff_sec": [1, 2, 4],
+        # 仅对事实表 backfill 设置的锁等待超时（None 表示沿用 db.session.lock_timeout_ms）
+        "fact_upsert_lock_timeout_ms": None,
+    },
+
+}
+
+# 任务代码常量
+TASK_ORDERS = "ORDERS"
+TASK_PAYMENTS = "PAYMENTS"
+TASK_REFUNDS = "REFUNDS"
+TASK_INVENTORY_CHANGE = "INVENTORY_CHANGE"
+TASK_COUPON_USAGE = "COUPON_USAGE"
+TASK_MEMBERS = "MEMBERS"
+TASK_ASSISTANTS = "ASSISTANTS"
+TASK_PRODUCTS = "PRODUCTS"
+TASK_TABLES = "TABLES"
+TASK_PACKAGES_DEF = "PACKAGES_DEF"
+TASK_TOPUPS = "TOPUPS"
+TASK_TABLE_DISCOUNT = "TABLE_DISCOUNT"
+TASK_ASSISTANT_ABOLISH = "ASSISTANT_ABOLISH"
+TASK_LEDGER = "LEDGER"