root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

迁移代码到Git

Browse Source
This commit is contained in:
Neo
2025-11-18 02:32:00 +08:00
parent 7f87421678
commit c3749474c6
85 changed files with 185478 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
etl_billiards/config/__init__.py Normal file
View File

95
etl_billiards/config/defaults.py Normal file
View File

@@ -0,0 +1,95 @@
# -*- coding: utf-8 -*-
"""配置默认值"""
DEFAULTS = {
"app": {
"timezone": "Asia/Taipei",
"store_id": "",
"schema_oltp": "billiards",
"schema_etl": "etl_admin",
},
"db": {
"dsn": "",
"host": "",
"port": "",
"name": "",
"user": "",
"password": "",
"connect_timeout_sec": 5,
"batch_size": 1000,
"session": {
"timezone": "Asia/Taipei",
"statement_timeout_ms": 30000,
"lock_timeout_ms": 5000,
"idle_in_tx_timeout_ms": 600000
},
},
"api": {
"base_url": None,
"token": None,
"timeout_sec": 20,
"page_size": 200,
"retries": {
"max_attempts": 3,
"backoff_sec": [1, 2, 4],
},
"headers_extra": {},
},
"run": {
"tasks": [
"PRODUCTS", "TABLES", "MEMBERS", "ASSISTANTS", "PACKAGES_DEF",
"ORDERS", "PAYMENTS", "REFUNDS", "COUPON_USAGE", "INVENTORY_CHANGE",
"TOPUPS", "TABLE_DISCOUNT", "ASSISTANT_ABOLISH",
"LEDGER",
],
"window_minutes": {
"default_busy": 30,
"default_idle": 180,
},
"overlap_seconds": 120,
"idle_window": {
"start": "04:00",
"end": "16:00",
},
"allow_empty_result_advance": True,
},
"io": {
"export_root": r"D:\LLZQ\DB\export",
"log_root": r"D:\LLZQ\DB\logs",
"manifest_name": "manifest.json",
"ingest_report_name": "ingest_report.json",
"write_pretty_json": False,
"max_file_bytes": 50 * 1024 * 1024,
},
"clean": {
"log_unknown_fields": True,
"unknown_fields_limit": 50,
"hash_key": {
"algo": "sha1",
"salt": "",
},
"strict_numeric": True,
"round_money_scale": 2,
},
"security": {
"redact_in_logs": True,
"redact_keys": ["token", "password", "Authorization"],
"echo_token_in_logs": False,
},
}
# 任务代码常量
TASK_ORDERS = "ORDERS"
TASK_PAYMENTS = "PAYMENTS"
TASK_REFUNDS = "REFUNDS"
TASK_INVENTORY_CHANGE = "INVENTORY_CHANGE"
TASK_COUPON_USAGE = "COUPON_USAGE"
TASK_MEMBERS = "MEMBERS"
TASK_ASSISTANTS = "ASSISTANTS"
TASK_PRODUCTS = "PRODUCTS"
TASK_TABLES = "TABLES"
TASK_PACKAGES_DEF = "PACKAGES_DEF"
TASK_TOPUPS = "TOPUPS"
TASK_TABLE_DISCOUNT = "TABLE_DISCOUNT"
TASK_ASSISTANT_ABOLISH = "ASSISTANT_ABOLISH"
TASK_LEDGER = "LEDGER"

62
etl_billiards/config/env_parser.py Normal file
View File

@@ -0,0 +1,62 @@
# -*- coding: utf-8 -*-
"""环境变量解析"""
import os
import json
from copy import deepcopy
ENV_MAP = {
"TIMEZONE": ("app.timezone",),
"STORE_ID": ("app.store_id",),
"SCHEMA_OLTP": ("app.schema_oltp",),
"SCHEMA_ETL": ("app.schema_etl",),
"PG_DSN": ("db.dsn",),
"PG_HOST": ("db.host",),
"PG_PORT": ("db.port",),
"PG_NAME": ("db.name",),
"PG_USER": ("db.user",),
"PG_PASSWORD": ("db.password",),
"API_BASE": ("api.base_url",),
"API_TOKEN": ("api.token",),
"API_TIMEOUT": ("api.timeout_sec",),
"API_PAGE_SIZE": ("api.page_size",),
"EXPORT_ROOT": ("io.export_root",),
"LOG_ROOT": ("io.log_root",),
"OVERLAP_SECONDS": ("run.overlap_seconds",),
"WINDOW_BUSY_MIN": ("run.window_minutes.default_busy",),
"WINDOW_IDLE_MIN": ("run.window_minutes.default_idle",),
}
def _deep_set(d, dotted_keys, value):
cur = d
for k in dotted_keys[:-1]:
cur = cur.setdefault(k, {})
cur[dotted_keys[-1]] = value
def _coerce_env(v: str):
if v is None:
return None
s = v.strip()
if s.lower() in ("true", "false"):
return s.lower() == "true"
try:
if s.isdigit() or (s.startswith("-") and s[1:].isdigit()):
return int(s)
except Exception:
pass
if (s.startswith("{") and s.endswith("}")) or (s.startswith("[") and s.endswith("]")):
try:
return json.loads(s)
except Exception:
return s
return s
def load_env_overrides(defaults: dict) -> dict:
cfg = deepcopy(defaults)
for env_key, dotted in ENV_MAP.items():
val = os.environ.get(env_key)
if val is None:
continue
v2 = _coerce_env(val)
for path in dotted:
_deep_set(cfg, path.split("."), v2)
return cfg

85
etl_billiards/config/settings.py Normal file
View File

@@ -0,0 +1,85 @@
# -*- coding: utf-8 -*-
"""配置管理主类"""
from copy import deepcopy
from .defaults import DEFAULTS
from .env_parser import load_env_overrides
class AppConfig:
"""应用配置管理器"""
def __init__(self, config_dict: dict):
self.config = config_dict
@classmethod
def load(cls, cli_overrides: dict = None):
"""加载配置: DEFAULTS < ENV < CLI"""
cfg = load_env_overrides(DEFAULTS)
if cli_overrides:
cls._deep_merge(cfg, cli_overrides)
# 规范化
cls._normalize(cfg)
cls._validate(cfg)
return cls(cfg)
@staticmethod
def _deep_merge(dst, src):
"""深度合并字典"""
for k, v in src.items():
if isinstance(v, dict) and isinstance(dst.get(k), dict):
AppConfig._deep_merge(dst[k], v)
else:
dst[k] = v
@staticmethod
def _normalize(cfg):
"""规范化配置"""
# 转换 store_id 为整数
try:
cfg["app"]["store_id"] = int(str(cfg["app"]["store_id"]).strip())
except Exception:
raise SystemExit("app.store_id 必须为整数")
# DSN 组装
if not cfg["db"]["dsn"]:
cfg["db"]["dsn"] = (
f"postgresql://{cfg['db']['user']}:{cfg['db']['password']}"
f"@{cfg['db']['host']}:{cfg['db']['port']}/{cfg['db']['name']}"
)
# 会话参数
cfg["db"].setdefault("session", {})
sess = cfg["db"]["session"]
sess.setdefault("timezone", cfg["app"]["timezone"])
for k in ("statement_timeout_ms", "lock_timeout_ms", "idle_in_tx_timeout_ms"):
if k in sess and sess[k] is not None:
try:
sess[k] = int(sess[k])
except Exception:
raise SystemExit(f"db.session.{k} 需为整数毫秒")
@staticmethod
def _validate(cfg):
"""验证必填配置"""
missing = []
if not cfg["app"]["store_id"]:
missing.append("app.store_id")
if missing:
raise SystemExit("缺少必需配置: " + ", ".join(missing))
def get(self, key: str, default=None):
"""获取配置值（支持点号路径）"""
keys = key.split(".")
val = self.config
for k in keys:
if isinstance(val, dict):
val = val.get(k)
else:
return default
return val if val is not None else default
def __getitem__(self, key):
return self.config[key]