183 lines
5.3 KiB
Python
183 lines
5.3 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""配置默认值定义"""
|
||
|
||
DEFAULTS = {
|
||
"app": {
|
||
"timezone": "Asia/Shanghai",
|
||
"store_id": "",
|
||
# CHANGE 2026-02-15 | 对齐新库 etl_feiqiu 六层架构
|
||
"schema_oltp": "ods",
|
||
"schema_etl": "meta",
|
||
},
|
||
"db": {
|
||
"dsn": "",
|
||
"host": "",
|
||
"port": "",
|
||
"name": "",
|
||
"user": "",
|
||
"password": "",
|
||
"connect_timeout_sec": 20,
|
||
"batch_size": 1000,
|
||
"session": {
|
||
"timezone": "Asia/Shanghai",
|
||
"statement_timeout_ms": 30000,
|
||
"lock_timeout_ms": 5000,
|
||
"idle_in_tx_timeout_ms": 600000,
|
||
},
|
||
},
|
||
"api": {
|
||
"base_url": "https://pc.ficoo.vip/apiprod/admin/v1",
|
||
"token": None,
|
||
"timeout_sec": 20,
|
||
"page_size": 200,
|
||
"params": {},
|
||
"retries": {
|
||
"max_attempts": 3,
|
||
"backoff_sec": [1, 2, 4],
|
||
},
|
||
"headers_extra": {},
|
||
},
|
||
"run": {
|
||
"data_source": "hybrid",
|
||
"tasks": [
|
||
"PRODUCTS",
|
||
"TABLES",
|
||
"MEMBERS",
|
||
"ASSISTANTS",
|
||
"PACKAGES_DEF",
|
||
"ORDERS",
|
||
"PAYMENTS",
|
||
"REFUNDS",
|
||
"COUPON_USAGE",
|
||
"INVENTORY_CHANGE",
|
||
"TOPUPS",
|
||
"TABLE_DISCOUNT",
|
||
"ASSISTANT_ABOLISH",
|
||
"LEDGER",
|
||
],
|
||
"dws_tasks": [],
|
||
"index_tasks": [],
|
||
"index_lookback_days": 60,
|
||
"window_minutes": {
|
||
"default_busy": 30,
|
||
"default_idle": 180,
|
||
},
|
||
"overlap_seconds": 600,
|
||
"snapshot_missing_delete": True,
|
||
"snapshot_allow_empty_delete": False,
|
||
"snapshot_protect_early_cutoff": True,
|
||
# CHANGE 2026-02-18 | 强制全量更新：跳过 ODS hash 去重 + DWD 变更对比，无条件写入
|
||
"force_full_update": False,
|
||
"window_split": {
|
||
"unit": "day",
|
||
"days": 10,
|
||
"compensation_hours": 2,
|
||
},
|
||
"idle_window": {
|
||
"start": "04:00",
|
||
"end": "16:00",
|
||
},
|
||
"allow_empty_result_advance": True,
|
||
},
|
||
"io": {
|
||
"export_root": "",
|
||
"log_root": "",
|
||
"fetch_root": "",
|
||
"ingest_source_dir": "",
|
||
"manifest_name": "manifest.json",
|
||
"ingest_report_name": "ingest_report.json",
|
||
"write_pretty_json": True,
|
||
"max_file_bytes": 50 * 1024 * 1024,
|
||
},
|
||
"pipeline": {
|
||
# 旧配置节（保留以兼容旧 .env 和调用方）
|
||
# 运行流程：FETCH_ONLY（仅在线抓取落盘）、INGEST_ONLY（本地清洗入库）、FULL（抓取 + 清洗入库）
|
||
"flow": "FULL",
|
||
# 在线抓取 JSON 输出根目录（按任务、run_id 与时间自动创建子目录）
|
||
"fetch_root": "",
|
||
# 本地清洗入库时的 JSON 输入目录（为空则默认使用本次抓取目录）
|
||
"ingest_source_dir": "",
|
||
},
|
||
"clean": {
|
||
"log_unknown_fields": True,
|
||
"unknown_fields_limit": 50,
|
||
"hash_key": {
|
||
"algo": "sha1",
|
||
"salt": "",
|
||
},
|
||
"strict_numeric": True,
|
||
"round_money_scale": 2,
|
||
},
|
||
"security": {
|
||
"redact_in_logs": True,
|
||
"redact_keys": ["token", "password", "Authorization"],
|
||
"echo_token_in_logs": False,
|
||
},
|
||
"ods": {
|
||
# ODS 离线重建/回放相关（仅开发/运维使用）
|
||
"json_doc_dir": "",
|
||
"include_files": "",
|
||
"drop_schema_first": True,
|
||
},
|
||
"integrity": {
|
||
"mode": "history",
|
||
"history_start": "2025-07-01",
|
||
"history_end": "",
|
||
"include_dimensions": True,
|
||
"auto_check": False,
|
||
"auto_backfill": False,
|
||
"compare_content": True,
|
||
"content_sample_limit": 50,
|
||
"backfill_mismatch": True,
|
||
"recheck_after_backfill": True,
|
||
"ods_task_codes": "",
|
||
"force_monthly_split": True,
|
||
},
|
||
"verification": {
|
||
"skip_ods_when_fetch_before_verify": True,
|
||
"ods_use_local_json": True,
|
||
},
|
||
"dws": {
|
||
"monthly": {
|
||
"allow_history": False,
|
||
"prev_month_grace_days": 5,
|
||
"history_months": 0,
|
||
"new_hire_cap_effective_from": "2026-03-01",
|
||
"new_hire_cap_day": 25,
|
||
"new_hire_max_tier_level": 2,
|
||
},
|
||
"salary": {
|
||
"run_days": 5,
|
||
"allow_out_of_cycle": False,
|
||
"room_course_price": 138,
|
||
},
|
||
},
|
||
"dwd": {
|
||
"fact_upsert": True,
|
||
# 事实表补齐 UPSERT 批量参数（可按锁冲突情况调优）
|
||
"fact_upsert_batch_size": 1000,
|
||
"fact_upsert_min_batch_size": 100,
|
||
"fact_upsert_max_retries": 2,
|
||
"fact_upsert_retry_backoff_sec": [1, 2, 4],
|
||
# 仅对事实表 backfill 设置的锁等待超时（None 表示沿用 db.session.lock_timeout_ms）
|
||
"fact_upsert_lock_timeout_ms": None,
|
||
},
|
||
|
||
}
|
||
|
||
# 任务代码常量
|
||
TASK_ORDERS = "ORDERS"
|
||
TASK_PAYMENTS = "PAYMENTS"
|
||
TASK_REFUNDS = "REFUNDS"
|
||
TASK_INVENTORY_CHANGE = "INVENTORY_CHANGE"
|
||
TASK_COUPON_USAGE = "COUPON_USAGE"
|
||
TASK_MEMBERS = "MEMBERS"
|
||
TASK_ASSISTANTS = "ASSISTANTS"
|
||
TASK_PRODUCTS = "PRODUCTS"
|
||
TASK_TABLES = "TABLES"
|
||
TASK_PACKAGES_DEF = "PACKAGES_DEF"
|
||
TASK_TOPUPS = "TOPUPS"
|
||
TASK_TABLE_DISCOUNT = "TABLE_DISCOUNT"
|
||
TASK_ASSISTANT_ABOLISH = "ASSISTANT_ABOLISH"
|
||
TASK_LEDGER = "LEDGER"
|