ODS 完成

This commit is contained in:
Neo
2025-11-30 07:19:05 +08:00
parent b9b050bb5d
commit a6ad343092
81 changed files with 15695 additions and 227180 deletions

View File

@@ -65,18 +65,18 @@ DEFAULTS = {
"allow_empty_result_advance": True,
},
"io": {
"export_root": r"D:\LLZQ\DB\export",
"log_root": r"D:\LLZQ\DB\logs",
"export_root": r"C:\dev\LLTQ\export\JSON",
"log_root": r"C:\dev\LLTQ\export\LOG",
"manifest_name": "manifest.json",
"ingest_report_name": "ingest_report.json",
"write_pretty_json": False,
"write_pretty_json": True,
"max_file_bytes": 50 * 1024 * 1024,
},
"pipeline": {
# 运行流程FETCH_ONLY仅在线抓取落盘、INGEST_ONLY本地清洗入库、FULL抓取 + 清洗入库)
"flow": "FULL",
# 在线抓取 JSON 输出根目录按任务、run_id 与时间自动创建子目录)
"fetch_root": r"D:\LLZQ\DB\json_fetch",
"fetch_root": r"C:\dev\LLTQ\export\JSON",
# 本地清洗入库时的 JSON 输入目录(为空则默认使用本次抓取目录)
"ingest_source_dir": "",
},
@@ -95,6 +95,12 @@ DEFAULTS = {
"redact_keys": ["token", "password", "Authorization"],
"echo_token_in_logs": False,
},
"ods": {
# ODS 离线重建/回放相关(仅开发/运维使用)
"json_doc_dir": r"C:\dev\LLTQ\export\test-json-doc",
"include_files": "",
"drop_schema_first": True,
},
}
# 任务代码常量