Compare commits

..

10 Commits

Author SHA1 Message Date
Neo
294c6edbc9 更新数据库文档 20260201-2 2026-02-01 23:42:18 +08:00
Neo
9b2c2c5c78 更新20260201-1 2026-02-01 22:04:15 +08:00
Neo
076f5755ca 数据库 数据校验写入等逻辑更新。 2026-02-01 03:46:16 +08:00
Neo
9948000b71 更新1 2026-01-28 20:23:54 +08:00
Neo
4fafc80254 提交 2026-01-28 00:04:30 +08:00
Neo
1a76108209 修复脚本 2026-01-28 00:02:15 +08:00
Neo
c42b516895 同步? 2026-01-28 00:00:22 +08:00
Neo
7e67bc4218 更新 2026-01-27 23:45:36 +08:00
Neo
64a3159f9e 移除依赖 2026-01-27 23:22:21 +08:00
Neo
ba00654ac5 exe 依赖添加 2026-01-27 23:19:54 +08:00
400 changed files with 528022 additions and 17476 deletions

60
collect_env_report.ps1 Normal file
View File

@@ -0,0 +1,60 @@
$ErrorActionPreference = "Stop"
Write-Host "[1/6] Collecting environment info..." -ForegroundColor Cyan
$report = @()
$report += "# ETL Manager Environment Report"
$report += "Timestamp: $(Get-Date -Format 'yyyy-MM-dd HH:mm:ss')"
$report += ""
Write-Host "[2/6] Resolving python/pip..." -ForegroundColor Cyan
$report += "## Executables"
$py = (Get-Command python -ErrorAction SilentlyContinue)
$pip = (Get-Command pip -ErrorAction SilentlyContinue)
if ($py) { $report += "python: $($py.Source)" } else { $report += "python: NOT FOUND" }
if ($pip) { $report += "pip: $($pip.Source)" } else { $report += "pip: NOT FOUND" }
$report += ""
Write-Host "[3/6] Python details..." -ForegroundColor Cyan
$report += "## Python info"
$report += (python -c "import sys, platform; print('version='+sys.version.replace('\n',' ')); print('executable='+sys.executable); print('prefix='+sys.prefix); print('base_prefix='+sys.base_prefix); print('arch='+platform.architecture()[0]); print('platform='+platform.platform())" 2>&1)
$report += ""
Write-Host "[4/6] pip details..." -ForegroundColor Cyan
$report += "## pip"
$report += (python -m pip --version 2>&1)
$report += ""
Write-Host "[5/6] PySide6 details..." -ForegroundColor Cyan
$report += "## PySide6"
$pyside = @'
try:
import PySide6
from PySide6 import QtCore
print('PySide6='+PySide6.__version__)
print('Qt='+QtCore.qVersion())
print('PySide6_path='+PySide6.__file__)
print('Qt_plugins_path='+QtCore.QLibraryInfo.path(QtCore.QLibraryInfo.PluginsPath))
except Exception as e:
print('PySide6_import_error='+repr(e))
'@
$report += ($pyside | python - 2>&1)
$report += ""
Write-Host "[6/6] Installed packages..." -ForegroundColor Cyan
$report += "## Installed packages (freeze)"
$report += (python -m pip list --format=freeze 2>&1)
$reportPath = "D:\env_report_local.txt"
if (-not (Test-Path "D:\")) {
Write-Host "[WARN] D: not found, fallback to current directory." -ForegroundColor Yellow
$reportPath = ".\env_report_local.txt"
}
$reportDir = Split-Path $reportPath
if ($reportDir -and -not (Test-Path $reportDir)) {
New-Item -ItemType Directory -Path $reportDir -Force | Out-Null
}
Write-Host "[WRITE] $reportPath" -ForegroundColor Green
$report -join "`n" | Set-Content -Path $reportPath -Encoding UTF8
Write-Host "[DONE]" -ForegroundColor Green
Write-Output $reportPath

121
env_report_local.txt Normal file
View File

@@ -0,0 +1,121 @@
# ETL Manager Environment Report
Timestamp: 2026-01-28 00:56:34
## Executables
python: C:\ProgramData\miniconda3\python.exe
pip: C:\ProgramData\miniconda3\Scripts\pip.exe
## Python info
version=3.13.9 | packaged by Anaconda, Inc. | (main, Oct 21 2025, 19:09:58) [MSC v.1929 64 bit (AMD64)]
executable=C:\ProgramData\miniconda3\python.exe
prefix=C:\ProgramData\miniconda3
base_prefix=C:\ProgramData\miniconda3
arch=64bit
platform=Windows-10-10.0.17763-SP0
## pip
pip 25.3 from C:\ProgramData\miniconda3\Lib\site-packages\pip (python 3.13)
## PySide6
PySide6=6.10.1
Qt=6.10.1
PySide6_path=C:\ProgramData\miniconda3\Lib\site-packages\PySide6\__init__.py
Qt_plugins_path=C:/ProgramData/miniconda3/Lib/site-packages/PySide6/plugins
## Installed packages (freeze)
altgraph==0.17.5
anaconda-anon-usage==0.7.4
annotated-types==0.6.0
archspec==0.2.5
beautifulsoup4==4.13.4
boltons==25.0.0
brotlicffi==1.1.0.0
certifi==2025.11.12
cffi==2.0.0
chardet==3.0.4
charset-normalizer==3.4.4
colorama==0.4.6
conda==25.9.1
conda-anaconda-telemetry==0.3.0
conda-anaconda-tos==0.2.2
conda-content-trust==0.2.0
conda-libmamba-solver==25.4.0
conda-package-handling==2.4.0
conda_package_streaming==0.12.0
coverage==7.12.0
cryptography==46.0.3
distro==1.9.0
et_xmlfile==2.0.0
frozendict==2.4.6
ftfy==6.3.1
googletrans==4.0.0rc1
h11==0.9.0
h2==3.2.0
hpack==3.0.0
hstspreload==2025.1.1
httpcore==0.9.1
httpx==0.13.3
hyperframe==5.2.0
idna==2.10
iniconfig==2.3.0
jsonpatch==1.33
jsonpointer==3.0.0
libmambapy==2.3.2
lxml==6.0.2
markdown-it-py==4.0.0
mdurl==0.1.2
menuinst==2.4.1
numpy==2.3.2
openpyxl==3.1.5
packaging==25.0
pandas==2.3.1
pefile==2024.8.26
pip==25.2
platformdirs==4.5.0
pluggy==1.5.0
psycopg2==2.9.10
psycopg2-binary==2.9.11
pyaes==1.6.1
pyasn1==0.6.1
pycosat==0.6.6
pycparser==2.23
pydantic==2.12.3
pydantic_core==2.41.4
Pygments==2.19.2
pyinstaller==6.18.0
pyinstaller-hooks-contrib==2025.11
PySide6==6.10.1
PySide6_Addons==6.10.1
PySide6_Essentials==6.10.1
PySocks==1.7.1
pytest==9.0.1
pytest-cov==7.0.0
python-dateutil==2.9.0.post0
python-docx==1.2.0
python-dotenv==1.2.1
pytz==2025.2
pywin32-ctypes==0.2.3
requests==2.32.5
rfc3986==1.5.0
rich==14.2.0
rsa==4.9.1
ruamel.yaml==0.18.16
ruamel.yaml.clib==0.2.14
setuptools==80.9.0
shiboken6==6.10.1
six==1.17.0
sniffio==1.3.1
soupsieve==2.7
Telethon==1.40.0
tqdm==4.67.1
truststore==0.10.1
typing_extensions==4.15.0
typing-inspection==0.4.2
tzdata==2025.2
urllib3==2.5.0
wcwidth==0.2.14
wheel==0.45.1
win_inet_pton==1.1.0
xlsxwriter==3.2.9
zstandard==0.24.0

View File

@@ -32,7 +32,7 @@ SCHEMA_ETL=etl_admin
# API 配置 # API 配置
# ------------------------------------------------------------------------------ # ------------------------------------------------------------------------------
API_BASE=https://pc.ficoo.vip/apiprod/admin/v1/ API_BASE=https://pc.ficoo.vip/apiprod/admin/v1/
API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6Iks1ZnBhYlRTNkFsR0FpMmN4WGYrMHdJVkk0L2UvTVQrSVBHM3V5VWRrSjg9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzEvMzEg5LiL5Y2IMTA6MTQ6NTEiLCJuZWVkQ2hlY2tUb2tlbiI6ImZhbHNlIiwiZXhwIjoxNzY5ODY4ODkxLCJpc3MiOiJ0ZXN0IiwiYXVkIjoiVXNlciJ9.BH3-iwwrBczb8aFfI__6kwe3AIsEPacN9TruaTrQ3nY API_TOKEN=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnQtdHlwZSI6IjQiLCJ1c2VyLXR5cGUiOiIxIiwiaHR0cDovL3NjaGVtYXMubWljcm9zb2Z0LmNvbS93cy8yMDA4LzA2L2lkZW50aXR5L2NsYWltcy9yb2xlIjoiMTIiLCJyb2xlLWlkIjoiMTIiLCJ0ZW5hbnQtaWQiOiIyNzkwNjgzMTYwNzA5OTU3Iiwibmlja25hbWUiOiLnp5_miLfnrqHnkIblkZjvvJrmganmgakxIiwic2l0ZS1pZCI6IjAiLCJtb2JpbGUiOiIxMzgxMDUwMjMwNCIsInNpZCI6IjI5NTA0ODk2NTgzOTU4NDUiLCJzdGFmZi1pZCI6IjMwMDk5MTg2OTE1NTkwNDUiLCJvcmctaWQiOiIwIiwicm9sZS10eXBlIjoiMyIsInJlZnJlc2hUb2tlbiI6IktlbTVsdHRqZ2tSUExOcVA2ajhNakdQYnFrNW5mRzBQNzRvMHE0b295VVE9IiwicmVmcmVzaEV4cGlyeVRpbWUiOiIyMDI2LzIvOCDkuIvljYg2OjU3OjA1IiwibmVlZENoZWNrVG9rZW4iOiJmYWxzZSIsImV4cCI6MTc3MDU0ODIyNSwiaXNzIjoidGVzdCIsImF1ZCI6IlVzZXIifQ.wJlm7pTqUzp769nUGdxx0e1bVMy4x9Prp9U_UMWQvlk
# API 请求超时(秒) # API 请求超时(秒)
API_TIMEOUT=20 API_TIMEOUT=20

View File

@@ -1,44 +0,0 @@
# -*- mode: python ; coding: utf-8 -*-
a = Analysis(
['C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\gui\\main.py'],
pathex=[],
binaries=[],
datas=[('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\gui\\resources', 'gui/resources'), ('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\database\\schema_dwd_doc.sql', 'database'), ('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\database\\schema_dws.sql', 'database'), ('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\database\\schema_etl_admin.sql', 'database'), ('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\database\\schema_ODS_doc.sql', 'database'), ('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\database\\seed_ods_tasks.sql', 'database'), ('C:\\dev\\LLTQ\\ETL\\feiqiu-ETL\\etl_billiards\\database\\seed_scheduler_tasks.sql', 'database')],
hiddenimports=['PySide6.QtCore', 'PySide6.QtGui', 'PySide6.QtWidgets', 'psycopg2', 'psycopg2.extras', 'psycopg2.extensions', 'gui.models.task_model', 'gui.models.schedule_model', 'gui.utils.cli_builder', 'gui.utils.config_helper', 'gui.utils.app_settings', 'gui.workers.task_worker', 'gui.workers.db_worker', 'gui.widgets.settings_dialog'],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=['matplotlib', 'numpy', 'pandas', 'scipy', 'PIL', 'cv2', 'tkinter'],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=True,
name='ETL_Manager',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
)
coll = COLLECT(
exe,
a.binaries,
a.datas,
strip=False,
upx=True,
upx_exclude=[],
name='ETL_Manager',
)

View File

@@ -4,7 +4,9 @@ from __future__ import annotations
from datetime import datetime from datetime import datetime
from pathlib import Path from pathlib import Path
import time
from typing import Any, Iterable, Tuple from typing import Any, Iterable, Tuple
from zoneinfo import ZoneInfo
from api.client import APIClient from api.client import APIClient
from api.endpoint_routing import plan_calls from api.endpoint_routing import plan_calls
@@ -128,3 +130,57 @@ class RecordingAPIClient:
"pages": len(pages), "pages": len(pages),
"records": total_records, "records": total_records,
} }
def _cfg_get(cfg, key: str, default=None):
if isinstance(cfg, dict):
cur = cfg
for part in key.split("."):
if not isinstance(cur, dict) or part not in cur:
return default
cur = cur[part]
return cur
getter = getattr(cfg, "get", None)
if callable(getter):
return getter(key, default)
return default
def build_recording_client(
cfg,
*,
task_code: str,
output_dir: Path | str | None = None,
run_id: int | None = None,
write_pretty: bool | None = None,
):
"""Build RecordingAPIClient from AppConfig or dict config."""
base_client = APIClient(
base_url=_cfg_get(cfg, "api.base_url") or "",
token=_cfg_get(cfg, "api.token"),
timeout=int(_cfg_get(cfg, "api.timeout_sec", 20) or 20),
retry_max=int(_cfg_get(cfg, "api.retries.max_attempts", 3) or 3),
headers_extra=_cfg_get(cfg, "api.headers_extra") or {},
)
if write_pretty is None:
write_pretty = bool(_cfg_get(cfg, "io.write_pretty_json", False))
if run_id is None:
run_id = int(time.time())
if output_dir is None:
tz_name = _cfg_get(cfg, "app.timezone", "Asia/Taipei") or "Asia/Taipei"
tz = ZoneInfo(tz_name)
ts = datetime.now(tz).strftime("%Y%m%d-%H%M%S")
fetch_root = _cfg_get(cfg, "pipeline.fetch_root") or _cfg_get(cfg, "io.export_root") or "export/JSON"
task_upper = str(task_code).upper()
output_dir = Path(fetch_root) / task_upper / f"{task_upper}-{run_id}-{ts}"
return RecordingAPIClient(
base_client=base_client,
output_dir=output_dir,
task_code=str(task_code),
run_id=int(run_id),
write_pretty=bool(write_pretty),
)

View File

@@ -90,9 +90,11 @@ def build_exe(onefile: bool = False, console: bool = False):
# 隐式导入 # 隐式导入
hidden_imports = [ hidden_imports = [
# PySide6 核心模块
"PySide6.QtCore", "PySide6.QtCore",
"PySide6.QtGui", "PySide6.QtGui",
"PySide6.QtWidgets", "PySide6.QtWidgets",
# 数据库
"psycopg2", "psycopg2",
"psycopg2.extras", "psycopg2.extras",
"psycopg2.extensions", "psycopg2.extensions",
@@ -138,6 +140,9 @@ def build_exe(onefile: bool = False, console: bool = False):
result = subprocess.run(cmd, cwd=str(project_root)) result = subprocess.run(cmd, cwd=str(project_root))
if result.returncode == 0: if result.returncode == 0:
# 打包后精简:删除不需要的文件
slim_dist(project_root / "dist" / "ETL_Manager" / "_internal")
print() print()
print("=" * 50) print("=" * 50)
print("打包成功!") print("打包成功!")
@@ -149,6 +154,77 @@ def build_exe(onefile: bool = False, console: bool = False):
sys.exit(1) sys.exit(1)
def slim_dist(internal_dir: Path):
"""精简打包后的文件,删除不需要的内容"""
if not internal_dir.exists():
return
print()
print("精简打包文件...")
removed_size = 0
# 1. 删除不需要的翻译文件(只保留中文和英文)
translations_dir = internal_dir / "PySide6" / "translations"
if translations_dir.exists():
keep_langs = {"zh_CN", "zh_TW", "en"}
for qm_file in translations_dir.glob("*.qm"):
# 检查是否是需要保留的语言
keep = False
for lang in keep_langs:
if lang in qm_file.name:
keep = True
break
if not keep:
size = qm_file.stat().st_size
qm_file.unlink()
removed_size += size
# 2. 删除 opengl32sw.dll软件渲染20MB通常不需要
opengl_sw = internal_dir / "PySide6" / "opengl32sw.dll"
if opengl_sw.exists():
size = opengl_sw.stat().st_size
opengl_sw.unlink()
removed_size += size
print(f" 删除: opengl32sw.dll ({size / 1024 / 1024:.1f} MB)")
# 3. 删除不需要的 Qt 模块 DLL如果存在
unnecessary_dlls = [
"Qt6Pdf.dll", # PDF 支持
"Qt6Qml.dll", # QML 引擎
"Qt6QmlMeta.dll",
"Qt6QmlModels.dll",
"Qt6QmlWorkerScript.dll",
"Qt6Quick.dll", # Quick UI
"Qt6VirtualKeyboard.dll", # 虚拟键盘
]
pyside6_dir = internal_dir / "PySide6"
for dll_name in unnecessary_dlls:
dll_path = pyside6_dir / dll_name
if dll_path.exists():
size = dll_path.stat().st_size
dll_path.unlink()
removed_size += size
print(f" 删除: {dll_name} ({size / 1024 / 1024:.1f} MB)")
# 4. 删除不需要的插件目录
unnecessary_plugins = [
"networkinformation", # 网络信息
"tls", # TLS 支持(数据库已有)
]
plugins_dir = pyside6_dir / "plugins"
if plugins_dir.exists():
for plugin_name in unnecessary_plugins:
plugin_path = plugins_dir / plugin_name
if plugin_path.exists():
size = sum(f.stat().st_size for f in plugin_path.rglob("*") if f.is_file())
shutil.rmtree(plugin_path)
removed_size += size
print(f" 删除插件: {plugin_name}")
print(f"共节省: {removed_size / 1024 / 1024:.1f} MB")
def main(): def main():
"""主函数""" """主函数"""
import argparse import argparse

View File

@@ -109,9 +109,18 @@ DEFAULTS = {
"mode": "history", "mode": "history",
"history_start": "2025-07-01", "history_start": "2025-07-01",
"history_end": "", "history_end": "",
"include_dimensions": False, "include_dimensions": True,
"auto_check": False, "auto_check": False,
"auto_backfill": False,
"compare_content": True,
"content_sample_limit": 50,
"backfill_mismatch": True,
"recheck_after_backfill": True,
"ods_task_codes": "", "ods_task_codes": "",
"force_monthly_split": True,
},
"dwd": {
"fact_upsert": True,
}, },
} }

View File

@@ -55,7 +55,12 @@ ENV_MAP = {
"INTEGRITY_INCLUDE_DIMENSIONS": ("integrity.include_dimensions",), "INTEGRITY_INCLUDE_DIMENSIONS": ("integrity.include_dimensions",),
"INTEGRITY_AUTO_CHECK": ("integrity.auto_check",), "INTEGRITY_AUTO_CHECK": ("integrity.auto_check",),
"INTEGRITY_AUTO_BACKFILL": ("integrity.auto_backfill",), "INTEGRITY_AUTO_BACKFILL": ("integrity.auto_backfill",),
"INTEGRITY_COMPARE_CONTENT": ("integrity.compare_content",),
"INTEGRITY_CONTENT_SAMPLE_LIMIT": ("integrity.content_sample_limit",),
"INTEGRITY_BACKFILL_MISMATCH": ("integrity.backfill_mismatch",),
"INTEGRITY_RECHECK_AFTER_BACKFILL": ("integrity.recheck_after_backfill",),
"INTEGRITY_ODS_TASK_CODES": ("integrity.ods_task_codes",), "INTEGRITY_ODS_TASK_CODES": ("integrity.ods_task_codes",),
"DWD_FACT_UPSERT": ("dwd.fact_upsert",),
} }

File diff suppressed because it is too large Load Diff

View File

@@ -84,7 +84,7 @@ CREATE TABLE IF NOT EXISTS dim_site (
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31', SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1, SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1, SCD2_version INT DEFAULT 1,
PRIMARY KEY (site_id) PRIMARY KEY (site_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_site IS 'DWD 维度表dim_site。ODS 来源表billiards_ods.table_fee_transactions对应 JSONtable_fee_transactions.json分析table_fee_transactions-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_site IS 'DWD 维度表dim_site。ODS 来源表billiards_ods.table_fee_transactions对应 JSONtable_fee_transactions.json分析table_fee_transactions-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -133,7 +133,7 @@ CREATE TABLE IF NOT EXISTS dim_site_Ex (
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31', SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1, SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1, SCD2_version INT DEFAULT 1,
PRIMARY KEY (site_id) PRIMARY KEY (site_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_site_ex IS 'DWD 维度表扩展字段表dim_site_ex。ODS 来源表billiards_ods.table_fee_transactions对应 JSONtable_fee_transactions.json分析table_fee_transactions-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_site_ex IS 'DWD 维度表扩展字段表dim_site_ex。ODS 来源表billiards_ods.table_fee_transactions对应 JSONtable_fee_transactions.json分析table_fee_transactions-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -176,7 +176,7 @@ CREATE TABLE IF NOT EXISTS dim_table (
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31', SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1, SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1, SCD2_version INT DEFAULT 1,
PRIMARY KEY (table_id) PRIMARY KEY (table_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_table IS 'DWD 维度表dim_table。ODS 来源表billiards_ods.site_tables_master对应 JSONsite_tables_master.json分析site_tables_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_table IS 'DWD 维度表dim_table。ODS 来源表billiards_ods.site_tables_master对应 JSONsite_tables_master.json分析site_tables_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -204,7 +204,7 @@ CREATE TABLE IF NOT EXISTS dim_table_Ex (
SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31', SCD2_end_time TIMESTAMPTZ DEFAULT '9999-12-31',
SCD2_is_current INT DEFAULT 1, SCD2_is_current INT DEFAULT 1,
SCD2_version INT DEFAULT 1, SCD2_version INT DEFAULT 1,
PRIMARY KEY (table_id) PRIMARY KEY (table_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_table_ex IS 'DWD 维度表扩展字段表dim_table_ex。ODS 来源表billiards_ods.site_tables_master对应 JSONsite_tables_master.json分析site_tables_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_table_ex IS 'DWD 维度表扩展字段表dim_table_ex。ODS 来源表billiards_ods.site_tables_master对应 JSONsite_tables_master.json分析site_tables_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -240,7 +240,7 @@ CREATE TABLE IF NOT EXISTS dim_assistant (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (assistant_id) PRIMARY KEY (assistant_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_assistant IS 'DWD 维度表dim_assistant。ODS 来源表billiards_ods.assistant_accounts_master对应 JSONassistant_accounts_master.json分析assistant_accounts_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_assistant IS 'DWD 维度表dim_assistant。ODS 来源表billiards_ods.assistant_accounts_master对应 JSONassistant_accounts_master.json分析assistant_accounts_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -314,7 +314,7 @@ CREATE TABLE IF NOT EXISTS dim_assistant_Ex (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (assistant_id) PRIMARY KEY (assistant_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_assistant_ex IS 'DWD 维度表扩展字段表dim_assistant_ex。ODS 来源表billiards_ods.assistant_accounts_master对应 JSONassistant_accounts_master.json分析assistant_accounts_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_assistant_ex IS 'DWD 维度表扩展字段表dim_assistant_ex。ODS 来源表billiards_ods.assistant_accounts_master对应 JSONassistant_accounts_master.json分析assistant_accounts_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -383,7 +383,7 @@ CREATE TABLE IF NOT EXISTS dim_member (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (member_id) PRIMARY KEY (member_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_member IS 'DWD 维度表dim_member。ODS 来源表billiards_ods.member_profiles对应 JSONmember_profiles.json分析member_profiles-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_member IS 'DWD 维度表dim_member。ODS 来源表billiards_ods.member_profiles对应 JSONmember_profiles.json分析member_profiles-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -415,7 +415,7 @@ CREATE TABLE IF NOT EXISTS dim_member_Ex (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (member_id) PRIMARY KEY (member_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_member_ex IS 'DWD 维度表扩展字段表dim_member_ex。ODS 来源表billiards_ods.member_profiles对应 JSONmember_profiles.json分析member_profiles-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_member_ex IS 'DWD 维度表扩展字段表dim_member_ex。ODS 来源表billiards_ods.member_profiles对应 JSONmember_profiles.json分析member_profiles-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -454,7 +454,7 @@ CREATE TABLE IF NOT EXISTS dim_member_card_account (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (member_card_id) PRIMARY KEY (member_card_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_member_card_account IS 'DWD 维度表dim_member_card_account。ODS 来源表billiards_ods.member_stored_value_cards对应 JSONmember_stored_value_cards.json分析member_stored_value_cards-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_member_card_account IS 'DWD 维度表dim_member_card_account。ODS 来源表billiards_ods.member_stored_value_cards对应 JSONmember_stored_value_cards.json分析member_stored_value_cards-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -538,7 +538,7 @@ CREATE TABLE IF NOT EXISTS dim_member_card_account_Ex (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (member_card_id) PRIMARY KEY (member_card_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_member_card_account_ex IS 'DWD 维度表扩展字段表dim_member_card_account_ex。ODS 来源表billiards_ods.member_stored_value_cards对应 JSONmember_stored_value_cards.json分析member_stored_value_cards-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_member_card_account_ex IS 'DWD 维度表扩展字段表dim_member_card_account_ex。ODS 来源表billiards_ods.member_stored_value_cards对应 JSONmember_stored_value_cards.json分析member_stored_value_cards-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -619,7 +619,7 @@ CREATE TABLE IF NOT EXISTS dim_tenant_goods (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (tenant_goods_id) PRIMARY KEY (tenant_goods_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_tenant_goods IS 'DWD 维度表dim_tenant_goods。ODS 来源表billiards_ods.tenant_goods_master对应 JSONtenant_goods_master.json分析tenant_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_tenant_goods IS 'DWD 维度表dim_tenant_goods。ODS 来源表billiards_ods.tenant_goods_master对应 JSONtenant_goods_master.json分析tenant_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -666,7 +666,7 @@ CREATE TABLE IF NOT EXISTS dim_tenant_goods_Ex (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (tenant_goods_id) PRIMARY KEY (tenant_goods_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_tenant_goods_ex IS 'DWD 维度表扩展字段表dim_tenant_goods_ex。ODS 来源表billiards_ods.tenant_goods_master对应 JSONtenant_goods_master.json分析tenant_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_tenant_goods_ex IS 'DWD 维度表扩展字段表dim_tenant_goods_ex。ODS 来源表billiards_ods.tenant_goods_master对应 JSONtenant_goods_master.json分析tenant_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -719,7 +719,7 @@ CREATE TABLE IF NOT EXISTS dim_store_goods (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (site_goods_id) PRIMARY KEY (site_goods_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_store_goods IS 'DWD 维度表dim_store_goods。ODS 来源表billiards_ods.store_goods_master对应 JSONstore_goods_master.json分析store_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_store_goods IS 'DWD 维度表dim_store_goods。ODS 来源表billiards_ods.store_goods_master对应 JSONstore_goods_master.json分析store_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -780,7 +780,7 @@ CREATE TABLE IF NOT EXISTS dim_store_goods_Ex (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (site_goods_id) PRIMARY KEY (site_goods_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_store_goods_ex IS 'DWD 维度表扩展字段表dim_store_goods_ex。ODS 来源表billiards_ods.store_goods_master对应 JSONstore_goods_master.json分析store_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_store_goods_ex IS 'DWD 维度表扩展字段表dim_store_goods_ex。ODS 来源表billiards_ods.store_goods_master对应 JSONstore_goods_master.json分析store_goods_master-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -833,7 +833,7 @@ CREATE TABLE IF NOT EXISTS dim_goods_category (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (category_id) PRIMARY KEY (category_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_goods_category IS 'DWD 维度表dim_goods_category。ODS 来源表billiards_ods.stock_goods_category_tree对应 JSONstock_goods_category_tree.json分析stock_goods_category_tree-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_goods_category IS 'DWD 维度表dim_goods_category。ODS 来源表billiards_ods.stock_goods_category_tree对应 JSONstock_goods_category_tree.json分析stock_goods_category_tree-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -876,7 +876,7 @@ CREATE TABLE IF NOT EXISTS dim_groupbuy_package (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (groupbuy_package_id) PRIMARY KEY (groupbuy_package_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_groupbuy_package IS 'DWD 维度表dim_groupbuy_package。ODS 来源表billiards_ods.group_buy_packages对应 JSONgroup_buy_packages.json分析group_buy_packages-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_groupbuy_package IS 'DWD 维度表dim_groupbuy_package。ODS 来源表billiards_ods.group_buy_packages对应 JSONgroup_buy_packages.json分析group_buy_packages-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';
@@ -927,7 +927,7 @@ CREATE TABLE IF NOT EXISTS dim_groupbuy_package_Ex (
SCD2_end_time TIMESTAMPTZ, SCD2_end_time TIMESTAMPTZ,
SCD2_is_current INT, SCD2_is_current INT,
SCD2_version INT, SCD2_version INT,
PRIMARY KEY (groupbuy_package_id) PRIMARY KEY (groupbuy_package_id, scd2_start_time)
); );
COMMENT ON TABLE billiards_dwd.dim_groupbuy_package_ex IS 'DWD 维度表扩展字段表dim_groupbuy_package_ex。ODS 来源表billiards_ods.group_buy_packages对应 JSONgroup_buy_packages.json分析group_buy_packages-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask'; COMMENT ON TABLE billiards_dwd.dim_groupbuy_package_ex IS 'DWD 维度表扩展字段表dim_groupbuy_package_ex。ODS 来源表billiards_ods.group_buy_packages对应 JSONgroup_buy_packages.json分析group_buy_packages-Analysis.md。装载/清洗逻辑参考etl_billiards/tasks/dwd_load_task.pyDwdLoadTask';

Binary file not shown.

Some files were not shown because too many files have changed in this diff Show More