root/feiqiu-ETL
Template
1
0
Fork 0
Code Pull Requests Packages Releases Wiki Activity

更新数据库文档 20260201-2

Browse Source
This commit is contained in:
Neo
2026-02-01 23:42:18 +08:00
parent 9b2c2c5c78
commit 294c6edbc9
47 changed files with 961 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
etl_billiards/gui/widgets/task_panel.py
View File

@@ -163,7 +163,7 @@ class TaskPanel(QWidget):
sync_layout.setSpacing(8)
# 说明
sync_desc = QLabel("从 API 抓取最新数据到 ODS，然后装载到 DWD。适用于日常增量更新。")
sync_desc = QLabel("从 API 抓取最新数据到 ODS，然后装载到 DWD。适用于日常增量更新、数据修复和历史补全。")
sync_desc.setProperty("subheading", True)
sync_desc.setWordWrap(True)
sync_layout.addWidget(sync_desc)
@@ -202,7 +202,7 @@ class TaskPanel(QWidget):
show_dimensions=True,
show_facts=True,
default_select_facts=True,
default_select_dimensions=False,
default_select_dimensions=True, # 默认包含维度表
compact=True,
max_height=200,
)
@@ -242,7 +242,7 @@ class TaskPanel(QWidget):
reload_layout.setSpacing(8)
# 说明
reload_desc = QLabel("指定时间窗口，重新抓取 ODS 数据并装载到 DWD。适用于数据修复或历史回补。")
reload_desc = QLabel("指定精确时间窗口，重新抓取 ODS 数据并装载到 DWD。适用于大范围历史数据重刷、跨月补跑等场景。")
reload_desc.setProperty("subheading", True)
reload_desc.setWordWrap(True)
reload_layout.addWidget(reload_desc)

72
etl_billiards/gui/workers/task_worker.py
View File

@@ -189,114 +189,60 @@ class TaskWorker(QThread):
# 解析 DWD 装载统计
if 'tables' in stats:
total_dim_inserted = 0
total_dim_updated = 0
total_fact_inserted = 0
total_fact_updated = 0
dim_tables = [] # ?????
fact_tables = [] # ?????
dim_tables = [] # 维表明细
fact_tables = [] # 事实表明细
for tbl in stats['tables']:
table_name = tbl.get('table', '').replace('billiards_dwd.', '')
mode = tbl.get('mode', '')
processed = int(tbl.get('processed', 0) or 0)
inserted = int(tbl.get('inserted', 0) or 0)
updated = int(tbl.get('updated', 0) or 0)
has_new_counts = ('inserted' in tbl) or ('updated' in tbl)
# ?? _ex ?????????
# 忽略 _ex 扩展表
if table_name.endswith('_ex'):
continue
is_dim = table_name.startswith('dim_') or mode == 'SCD2'
if is_dim:
if has_new_counts:
total_dim_inserted += inserted
total_dim_updated += updated
if inserted or updated:
dim_tables.append(f"{table_name}: +{inserted}, ~{updated}")
elif processed > 0:
total_dim_updated += processed
dim_tables.append(f"{table_name}: {processed}")
else:
if has_new_counts:
total_fact_inserted += inserted
total_fact_updated += updated
if inserted or updated:
fact_tables.append(f"{table_name}: +{inserted}, ~{updated}")
elif processed > 0 or inserted > 0:
total_fact_inserted += inserted
if inserted > 0:
fact_tables.append(f"{table_name}: +{inserted}")
if (total_dim_inserted or total_dim_updated or total_fact_inserted or total_fact_updated):
dwd_stats.append(
f"????: {total_dim_inserted}?, ????: {total_dim_updated}?, "
f"????: {total_fact_inserted}?, ????: {total_fact_updated}?"
f"维表新增: {total_dim_inserted}条, 维表更新: {total_dim_updated}条, "
f"事实表新增: {total_fact_inserted}条, 事实表更新: {total_fact_updated}条"
)
# ???????
# 维表明细
if dim_tables:
dwd_stats.append(" 维表: " + ", ".join(dim_tables))
dwd_stats.append(" ???: " + ", ".join(dim_tables))
# ???????
# 事实表明细
if fact_tables:
dwd_stats.append(" ???: " + ", ".join(fact_tables))
dwd_stats.append(" 事实表: " + ", ".join(fact_tables))