改 相对路径 完成客户端

This commit is contained in:
Neo
2026-01-27 22:14:01 +08:00
parent 04c064793a
commit 9f8976e75a
292 changed files with 307062 additions and 678 deletions

View File

@@ -240,9 +240,8 @@ class MissingDataBackfiller:
headers_extra=cfg["api"].get("headers_extra") or {},
)
# 数据库连接
# 数据库连接DatabaseConnection 构造时已设置 autocommit=False
self.db = DatabaseConnection(dsn=cfg["db"]["dsn"], session=cfg["db"].get("session"))
self.db.conn.autocommit = False
def close(self):
"""关闭连接"""
@@ -264,7 +263,7 @@ class MissingDataBackfiller:
Returns:
补全结果统计
"""
self.logger.info("BACKFILL_START start=%s end=%s", start.isoformat(), end.isoformat())
self.logger.info("数据补全开始 起始=%s 结束=%s", start.isoformat(), end.isoformat())
# 计算窗口大小
total_seconds = max(0, int((end - start).total_seconds()))
@@ -276,7 +275,7 @@ class MissingDataBackfiller:
window_hours = max(1, total_seconds // 3600 or 1)
# 运行 gap check
self.logger.info("RUNNING_GAP_CHECK...")
self.logger.info("正在执行缺失检查...")
gap_result = run_gap_check(
cfg=self.cfg,
start=start,
@@ -297,10 +296,10 @@ class MissingDataBackfiller:
total_missing = gap_result.get("total_missing", 0)
if total_missing == 0:
self.logger.info("NO_MISSING_DATA")
self.logger.info("数据完整,无缺失记录")
return {"backfilled": 0, "errors": 0, "details": []}
self.logger.info("GAP_CHECK_DONE total_missing=%s", total_missing)
self.logger.info("缺失检查完成 总缺失=%s", total_missing)
# 补全每个任务的丢失数据
results = []
@@ -316,7 +315,7 @@ class MissingDataBackfiller:
continue
self.logger.info(
"BACKFILL_TASK task=%s missing=%s samples=%s",
"开始补全任务 任务=%s 缺失=%s 样本数=%s",
task_code, missing, len(missing_samples)
)
@@ -339,7 +338,7 @@ class MissingDataBackfiller:
})
total_backfilled += backfilled
except Exception as exc:
self.logger.exception("BACKFILL_ERROR task=%s", task_code)
self.logger.exception("补全失败 任务=%s", task_code)
results.append({
"task_code": task_code,
"missing": missing,
@@ -349,7 +348,7 @@ class MissingDataBackfiller:
total_errors += 1
self.logger.info(
"BACKFILL_DONE total_missing=%s backfilled=%s errors=%s",
"数据补全完成 总缺失=%s 已补全=%s 错误数=%s",
total_missing, total_backfilled, total_errors
)
@@ -375,14 +374,14 @@ class MissingDataBackfiller:
"""补全单个任务的丢失数据"""
spec = _get_spec(task_code)
if not spec:
self.logger.warning("SPEC_NOT_FOUND task=%s", task_code)
self.logger.warning("未找到任务规格 任务=%s", task_code)
return 0
if not pk_columns:
pk_columns = _get_table_pk_columns(self.db.conn, table)
if not pk_columns:
self.logger.warning("NO_PK_COLUMNS task=%s table=%s", task_code, table)
self.logger.warning("未找到主键列 任务=%s =%s", task_code, table)
return 0
# 提取丢失的 PK 值
@@ -393,11 +392,11 @@ class MissingDataBackfiller:
missing_pks.add(pk_tuple)
if not missing_pks:
self.logger.info("NO_MISSING_PKS task=%s", task_code)
self.logger.info("无缺失主键 任务=%s", task_code)
return 0
self.logger.info(
"BACKFILL_FETCHING task=%s missing_pks=%s",
"开始获取数据 任务=%s 缺失主键数=%s",
task_code, len(missing_pks)
)
@@ -436,7 +435,7 @@ class MissingDataBackfiller:
if self.dry_run:
backfilled += len(records_to_insert)
self.logger.info(
"DRY_RUN task=%s page=%s would_insert=%s",
"模拟运行 任务=%s =%s 将插入=%s",
task_code, page_no, len(records_to_insert)
)
else:
@@ -449,14 +448,14 @@ class MissingDataBackfiller:
)
backfilled += inserted
self.logger.info(
"INSERTED task=%s page=%s count=%s",
"已插入 任务=%s =%s 数量=%s",
task_code, page_no, inserted
)
if not self.dry_run:
self.db.conn.commit()
self.logger.info("BACKFILL_TASK_DONE task=%s backfilled=%s", task_code, backfilled)
self.logger.info("任务补全完成 任务=%s 已补全=%s", task_code, backfilled)
return backfilled
except Exception: