改 相对路径 完成客户端
This commit is contained in:
@@ -240,9 +240,8 @@ class MissingDataBackfiller:
|
||||
headers_extra=cfg["api"].get("headers_extra") or {},
|
||||
)
|
||||
|
||||
# 数据库连接
|
||||
# 数据库连接(DatabaseConnection 构造时已设置 autocommit=False)
|
||||
self.db = DatabaseConnection(dsn=cfg["db"]["dsn"], session=cfg["db"].get("session"))
|
||||
self.db.conn.autocommit = False
|
||||
|
||||
def close(self):
|
||||
"""关闭连接"""
|
||||
@@ -264,7 +263,7 @@ class MissingDataBackfiller:
|
||||
Returns:
|
||||
补全结果统计
|
||||
"""
|
||||
self.logger.info("BACKFILL_START start=%s end=%s", start.isoformat(), end.isoformat())
|
||||
self.logger.info("数据补全开始 起始=%s 结束=%s", start.isoformat(), end.isoformat())
|
||||
|
||||
# 计算窗口大小
|
||||
total_seconds = max(0, int((end - start).total_seconds()))
|
||||
@@ -276,7 +275,7 @@ class MissingDataBackfiller:
|
||||
window_hours = max(1, total_seconds // 3600 or 1)
|
||||
|
||||
# 运行 gap check
|
||||
self.logger.info("RUNNING_GAP_CHECK...")
|
||||
self.logger.info("正在执行缺失检查...")
|
||||
gap_result = run_gap_check(
|
||||
cfg=self.cfg,
|
||||
start=start,
|
||||
@@ -297,10 +296,10 @@ class MissingDataBackfiller:
|
||||
|
||||
total_missing = gap_result.get("total_missing", 0)
|
||||
if total_missing == 0:
|
||||
self.logger.info("NO_MISSING_DATA")
|
||||
self.logger.info("数据完整,无缺失记录")
|
||||
return {"backfilled": 0, "errors": 0, "details": []}
|
||||
|
||||
self.logger.info("GAP_CHECK_DONE total_missing=%s", total_missing)
|
||||
self.logger.info("缺失检查完成 总缺失=%s", total_missing)
|
||||
|
||||
# 补全每个任务的丢失数据
|
||||
results = []
|
||||
@@ -316,7 +315,7 @@ class MissingDataBackfiller:
|
||||
continue
|
||||
|
||||
self.logger.info(
|
||||
"BACKFILL_TASK task=%s missing=%s samples=%s",
|
||||
"开始补全任务 任务=%s 缺失=%s 样本数=%s",
|
||||
task_code, missing, len(missing_samples)
|
||||
)
|
||||
|
||||
@@ -339,7 +338,7 @@ class MissingDataBackfiller:
|
||||
})
|
||||
total_backfilled += backfilled
|
||||
except Exception as exc:
|
||||
self.logger.exception("BACKFILL_ERROR task=%s", task_code)
|
||||
self.logger.exception("补全失败 任务=%s", task_code)
|
||||
results.append({
|
||||
"task_code": task_code,
|
||||
"missing": missing,
|
||||
@@ -349,7 +348,7 @@ class MissingDataBackfiller:
|
||||
total_errors += 1
|
||||
|
||||
self.logger.info(
|
||||
"BACKFILL_DONE total_missing=%s backfilled=%s errors=%s",
|
||||
"数据补全完成 总缺失=%s 已补全=%s 错误数=%s",
|
||||
total_missing, total_backfilled, total_errors
|
||||
)
|
||||
|
||||
@@ -375,14 +374,14 @@ class MissingDataBackfiller:
|
||||
"""补全单个任务的丢失数据"""
|
||||
spec = _get_spec(task_code)
|
||||
if not spec:
|
||||
self.logger.warning("SPEC_NOT_FOUND task=%s", task_code)
|
||||
self.logger.warning("未找到任务规格 任务=%s", task_code)
|
||||
return 0
|
||||
|
||||
if not pk_columns:
|
||||
pk_columns = _get_table_pk_columns(self.db.conn, table)
|
||||
|
||||
if not pk_columns:
|
||||
self.logger.warning("NO_PK_COLUMNS task=%s table=%s", task_code, table)
|
||||
self.logger.warning("未找到主键列 任务=%s 表=%s", task_code, table)
|
||||
return 0
|
||||
|
||||
# 提取丢失的 PK 值
|
||||
@@ -393,11 +392,11 @@ class MissingDataBackfiller:
|
||||
missing_pks.add(pk_tuple)
|
||||
|
||||
if not missing_pks:
|
||||
self.logger.info("NO_MISSING_PKS task=%s", task_code)
|
||||
self.logger.info("无缺失主键 任务=%s", task_code)
|
||||
return 0
|
||||
|
||||
self.logger.info(
|
||||
"BACKFILL_FETCHING task=%s missing_pks=%s",
|
||||
"开始获取数据 任务=%s 缺失主键数=%s",
|
||||
task_code, len(missing_pks)
|
||||
)
|
||||
|
||||
@@ -436,7 +435,7 @@ class MissingDataBackfiller:
|
||||
if self.dry_run:
|
||||
backfilled += len(records_to_insert)
|
||||
self.logger.info(
|
||||
"DRY_RUN task=%s page=%s would_insert=%s",
|
||||
"模拟运行 任务=%s 页=%s 将插入=%s",
|
||||
task_code, page_no, len(records_to_insert)
|
||||
)
|
||||
else:
|
||||
@@ -449,14 +448,14 @@ class MissingDataBackfiller:
|
||||
)
|
||||
backfilled += inserted
|
||||
self.logger.info(
|
||||
"INSERTED task=%s page=%s count=%s",
|
||||
"已插入 任务=%s 页=%s 数量=%s",
|
||||
task_code, page_no, inserted
|
||||
)
|
||||
|
||||
if not self.dry_run:
|
||||
self.db.conn.commit()
|
||||
|
||||
self.logger.info("BACKFILL_TASK_DONE task=%s backfilled=%s", task_code, backfilled)
|
||||
self.logger.info("任务补全完成 任务=%s 已补全=%s", task_code, backfilled)
|
||||
return backfilled
|
||||
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user