293 lines
10 KiB
Python
293 lines
10 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""统一任务日志器
|
|
|
|
提供统一的日志输出格式，支持：
|
|
- 任务开始/结束记录
|
|
- 进度追踪
|
|
- 统计计数
|
|
- 格式化的任务总结
|
|
"""
|
|
|
|
import logging
|
|
import time
|
|
from datetime import datetime
|
|
from typing import Any, Dict, Optional
|
|
|
|
|
|
# 统一日志格式
|
|
UNIFIED_LOG_FORMAT = "[%(asctime)s] %(levelname)-5s | %(name)s | %(message)s"
|
|
UNIFIED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
|
|
|
|
|
class TaskLogger:
|
|
"""任务日志器，统一 print 和 logging 输出"""
|
|
|
|
def __init__(
|
|
self,
|
|
task_code: str,
|
|
logger: Optional[logging.Logger] = None,
|
|
):
|
|
"""
|
|
初始化任务日志器
|
|
|
|
Args:
|
|
task_code: 任务代码
|
|
logger: 底层日志器，如果不提供则创建新的
|
|
"""
|
|
self.task_code = task_code
|
|
self.logger = logger or logging.getLogger(f"task.{task_code}")
|
|
|
|
# 任务状态
|
|
self.start_time: Optional[datetime] = None
|
|
self.end_time: Optional[datetime] = None
|
|
self.status: str = "pending"
|
|
|
|
# 统计计数
|
|
self.counts: Dict[str, int] = {
|
|
"fetched": 0,
|
|
"inserted": 0,
|
|
"updated": 0,
|
|
"skipped": 0,
|
|
"errors": 0,
|
|
}
|
|
|
|
# 额外信息
|
|
self.extra_info: Dict[str, Any] = {}
|
|
|
|
# 校验结果（如果有）
|
|
self.verification_result: Optional[dict] = None
|
|
|
|
def start(self, message: str = "任务开始"):
|
|
"""
|
|
记录任务开始
|
|
|
|
Args:
|
|
message: 开始消息
|
|
"""
|
|
self.start_time = datetime.now()
|
|
self.status = "running"
|
|
self.logger.info(
|
|
"%s | %s | 开始时间: %s",
|
|
self.task_code, message,
|
|
self.start_time.strftime(UNIFIED_DATE_FORMAT)
|
|
)
|
|
|
|
def progress(self, message: str, **kwargs):
|
|
"""
|
|
记录进度
|
|
|
|
Args:
|
|
message: 进度消息
|
|
**kwargs: 额外的统计信息
|
|
"""
|
|
# 更新计数
|
|
for key, value in kwargs.items():
|
|
if key in self.counts:
|
|
if isinstance(value, int):
|
|
self.counts[key] += value
|
|
else:
|
|
self.counts[key] = value
|
|
else:
|
|
self.extra_info[key] = value
|
|
|
|
# 构建进度字符串
|
|
counts_str = ", ".join(f"{k}={v}" for k, v in self.counts.items() if v > 0)
|
|
if counts_str:
|
|
self.logger.info("%s | %s | %s", self.task_code, message, counts_str)
|
|
else:
|
|
self.logger.info("%s | %s", self.task_code, message)
|
|
|
|
def info(self, message: str, *args):
|
|
"""记录信息级别日志"""
|
|
if args:
|
|
self.logger.info(f"{self.task_code} | {message}", *args)
|
|
else:
|
|
self.logger.info(f"{self.task_code} | {message}")
|
|
|
|
def warning(self, message: str, *args):
|
|
"""记录警告级别日志"""
|
|
if args:
|
|
self.logger.warning(f"{self.task_code} | {message}", *args)
|
|
else:
|
|
self.logger.warning(f"{self.task_code} | {message}")
|
|
|
|
def error(self, message: str, *args, exc_info: bool = False):
|
|
"""记录错误级别日志"""
|
|
self.counts["errors"] += 1
|
|
if args:
|
|
self.logger.error(f"{self.task_code} | {message}", *args, exc_info=exc_info)
|
|
else:
|
|
self.logger.error(f"{self.task_code} | {message}", exc_info=exc_info)
|
|
|
|
def set_counts(self, **counts):
|
|
"""直接设置计数"""
|
|
for key, value in counts.items():
|
|
if key in self.counts:
|
|
self.counts[key] = value
|
|
|
|
def add_counts(self, **counts):
|
|
"""累加计数"""
|
|
for key, value in counts.items():
|
|
if key in self.counts:
|
|
self.counts[key] += value
|
|
|
|
def set_verification_result(self, result: dict):
|
|
"""设置校验结果"""
|
|
self.verification_result = result
|
|
|
|
def end(self, status: str = "成功", error_message: Optional[str] = None) -> str:
|
|
"""
|
|
记录任务结束，返回格式化的总结
|
|
|
|
Args:
|
|
status: 状态 ("成功" / "失败" / "取消")
|
|
error_message: 错误信息（如果失败）
|
|
|
|
Returns:
|
|
格式化的任务总结字符串
|
|
"""
|
|
self.end_time = datetime.now()
|
|
self.status = status
|
|
|
|
# 计算耗时
|
|
if self.start_time:
|
|
elapsed = (self.end_time - self.start_time).total_seconds()
|
|
elapsed_str = self._format_duration(elapsed)
|
|
else:
|
|
elapsed = 0
|
|
elapsed_str = "-"
|
|
|
|
# 生成总结
|
|
summary = self._format_summary(status, elapsed_str, error_message)
|
|
|
|
# 记录日志
|
|
if status == "成功":
|
|
self.logger.info("\n%s", summary)
|
|
else:
|
|
self.logger.error("\n%s", summary)
|
|
|
|
return summary
|
|
|
|
def _format_duration(self, seconds: float) -> str:
|
|
"""格式化时长"""
|
|
if seconds < 60:
|
|
return f"{seconds:.1f}秒"
|
|
elif seconds < 3600:
|
|
mins = int(seconds // 60)
|
|
secs = seconds % 60
|
|
return f"{mins}分{secs:.0f}秒"
|
|
else:
|
|
hours = int(seconds // 3600)
|
|
mins = int((seconds % 3600) // 60)
|
|
return f"{hours}时{mins}分"
|
|
|
|
def _format_summary(
|
|
self,
|
|
status: str,
|
|
elapsed_str: str,
|
|
error_message: Optional[str] = None,
|
|
) -> str:
|
|
"""格式化任务总结"""
|
|
lines = [
|
|
"╔══════════════════════════════════════════════════════════════╗",
|
|
"║ 任务执行总结 ║",
|
|
"╠══════════════════════════════════════════════════════════════╣",
|
|
f"║ 任务代码: {self.task_code:<50} ║",
|
|
f"║ 执行状态: {status:<50} ║",
|
|
]
|
|
|
|
if self.start_time and self.end_time:
|
|
time_range = f"{self.start_time.strftime('%Y-%m-%d %H:%M:%S')} ~ {self.end_time.strftime('%H:%M:%S')} ({elapsed_str})"
|
|
lines.append(f"║ 执行时间: {time_range:<50} ║")
|
|
|
|
lines.extend([
|
|
"╠══════════════════════════════════════════════════════════════╣",
|
|
"║ 数据统计 ║",
|
|
f"║ - 获取记录: {self.counts['fetched']:>10,} ║",
|
|
f"║ - 新增记录: {self.counts['inserted']:>10,} ║",
|
|
f"║ - 更新记录: {self.counts['updated']:>10,} ║",
|
|
f"║ - 跳过记录: {self.counts['skipped']:>10,} ║",
|
|
f"║ - 错误记录: {self.counts['errors']:>10,} ║",
|
|
])
|
|
|
|
# 校验结果
|
|
if self.verification_result:
|
|
backfilled_missing = self.verification_result.get(
|
|
"backfilled_missing_count",
|
|
self.verification_result.get("backfilled_count", 0),
|
|
)
|
|
backfilled_mismatch = self.verification_result.get("backfilled_mismatch_count", 0)
|
|
lines.extend([
|
|
"╠══════════════════════════════════════════════════════════════╣",
|
|
"║ 校验结果 ║",
|
|
f"║ - 源数据量: {self.verification_result.get('source_count', 0):>10,} ║",
|
|
f"║ - 目标数据量: {self.verification_result.get('target_count', 0):>10,} ║",
|
|
f"║ - 缺失补齐: {backfilled_missing:>10,} ║",
|
|
f"║ - 不一致补齐: {backfilled_mismatch:>10,} ║",
|
|
])
|
|
|
|
# 错误信息
|
|
if error_message:
|
|
lines.extend([
|
|
"╠══════════════════════════════════════════════════════════════╣",
|
|
f"║ 错误信息: {error_message[:50]:<50} ║",
|
|
])
|
|
|
|
lines.append("╚══════════════════════════════════════════════════════════════╝")
|
|
|
|
return "\n".join(lines)
|
|
|
|
def get_result(self) -> dict:
|
|
"""获取任务结果字典"""
|
|
elapsed = 0
|
|
if self.start_time and self.end_time:
|
|
elapsed = (self.end_time - self.start_time).total_seconds()
|
|
|
|
return {
|
|
"task_code": self.task_code,
|
|
"status": self.status,
|
|
"start_time": self.start_time.isoformat() if self.start_time else None,
|
|
"end_time": self.end_time.isoformat() if self.end_time else None,
|
|
"elapsed_seconds": elapsed,
|
|
"counts": self.counts.copy(),
|
|
"extra_info": self.extra_info.copy(),
|
|
"verification_result": self.verification_result,
|
|
}
|
|
|
|
|
|
def configure_task_logging(
|
|
name: str = "fq_etl",
|
|
level: str = "INFO",
|
|
) -> logging.Logger:
|
|
"""
|
|
配置任务日志
|
|
|
|
Args:
|
|
name: 日志器名称
|
|
level: 日志级别
|
|
|
|
Returns:
|
|
配置好的日志器
|
|
"""
|
|
logger = logging.getLogger(name)
|
|
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
|
|
|
# 清除已有处理器
|
|
logger.handlers.clear()
|
|
|
|
# 添加控制台处理器
|
|
handler = logging.StreamHandler()
|
|
handler.setLevel(logging.DEBUG)
|
|
|
|
# 设置格式
|
|
formatter = logging.Formatter(
|
|
UNIFIED_LOG_FORMAT,
|
|
UNIFIED_DATE_FORMAT,
|
|
)
|
|
handler.setFormatter(formatter)
|
|
|
|
logger.addHandler(handler)
|
|
logger.propagate = False
|
|
|
|
return logger
|