在准备环境前提交次全部更改。
This commit is contained in:
292
apps/etl/connectors/feiqiu/utils/task_logger.py
Normal file
292
apps/etl/connectors/feiqiu/utils/task_logger.py
Normal file
@@ -0,0 +1,292 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""统一任务日志器
|
||||
|
||||
提供统一的日志输出格式，支持：
|
||||
- 任务开始/结束记录
|
||||
- 进度追踪
|
||||
- 统计计数
|
||||
- 格式化的任务总结
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
|
||||
# 统一日志格式
|
||||
UNIFIED_LOG_FORMAT = "[%(asctime)s] %(levelname)-5s | %(name)s | %(message)s"
|
||||
UNIFIED_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
|
||||
class TaskLogger:
|
||||
"""任务日志器，统一 print 和 logging 输出"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task_code: str,
|
||||
logger: Optional[logging.Logger] = None,
|
||||
):
|
||||
"""
|
||||
初始化任务日志器
|
||||
|
||||
Args:
|
||||
task_code: 任务代码
|
||||
logger: 底层日志器，如果不提供则创建新的
|
||||
"""
|
||||
self.task_code = task_code
|
||||
self.logger = logger or logging.getLogger(f"task.{task_code}")
|
||||
|
||||
# 任务状态
|
||||
self.start_time: Optional[datetime] = None
|
||||
self.end_time: Optional[datetime] = None
|
||||
self.status: str = "pending"
|
||||
|
||||
# 统计计数
|
||||
self.counts: Dict[str, int] = {
|
||||
"fetched": 0,
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"skipped": 0,
|
||||
"errors": 0,
|
||||
}
|
||||
|
||||
# 额外信息
|
||||
self.extra_info: Dict[str, Any] = {}
|
||||
|
||||
# 校验结果（如果有）
|
||||
self.verification_result: Optional[dict] = None
|
||||
|
||||
def start(self, message: str = "任务开始"):
|
||||
"""
|
||||
记录任务开始
|
||||
|
||||
Args:
|
||||
message: 开始消息
|
||||
"""
|
||||
self.start_time = datetime.now()
|
||||
self.status = "running"
|
||||
self.logger.info(
|
||||
"%s | %s | 开始时间: %s",
|
||||
self.task_code, message,
|
||||
self.start_time.strftime(UNIFIED_DATE_FORMAT)
|
||||
)
|
||||
|
||||
def progress(self, message: str, **kwargs):
|
||||
"""
|
||||
记录进度
|
||||
|
||||
Args:
|
||||
message: 进度消息
|
||||
**kwargs: 额外的统计信息
|
||||
"""
|
||||
# 更新计数
|
||||
for key, value in kwargs.items():
|
||||
if key in self.counts:
|
||||
if isinstance(value, int):
|
||||
self.counts[key] += value
|
||||
else:
|
||||
self.counts[key] = value
|
||||
else:
|
||||
self.extra_info[key] = value
|
||||
|
||||
# 构建进度字符串
|
||||
counts_str = ", ".join(f"{k}={v}" for k, v in self.counts.items() if v > 0)
|
||||
if counts_str:
|
||||
self.logger.info("%s | %s | %s", self.task_code, message, counts_str)
|
||||
else:
|
||||
self.logger.info("%s | %s", self.task_code, message)
|
||||
|
||||
def info(self, message: str, *args):
|
||||
"""记录信息级别日志"""
|
||||
if args:
|
||||
self.logger.info(f"{self.task_code} | {message}", *args)
|
||||
else:
|
||||
self.logger.info(f"{self.task_code} | {message}")
|
||||
|
||||
def warning(self, message: str, *args):
|
||||
"""记录警告级别日志"""
|
||||
if args:
|
||||
self.logger.warning(f"{self.task_code} | {message}", *args)
|
||||
else:
|
||||
self.logger.warning(f"{self.task_code} | {message}")
|
||||
|
||||
def error(self, message: str, *args, exc_info: bool = False):
|
||||
"""记录错误级别日志"""
|
||||
self.counts["errors"] += 1
|
||||
if args:
|
||||
self.logger.error(f"{self.task_code} | {message}", *args, exc_info=exc_info)
|
||||
else:
|
||||
self.logger.error(f"{self.task_code} | {message}", exc_info=exc_info)
|
||||
|
||||
def set_counts(self, **counts):
|
||||
"""直接设置计数"""
|
||||
for key, value in counts.items():
|
||||
if key in self.counts:
|
||||
self.counts[key] = value
|
||||
|
||||
def add_counts(self, **counts):
|
||||
"""累加计数"""
|
||||
for key, value in counts.items():
|
||||
if key in self.counts:
|
||||
self.counts[key] += value
|
||||
|
||||
def set_verification_result(self, result: dict):
|
||||
"""设置校验结果"""
|
||||
self.verification_result = result
|
||||
|
||||
def end(self, status: str = "成功", error_message: Optional[str] = None) -> str:
|
||||
"""
|
||||
记录任务结束，返回格式化的总结
|
||||
|
||||
Args:
|
||||
status: 状态 ("成功" / "失败" / "取消")
|
||||
error_message: 错误信息（如果失败）
|
||||
|
||||
Returns:
|
||||
格式化的任务总结字符串
|
||||
"""
|
||||
self.end_time = datetime.now()
|
||||
self.status = status
|
||||
|
||||
# 计算耗时
|
||||
if self.start_time:
|
||||
elapsed = (self.end_time - self.start_time).total_seconds()
|
||||
elapsed_str = self._format_duration(elapsed)
|
||||
else:
|
||||
elapsed = 0
|
||||
elapsed_str = "-"
|
||||
|
||||
# 生成总结
|
||||
summary = self._format_summary(status, elapsed_str, error_message)
|
||||
|
||||
# 记录日志
|
||||
if status == "成功":
|
||||
self.logger.info("\n%s", summary)
|
||||
else:
|
||||
self.logger.error("\n%s", summary)
|
||||
|
||||
return summary
|
||||
|
||||
def _format_duration(self, seconds: float) -> str:
|
||||
"""格式化时长"""
|
||||
if seconds < 60:
|
||||
return f"{seconds:.1f}秒"
|
||||
elif seconds < 3600:
|
||||
mins = int(seconds // 60)
|
||||
secs = seconds % 60
|
||||
return f"{mins}分{secs:.0f}秒"
|
||||
else:
|
||||
hours = int(seconds // 3600)
|
||||
mins = int((seconds % 3600) // 60)
|
||||
return f"{hours}时{mins}分"
|
||||
|
||||
def _format_summary(
|
||||
self,
|
||||
status: str,
|
||||
elapsed_str: str,
|
||||
error_message: Optional[str] = None,
|
||||
) -> str:
|
||||
"""格式化任务总结"""
|
||||
lines = [
|
||||
"╔══════════════════════════════════════════════════════════════╗",
|
||||
"║ 任务执行总结 ║",
|
||||
"╠══════════════════════════════════════════════════════════════╣",
|
||||
f"║ 任务代码: {self.task_code:<50} ║",
|
||||
f"║ 执行状态: {status:<50} ║",
|
||||
]
|
||||
|
||||
if self.start_time and self.end_time:
|
||||
time_range = f"{self.start_time.strftime('%Y-%m-%d %H:%M:%S')} ~ {self.end_time.strftime('%H:%M:%S')} ({elapsed_str})"
|
||||
lines.append(f"║ 执行时间: {time_range:<50} ║")
|
||||
|
||||
lines.extend([
|
||||
"╠══════════════════════════════════════════════════════════════╣",
|
||||
"║ 数据统计 ║",
|
||||
f"║ - 获取记录: {self.counts['fetched']:>10,} ║",
|
||||
f"║ - 新增记录: {self.counts['inserted']:>10,} ║",
|
||||
f"║ - 更新记录: {self.counts['updated']:>10,} ║",
|
||||
f"║ - 跳过记录: {self.counts['skipped']:>10,} ║",
|
||||
f"║ - 错误记录: {self.counts['errors']:>10,} ║",
|
||||
])
|
||||
|
||||
# 校验结果
|
||||
if self.verification_result:
|
||||
backfilled_missing = self.verification_result.get(
|
||||
"backfilled_missing_count",
|
||||
self.verification_result.get("backfilled_count", 0),
|
||||
)
|
||||
backfilled_mismatch = self.verification_result.get("backfilled_mismatch_count", 0)
|
||||
lines.extend([
|
||||
"╠══════════════════════════════════════════════════════════════╣",
|
||||
"║ 校验结果 ║",
|
||||
f"║ - 源数据量: {self.verification_result.get('source_count', 0):>10,} ║",
|
||||
f"║ - 目标数据量: {self.verification_result.get('target_count', 0):>10,} ║",
|
||||
f"║ - 缺失补齐: {backfilled_missing:>10,} ║",
|
||||
f"║ - 不一致补齐: {backfilled_mismatch:>10,} ║",
|
||||
])
|
||||
|
||||
# 错误信息
|
||||
if error_message:
|
||||
lines.extend([
|
||||
"╠══════════════════════════════════════════════════════════════╣",
|
||||
f"║ 错误信息: {error_message[:50]:<50} ║",
|
||||
])
|
||||
|
||||
lines.append("╚══════════════════════════════════════════════════════════════╝")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
def get_result(self) -> dict:
|
||||
"""获取任务结果字典"""
|
||||
elapsed = 0
|
||||
if self.start_time and self.end_time:
|
||||
elapsed = (self.end_time - self.start_time).total_seconds()
|
||||
|
||||
return {
|
||||
"task_code": self.task_code,
|
||||
"status": self.status,
|
||||
"start_time": self.start_time.isoformat() if self.start_time else None,
|
||||
"end_time": self.end_time.isoformat() if self.end_time else None,
|
||||
"elapsed_seconds": elapsed,
|
||||
"counts": self.counts.copy(),
|
||||
"extra_info": self.extra_info.copy(),
|
||||
"verification_result": self.verification_result,
|
||||
}
|
||||
|
||||
|
||||
def configure_task_logging(
|
||||
name: str = "fq_etl",
|
||||
level: str = "INFO",
|
||||
) -> logging.Logger:
|
||||
"""
|
||||
配置任务日志
|
||||
|
||||
Args:
|
||||
name: 日志器名称
|
||||
level: 日志级别
|
||||
|
||||
Returns:
|
||||
配置好的日志器
|
||||
"""
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
|
||||
# 清除已有处理器
|
||||
logger.handlers.clear()
|
||||
|
||||
# 添加控制台处理器
|
||||
handler = logging.StreamHandler()
|
||||
handler.setLevel(logging.DEBUG)
|
||||
|
||||
# 设置格式
|
||||
formatter = logging.Formatter(
|
||||
UNIFIED_LOG_FORMAT,
|
||||
UNIFIED_DATE_FORMAT,
|
||||
)
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
logger.addHandler(handler)
|
||||
logger.propagate = False
|
||||
|
||||
return logger
|
||||
Reference in New Issue
Block a user