为WinSW做准备
This commit is contained in:
9
app/etl_busy.py
Normal file
9
app/etl_busy.py
Normal file
@@ -0,0 +1,9 @@
|
||||
# app/etl_busy.py
|
||||
def run():
|
||||
"""
|
||||
忙时抓取逻辑。
|
||||
TODO: 这里写具体抓取流程(API 调用 / 网页解析 / 写入 PostgreSQL 等)
|
||||
"""
|
||||
print("Running busy-period ETL...")
|
||||
# 示例:后续在这里接 PostgreSQL 或 HTTP 抓取
|
||||
# ...
|
||||
8
app/etl_idle.py
Normal file
8
app/etl_idle.py
Normal file
@@ -0,0 +1,8 @@
|
||||
# app/etl_idle.py
|
||||
def run():
|
||||
"""
|
||||
闲时抓取逻辑。
|
||||
可以做全量同步、大批量历史修正等。
|
||||
"""
|
||||
print("Running idle-period ETL...")
|
||||
# ...
|
||||
31
app/runner.py
Normal file
31
app/runner.py
Normal file
@@ -0,0 +1,31 @@
|
||||
# app/runner.py
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
from . import etl_busy, etl_idle
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Feiqiu ETL Runner")
|
||||
parser.add_argument(
|
||||
"--mode",
|
||||
choices=["busy", "idle"],
|
||||
required=True,
|
||||
help="ETL mode: busy or idle",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
print(f"[{now}] Start ETL mode={args.mode}")
|
||||
|
||||
if args.mode == "busy":
|
||||
etl_busy.run()
|
||||
else:
|
||||
etl_idle.run()
|
||||
|
||||
print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] ETL finished.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
2
requirements.txt
Normal file
2
requirements.txt
Normal file
@@ -0,0 +1,2 @@
|
||||
requests
|
||||
psycopg2-binary
|
||||
Reference in New Issue
Block a user