From cbe48c8ee7f1cb0ca63ca02cefb9597f9d6f1411 Mon Sep 17 00:00:00 2001 From: Neo Date: Wed, 19 Nov 2025 05:05:11 +0800 Subject: [PATCH] =?UTF-8?q?=E4=B8=BAWinSW=E5=81=9A=E5=87=86=E5=A4=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/etl_busy.py | 9 +++++++++ app/etl_idle.py | 8 ++++++++ app/runner.py | 31 +++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 4 files changed, 50 insertions(+) create mode 100644 app/etl_busy.py create mode 100644 app/etl_idle.py create mode 100644 app/runner.py create mode 100644 requirements.txt diff --git a/app/etl_busy.py b/app/etl_busy.py new file mode 100644 index 0000000..c493046 --- /dev/null +++ b/app/etl_busy.py @@ -0,0 +1,9 @@ +# app/etl_busy.py +def run(): + """ + 忙时抓取逻辑。 + TODO: 这里写具体抓取流程(API 调用 / 网页解析 / 写入 PostgreSQL 等) + """ + print("Running busy-period ETL...") + # 示例:后续在这里接 PostgreSQL 或 HTTP 抓取 + # ... \ No newline at end of file diff --git a/app/etl_idle.py b/app/etl_idle.py new file mode 100644 index 0000000..06a4ac5 --- /dev/null +++ b/app/etl_idle.py @@ -0,0 +1,8 @@ +# app/etl_idle.py +def run(): + """ + 闲时抓取逻辑。 + 可以做全量同步、大批量历史修正等。 + """ + print("Running idle-period ETL...") + # ... \ No newline at end of file diff --git a/app/runner.py b/app/runner.py new file mode 100644 index 0000000..e2abc25 --- /dev/null +++ b/app/runner.py @@ -0,0 +1,31 @@ +# app/runner.py +import argparse +from datetime import datetime + +from . import etl_busy, etl_idle + + +def main(): + parser = argparse.ArgumentParser(description="Feiqiu ETL Runner") + parser.add_argument( + "--mode", + choices=["busy", "idle"], + required=True, + help="ETL mode: busy or idle", + ) + + args = parser.parse_args() + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + print(f"[{now}] Start ETL mode={args.mode}") + + if args.mode == "busy": + etl_busy.run() + else: + etl_idle.run() + + print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] ETL finished.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0b4b264 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +requests +psycopg2-binary