优化项目结构,添加注释

core/gether.py

@@ -0,0 +1,126 @@
+import os
+import traceback
+import concurrent.futures
+from utils import files, messages, clash, formats
+import lib
+from core import Config, types, goods
+
+def get_category_urls():
+    category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt"
+    # if os.path.exists(category_urls_path):
+    #     return
+    category_urls = lib.Server.get_category_urls()
+    files.save_list(category_urls_path, category_urls)
+
+def get_goods_urls(categorys_len: int = 3, home_mode: bool = False):
+    goods_urls_path = f"{lib.Server.__urls_folder__}/goods.txt"
+    category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt"
+    if home_mode:
+        goods_urls_path = goods_urls_path.replace('.txt', '(home).txt')
+        category_urls_path = category_urls_path.replace('.txt', '(home).txt')
+
+    category_index = {}
+    category_urls = files.load_lines(category_urls_path)
+
+    if os.path.exists(goods_urls_path):
+        old_goods_urls = files.load_lines(goods_urls_path)
+        for old_goods_url in old_goods_urls:
+            category = old_goods_url.split('#')[-1]
+            category_index[category] = None
+
+    index = 0
+    max_len = len(category_urls)
+    for category_url in category_urls:
+        category = category_url.split('#')[-1]
+        category_url = '#'.join(category_url.split('#')[:-1])
+
+        index += 1
+        if 'https' not in category_url:
+            continue
+        if Config.skip_category and home_mode == False and len(category.split('/')) < categorys_len:
+            continue
+        if category in category_index:
+            continue
+
+        try:
+            goods_urls = lib.Server.get_goods_urls(category_url)
+            goods_category_urls = []
+            for goods_url in goods_urls:
+                goods_category_urls.append(f"{goods_url}#{category}")
+
+            files.add_list(goods_urls_path, goods_category_urls)
+        except Exception as e:
+            files.add_text(f"{lib.Server.__errors_folder__}/category_errors.log", f"{category_url}:\n[{e}] {traceback.format_exc()}")
+            messages.sendError(str(e))
+        messages.sendInfo(f"{index}/{max_len} {len(goods_urls)}条")
+    if home_mode == False:
+        formats.de_repeat_urls(goods_urls_path)
+
+def get_goods_info():
+    p_urls = {}
+    def workers(goods_urls, index):
+        with concurrent.futures.ThreadPoolExecutor(max_workers=lib.Server.worker_num) as executor:
+            futures = []
+            for goods_url in goods_urls:
+                if goods_url in p_urls:
+                    continue
+                future = executor.submit(lib.Server.get_goods_info, goods_url)
+                futures.append(future)
+
+            results: list[types.GoodsInfo] = []
+            for future_index, future in enumerate(futures):
+                index += 1
+                try:
+                    result = future.result()  # 阻塞直到该任务完成
+                except Exception as e:
+                    try:
+                        files.add_text(f"{lib.Server.__errors_folder__}/goods_errors.log", f"{goods_urls[future_index]}:\n[{e}] {traceback.format_exc()}")
+                    except:
+                        pass
+                    messages.sendWarn(f"{goods_urls[future_index]} 出错")
+                    continue
+
+                results.append(result)
+                messages.sendInfo(f"{index}/{max_len}")
+
+            for goods_info in results:
+                try:
+                    GoodsModel.add_goods(goods_info)
+                    for p_url in goods_info.p_urls:
+                        p_urls[p_url] = None
+                except Exception as e:
+                    messages.sendWarn(f"{e}")
+
+        return index
+
+    goods_urls = files.load_lines(f"{lib.Server.__urls_folder__}/goods.txt")
+    messages.sendInfo('读取url完成')
+
+    messages.sendInfo('读取上次存档')
+    GoodsModel = goods.GoodsModel(lib.Server.__database_path__)
+    excel_index = GoodsModel.select_goods_to_dict()
+    messages.sendInfo('读取完成')
+
+    de_save_url = []
+    for goods_url in goods_urls:
+        if goods_url in excel_index:
+            continue
+        if 'http' not in goods_url:
+            continue
+        de_save_url.append(goods_url)
+    goods_urls = de_save_url
+    del excel_index
+
+    max_len = len(goods_urls)
+    index = 1
+    for i in range(0, len(goods_urls), lib.Server.bitch):
+        index = workers(goods_urls=goods_urls[i:i+lib.Server.bitch], index=index)
+        if lib.Server.switch_clash:
+            try:
+                clash.change_proxy()
+            except:
+                pass
+        if lib.Server.reflush_browsers:
+            if lib.Server.worker_num > 1:
+                lib.Server.reflush_browser()
+        messages.sendInfo("已保存")