132 lines
4.7 KiB
Python
132 lines
4.7 KiB
Python
import os
|
|
import traceback
|
|
import concurrent.futures
|
|
from utils import files, messages, clash, formats
|
|
import lib
|
|
from core import Config, types
|
|
from lib.model import goods
|
|
|
|
def get_category_urls():
|
|
category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt"
|
|
if os.path.exists(category_urls_path):
|
|
return
|
|
|
|
category_urls = lib.Server.get_category_urls()
|
|
for urls in category_urls:
|
|
if 'http' not in urls:
|
|
continue
|
|
files.save_line(category_urls_path, urls)
|
|
|
|
def get_goods_urls(categorys_len: int = 3):
|
|
goods_urls_path = f"{lib.Server.__urls_folder__}/goods.txt"
|
|
category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt"
|
|
if Config.home_mode:
|
|
goods_urls_path = goods_urls_path.replace('.txt', '(home).txt')
|
|
category_urls_path = category_urls_path.replace('.txt', '(home).txt')
|
|
|
|
category_index = {}
|
|
category_urls = files.load_lines(category_urls_path)
|
|
|
|
if os.path.exists(goods_urls_path):
|
|
old_goods_urls = files.load_lines(goods_urls_path)
|
|
for old_goods_url in old_goods_urls:
|
|
category = old_goods_url.split('#')[-1]
|
|
category_index[category] = None
|
|
|
|
index = 0
|
|
max_len = len(category_urls)
|
|
for category_url in category_urls:
|
|
category = category_url.split('#')[-1]
|
|
category_url = '#'.join(category_url.split('#')[:-1])
|
|
|
|
index += 1
|
|
if 'https' not in category_url:
|
|
continue
|
|
if Config.skip_category and Config.home_mode == False and len(category.split('/')) < categorys_len:
|
|
continue
|
|
if category in category_index:
|
|
continue
|
|
|
|
try:
|
|
goods_urls = lib.Server.get_goods_urls(category_url)
|
|
goods_category_urls = []
|
|
for goods_url in goods_urls:
|
|
goods_category_urls.append(f"{goods_url}#{category}")
|
|
|
|
files.add_list(goods_urls_path, goods_category_urls)
|
|
except Exception as e:
|
|
files.add_text(f"{lib.Server.__errors_folder__}/category_errors.log", f"{category_url}:\n[{e}] {traceback.format_exc()}")
|
|
messages.sendError(str(e))
|
|
messages.sendInfo(f"{index}/{max_len} {len(goods_urls)}条")
|
|
if Config.home_mode == False:
|
|
formats.de_repeat_urls(goods_urls_path)
|
|
|
|
def get_goods_info():
|
|
p_urls = {}
|
|
def workers(goods_urls, index):
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=lib.Server.worker_num) as executor:
|
|
futures = []
|
|
for goods_url in goods_urls:
|
|
if goods_url in p_urls:
|
|
continue
|
|
future = executor.submit(lib.Server.get_goods_info, goods_url)
|
|
futures.append(future)
|
|
|
|
results: list[types.GoodsInfo] = []
|
|
for future_index, future in enumerate(futures):
|
|
index += 1
|
|
try:
|
|
result = future.result() # 阻塞直到该任务完成
|
|
except Exception as e:
|
|
try:
|
|
files.add_text(f"{lib.Server.__errors_folder__}/goods_errors.log", f"{goods_urls[future_index]}:\n[{e}] {traceback.format_exc()}")
|
|
except:
|
|
pass
|
|
messages.sendWarn(f"{goods_urls[future_index]} 出错")
|
|
continue
|
|
|
|
results.append(result)
|
|
messages.sendInfo(f"{index}/{max_len}")
|
|
|
|
for goods_info in results:
|
|
try:
|
|
GoodsModel.add_goods(goods_info)
|
|
for p_url in goods_info.p_urls:
|
|
p_urls[p_url] = None
|
|
except Exception as e:
|
|
messages.sendWarn(f"{e}")
|
|
|
|
return index
|
|
|
|
goods_urls = files.load_lines(f"{lib.Server.__urls_folder__}/goods.txt")
|
|
messages.sendInfo('读取url完成')
|
|
|
|
messages.sendInfo('读取上次存档')
|
|
GoodsModel = goods.GoodsModel(lib.Server.__database_path__)
|
|
excel_index = GoodsModel.select_goods_to_dict()
|
|
messages.sendInfo('读取完成')
|
|
|
|
de_save_url = []
|
|
for goods_url in goods_urls:
|
|
if goods_url in excel_index:
|
|
continue
|
|
if 'http' not in goods_url:
|
|
continue
|
|
de_save_url.append(goods_url)
|
|
goods_urls = de_save_url
|
|
del excel_index
|
|
|
|
max_len = len(goods_urls)
|
|
index = 1
|
|
for i in range(0, len(goods_urls), lib.Server.bitch):
|
|
index = workers(goods_urls=goods_urls[i:i+lib.Server.bitch], index=index)
|
|
if lib.Server.switch_clash:
|
|
try:
|
|
clash.change_proxy()
|
|
except:
|
|
pass
|
|
if lib.Server.reflush_browsers:
|
|
if lib.Server.worker_num > 1:
|
|
lib.Server.reflush_browser()
|
|
messages.sendInfo("已保存")
|