diff --git a/lib/gether.py b/core/gether.py similarity index 90% rename from lib/gether.py rename to core/gether.py index 92ae5ec..115a754 100644 --- a/lib/gether.py +++ b/core/gether.py @@ -3,24 +3,19 @@ import traceback import concurrent.futures from utils import files, messages, clash, formats import lib -from core import Config, types -from lib.model import goods +from core import Config, types, goods def get_category_urls(): category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt" - if os.path.exists(category_urls_path): - return - + # if os.path.exists(category_urls_path): + # return category_urls = lib.Server.get_category_urls() - for urls in category_urls: - if 'http' not in urls: - continue - files.save_line(category_urls_path, urls) + files.save_list(category_urls_path, category_urls) -def get_goods_urls(categorys_len: int = 3): +def get_goods_urls(categorys_len: int = 3, home_mode: bool = False): goods_urls_path = f"{lib.Server.__urls_folder__}/goods.txt" category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt" - if Config.home_mode: + if home_mode: goods_urls_path = goods_urls_path.replace('.txt', '(home).txt') category_urls_path = category_urls_path.replace('.txt', '(home).txt') @@ -42,7 +37,7 @@ def get_goods_urls(categorys_len: int = 3): index += 1 if 'https' not in category_url: continue - if Config.skip_category and Config.home_mode == False and len(category.split('/')) < categorys_len: + if Config.skip_category and home_mode == False and len(category.split('/')) < categorys_len: continue if category in category_index: continue @@ -58,7 +53,7 @@ def get_goods_urls(categorys_len: int = 3): files.add_text(f"{lib.Server.__errors_folder__}/category_errors.log", f"{category_url}:\n[{e}] {traceback.format_exc()}") messages.sendError(str(e)) messages.sendInfo(f"{index}/{max_len} {len(goods_urls)}条") - if Config.home_mode == False: + if home_mode == False: formats.de_repeat_urls(goods_urls_path) def get_goods_info(): diff --git a/lib/model/goods.py b/core/goods.py similarity index 100% rename from lib/model/goods.py rename to core/goods.py diff --git a/lib/shopyy.py b/core/shopyy.py similarity index 100% rename from lib/shopyy.py rename to core/shopyy.py diff --git a/lib/__init__.py b/lib/__init__.py index 52ab1aa..3d83528 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -1,3 +1,3 @@ -from lib.module import example +from lib import example Server: example.SpiderModule diff --git a/lib/module/example.py b/lib/example.py similarity index 60% rename from lib/module/example.py rename to lib/example.py index e8d6b45..90cfcf0 100644 --- a/lib/module/example.py +++ b/lib/example.py @@ -7,19 +7,45 @@ class SpiderModule(spiders.Spiders): def __init__(self): super().__init__( self.project_name, - bitch = 10, - worker_num = 1, + bitch = 10, # 运行批次大小 + worker_num = 1, # 线程数 switch_clash = False, reflush_browser = False ) + # 获取导航入口 def get_category_urls(self) -> list[str]: category_urls = [] - pass + self.tab.get('') + # 一级分类 + one_menu_eles = self.tab.eles('') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-').replace(',', ' ') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + # 二级分类 + two_menu_eles = one_menu_ele.eles('') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-').replace(',', ' ') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + # 三级分类 + three_menu_eles = two_menu_ele.eles('') + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-').replace(',', ' ') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls + # 获取商品链接入口 def get_goods_urls(self, category_url: str) -> list[str]: self.tab.get(f"{category_url}") goods_urls = [] @@ -28,6 +54,7 @@ class SpiderModule(spiders.Spiders): return goods_urls + # 获取商品详情入口 def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] diff --git a/lib/model/__pycache__/goods.cpython-312.pyc b/lib/model/__pycache__/goods.cpython-312.pyc deleted file mode 100644 index db30e53..0000000 Binary files a/lib/model/__pycache__/goods.cpython-312.pyc and /dev/null differ diff --git a/main.py b/main.py index 41eb1cd..61e5386 100644 --- a/main.py +++ b/main.py @@ -1,45 +1,23 @@ import lib -from lib import gether, shopyy -from lib.model import goods +from core import shopyy, gether, goods from utils import formats, files, wpdata if __name__ == '__main__': - from lib.module import example + from lib import example Server:example.SpiderModule Server = example.SpiderModule() lib.Server = Server - # category_urls = Server.get_category_urls() - # files.save_list(f"{Server.__urls_folder__}/categorys.txt", category_urls) - # Server.get_home() - # gether.get_goods_urls(2) - # formats.de_repeat_urls(r'data\bricomarche\urls\goods(home).txt') - # gether.get_goods_info() + # gether.get_category_urls() # 网站导航 + # gether.get_goods_urls(3, False) # 商品链接 + # gether.get_goods_info() # 商品详情 GoodsModel = goods.GoodsModel(Server.__database_path__) - # GoodsModel.to_shopyy_xlsx(f"{Server.__excels_folder__}/goods.xlsx") - # GoodsModel.to_woo_csv(r'data\fs\urls\goods_2026-01-23.txt_old.txt', f"data/fs/goods.csv") - # shopyy.Shopyy(Server.__data_path__).generate_album( - # f"{Server.__urls_folder__}/goods.txt_old.txt" - # ) - # shopyy.Shopyy(Server.__data_path__).generate_navigation( - # f"{Server.__urls_folder__}/categorys.txt" - # ) - # shopyy.Shopyy(Server.__data_path__).generate_album( - # f"{Server.__urls_folder__}/goods(home).txt" - # ).generate_navigation( - # f"{Server.__urls_folder__}/categorys(home).txt" - # ) + # GoodsModel.to_shopyy_xlsx(f"{Server.__excels_folder__}/goods.xlsx") # 导出shopyy表格 - # wpdata.Wpdata( - # 'data/condom69', - # r'data\condom69\excels\goods_1.xlsx', - # r'data\condom69\urls\goods.txt_old.txt', - # ).run() - - # Verify = shopyy.ShopyyVerify('https://ssfshop.zenshop.cn') + # Verify = shopyy.ShopyyVerify('https://takeokikuchi001.zenshop.cn') # Verify.get_zero_collections(True) # Verify.get_error_images(True)