from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "example" def __init__(self): super().__init__( self.project_name, bitch = 10, # 运行批次大小 worker_num = 1, # 线程数 switch_clash = False, reflush_browser = False ) # 获取导航入口 def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('') # 一级分类 one_menu_eles = self.tab.eles('') for one_menu_ele in one_menu_eles: one_link_ele = one_menu_ele.ele('') one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-').replace(',', ' ') category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) # 二级分类 two_menu_eles = one_menu_ele.eles('') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele.ele('') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-').replace(',', ' ') category_urls.append(f"{two_url}#{one_name}/{two_name}") # 三级分类 three_menu_eles = two_menu_ele.eles('') for three_menu_ele in three_menu_eles: three_link_ele = three_menu_ele.ele('') three_url = three_link_ele.attr('href') three_name = three_link_ele.text.replace('/', '-').replace(',', ' ') category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls # 获取商品链接入口 def get_goods_urls(self, category_url: str) -> list[str]: self.tab.get(f"{category_url}") goods_urls = [] pass return goods_urls # 获取商品详情入口 def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] pass return image_urls tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) desc = "简介" brand = "品牌" title = "标题" price = 0.00 old_price = price main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) m_data = [] attr_items = [] if len(attr_items) == 0: goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] else: goods_info.attr = 'M' goods_info.attr_items = m_data for attr_item in attr_items: goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( attr_items=attr_item['items'], price=attr_item['price'], old_price=attr_item['old_price'], images=attr_item['images'] )) for image_url in attr_item['images']: if image_url not in main_images: main_images.append(image_url) goods_info.images = main_images try: tab.close() except: pass return goods_info