from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "example" def __init__(self): super().__init__( self.project_name, bitch = 10, worker_num = 1, switch_clash = False, reflush_browser = False ) def get_category_urls(self) -> list[str]: category_urls = [] pass return category_urls def get_goods_urls(self, category, category_url: str) -> list[str]: self.tab.get(f"{category_url}") goods_urls = [] pass return {'category': category, 'urls': goods_urls} def get_goods_info(self, category, url) -> types.GoodsInfo: def get_image_urls() -> list[str]: """ 获取商品 url 列表 :return list: [url: str] """ image_urls = [] # 获取图片列表元素 image_eles = tab.eles('') for image_ele in image_eles: # 定位到 img 标签并提取src image_url = image_ele.ele('').attr('src') image_urls.append(image_url) return image_urls def get_prices(self) -> float: """ 获取 价格(price)/原价(original_price) :return price, original_price """ price = tab.ele('').text # 价格 original_price = tab.ele('').text # 原价 ######################格式化处理####################### ###################################################### return price, original_price def get_attributes(self) -> list[dict]: """ 获取商品属性 attribute = { 'url': '变体链接(如果有)', 'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'], 'price': 0.0, 'original_price': 0.0, 'images': [] } :return list: [attribute1, attribute2, ...] """ def get_size_eles(): pass # ...更多子属性... attributes = [] #########################操作区域############################# # 示例 # color_eles = tab.eles('') # for color_ele in color_eles: # color_url = color_ele.ele('').attr('href') # color_name = color_ele.text # color_images = get_image_urls() # color_price, color_original_price = get_prices() # tab.get(color_url) # 链接访问或点击 # # color_ele.click() # size_eles = get_size_eles() # for size_ele in size_eles: # size_name = size_ele.text # # ...item3... # attributes.append({ # 'url': color_url, # 'items': [color_name, size_name], # 'price': color_price, # 'original_price': color_original_price, # 'images': [] # }) # if len(size_eles) == 0: # attributes.append({ # 'url': color_url, # 'items': [color_name], # 'price': color_price, # 'original_price': color_original_price, # 'images': color_images # }) ############################################################### return attributes tab = self.browser.new_tab(url) tab.set.window.max() #####################基本信息######################## spu = formats.url_to_spu(url) desc = "简介" brand = "品牌" title = "标题" #################################################### price, original_price = get_prices() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, original_price=original_price, images=get_image_urls() ) m_data = [] attributes = get_attributes() if len(attributes) == 0: goods_info.attribute_type = 'S' goods_info.attributes = [] goods_info.attribute_names = [] else: goods_info.attribute_type = 'M' goods_info.attribute_names = m_data for attribute in attributes: goods_info.attributes.append(types.GoodsInfo.Attributes( url=attribute['url'], attribute_values=attribute['items'], price=attribute['price'], original_price=attribute['original_price'], images=attribute['images'] )) for image_url in attribute['images']: if image_url not in goods_info.images: goods_info.images.append(image_url) try: tab.close() except: pass return goods_info