diff --git a/README.md b/README.md index a703eb4..4263627 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,270 @@ # spider +## 1.商品数据枚举类 +```python +class GoodsInfo(BaseModel): + """ + 商品类(包含处理) + + :param url: 主商品链接 [必填] + :param spu: [必填] + :param title: 商品标题 [必填] + :param attribute_type: 属性类型 S|M [必填] + :param images: 图片列表 [必填] + :param brand: 品牌 + :param desc: 简介 + :param attribute_names: 属性[名称]列表 [Color, Size] + :param price: 价格 + :param original_price: 原价 + :param attributes: 变体类列表 + :param attribute_urls: 变体链接列表 + :param other_data: 其他参数 + + """ + class Attributes(BaseModel): + """ + 变体类 + + :param attribute_values: 变体[值]列表 [red, 30] [必填] + :param price: 价格 [必填] + :param original_price: 原价 [必填] + :param url: 变体链接 + :param images: 变体图片列表 + :param other_data: 其他参数 + + """ + url: str = '' # 商品链接 + attribute_values: list # 属性值1,2,3 + price: float + original_price: float + images: list[str] = [] + other_data: dict = {} # 其他参数 + + url: str + spu: str + title: str + brand: str = '' # 品牌 + desc: str = '' # 简介 + attribute_type: str # 属性类型 S M + images: list[str] + attribute_names: list = [] # 属性名称1,2,3 + price: float = 0.00 + original_price: float = 0.00 + attributes: list[Attributes] = [] # 变体列表 + attribute_urls: list[str] = [] # 变体链接列表(筛选用,可略) + other_data: dict = {} # 其他参数 +``` + +## 2.统一操作方法 + +### 2.1 获取图片链接 +```python +def get_image_urls() -> list[str]: + """ + 获取商品 url 列表 + + :return list: [url: str] + """ + image_urls = [] + + # 获取图片列表元素 + image_eles = tab.eles('') + for image_ele in image_eles: + # 定位到 img 标签并提取src + image_url = image_ele.ele('').attr('src') + image_urls.append(image_url) + + return image_urls +``` + +### 2.2 获取价格 +```python +def get_prices(self) -> float: + """ + 获取 价格(price)/原价(original_price) + + :return price, original_price + """ + price = tab.ele('').text # 价格 + original_price = tab.ele('').text # 原价 + + ######################格式化处理####################### + + ###################################################### + + return price, original_price +``` + +### 2.3 获取变体 +```python +def get_attributes(self) -> list[dict]: + """ + 获取商品属性 + + attribute = { + 'url': '变体链接(如果有)', + 'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'], + 'price': 0.0, + 'original_price': 0.0, + 'images': [] + } + + :return list: [attribute1, attribute2, ...] + """ + def get_size_eles(): + pass + # ...更多子属性... + + attributes = [] + + #########################操作区域############################# + + ############################################################### + + return attributes +``` + +## 3.完整示例 +```python + def get_goods_info(self, category, url) -> types.GoodsInfo: + def get_image_urls() -> list[str]: + """ + 获取商品 url 列表 + + :return list: [url: str] + """ + image_urls = [] + + # 获取图片列表元素 + image_eles = tab.eles('') + for image_ele in image_eles: + # 定位到 img 标签并提取src + image_url = image_ele.ele('').attr('src') + image_urls.append(image_url) + + return image_urls + + def get_prices(self) -> float: + """ + 获取 价格(price)/原价(original_price) + + :return price, original_price + """ + price = tab.ele('').text # 价格 + original_price = tab.ele('').text # 原价 + + ######################格式化处理####################### + + ###################################################### + + return price, original_price + + def get_attributes(self) -> list[dict]: + """ + 获取商品属性 + + attribute = { + 'url': '变体链接(如果有)', + 'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'], + 'price': 0.0, + 'original_price': 0.0, + 'images': [] + } + + :return list: [attribute1, attribute2, ...] + """ + def get_size_eles(): + pass + # ...更多子属性... + + attributes = [] + + #########################操作区域############################# + # 示例 + # color_eles = tab.eles('') + # for color_ele in color_eles: + # color_url = color_ele.ele('').attr('href') + # color_name = color_ele.text + + # color_images = get_image_urls() + # color_price, color_original_price = get_prices() + + # tab.get(color_url) # 链接访问或点击 + # # color_ele.click() + + # size_eles = get_size_eles() + # for size_ele in size_eles: + # size_name = size_ele.text + + # # ...item3... + + # attributes.append({ + # 'url': color_url, + # 'items': [color_name, size_name], + # 'price': color_price, + # 'original_price': color_original_price, + # 'images': [] + # }) + + # if len(size_eles) == 0: + # attributes.append({ + # 'url': color_url, + # 'items': [color_name], + # 'price': color_price, + # 'original_price': color_original_price, + # 'images': color_images + # }) + ############################################################### + + return attributes + + tab = self.browser.new_tab(url) + tab.set.window.max() + + #####################基本信息######################## + spu = formats.url_to_spu(url) + desc = "简介" + brand = "品牌" + title = "标题" + #################################################### + + price, original_price = get_prices() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + original_price=original_price, + images=get_image_urls() + ) + + m_data = [] + attributes = get_attributes() + + if len(attributes) == 0: + goods_info.attribute_type = 'S' + goods_info.attributes = [] + goods_info.attribute_names = [] + else: + goods_info.attribute_type = 'M' + goods_info.attribute_names = m_data + for attribute in attributes: + goods_info.attributes.append(types.GoodsInfo.Attributes( + url=attribute['url'], + attribute_values=attribute['items'], + price=attribute['price'], + original_price=attribute['original_price'], + images=attribute['images'] + )) + for image_url in attribute['images']: + if image_url not in goods_info.images: + goods_info.images.append(image_url) + try: + tab.close() + except: + pass + return goods_info +``` diff --git a/core/goods.py b/core/goods.py index 15d079e..611d7ed 100644 --- a/core/goods.py +++ b/core/goods.py @@ -186,7 +186,7 @@ class GoodsModel: for item in db_data: if item['spu'] in index_data: index_data[item['spu']].p_lists.append( - types.GoodsInfo.GoodsInfoP( + types.GoodsInfo.Attributes( attr_items = [item['son_item1'], item['son_item2'], item['son_item3']], price = item['son_price'], old_price = item['son_old_price'], @@ -212,7 +212,7 @@ class GoodsModel: ) if item['attr'] == 'M': index_data[item['spu']].p_lists.append( - types.GoodsInfo.GoodsInfoP( + types.GoodsInfo.Attributes( attr_items = [item['son_item1'], item['son_item2'], item['son_item3']], price = item['son_price'], old_price = item['son_old_price'], diff --git a/core/types.py b/core/types.py index 46cd50d..cc7f25a 100644 --- a/core/types.py +++ b/core/types.py @@ -10,47 +10,52 @@ class GoodsInfo(BaseModel): """ 商品类(包含处理) + :param url: 主商品链接 [必填] :param spu: [必填] :param title: 商品标题 [必填] + :param attribute_type: 属性类型 S|M [必填] + :param images: 图片列表 [必填] :param brand: 品牌 :param desc: 简介 - :param attr: 属性 S|M - :param attr_items: 变体[名称]列表 [Color, Size] + :param attribute_names: 属性[名称]列表 [Color, Size] :param price: 价格 - :param old_price: 原价 - :param images: 图片列表 - :param url: 主商品链接 - :param p_urls: 变体链接列表 - :param p_lists: 变体类列表 + :param original_price: 原价 + :param attributes: 变体类列表 + :param attribute_urls: 变体链接列表 :param other_data: 其他参数 """ - class GoodsInfoP(BaseModel): + class Attributes(BaseModel): """ 变体类 - :param attr_items: 变体[值]列表 [red, 30] + :param attribute_values: 变体[值]列表 [red, 30] [必填] + :param price: 价格 [必填] + :param original_price: 原价 [必填] + :param url: 变体链接 + :param images: 变体图片列表 + :param other_data: 其他参数 """ - attr_items: list = [] # 变体名称1,2,3 - price: float = 0.00 - old_price: float = 0.00 - images: list[str] = [] url: str = '' # 商品链接 + attribute_values: list # 属性值1,2,3 + price: float + original_price: float + images: list[str] = [] other_data: dict = {} # 其他参数 + url: str spu: str title: str brand: str = '' # 品牌 desc: str = '' # 简介 - attr: str = 'S' # 属性 S M - attr_items: list = [] # 变体名称1,2,3 + attribute_type: str # 属性类型 S M + images: list[str] + attribute_names: list = [] # 属性名称1,2,3 price: float = 0.00 - old_price: float = 0.00 - images: list[str] = [] - url: str = '' # 商品链接 - p_urls: list[str] = [] # 变体链接列表(筛选用,可略) - p_lists: list[GoodsInfoP] = [] # 变体列表 + original_price: float = 0.00 + attributes: list[Attributes] = [] # 变体列表 + attribute_urls: list[str] = [] # 变体链接列表(筛选用,可略) other_data: dict = {} # 其他参数 def to_db_data(self): @@ -60,30 +65,30 @@ class GoodsInfo(BaseModel): 'spu': self.spu, 'title': self.title, 'brand': self.brand, - 'attr': self.attr, + 'attr': self.attribute_type, 'desc': self.desc, 'price': self.price, - 'old_price': self.old_price, + 'original_price': self.original_price, 'images': orjson.dumps(self.images), 'url': self.url, 'other_data': orjson.dumps(self.other_data) } - for index, item_name in enumerate(self.attr_items): + for index, item_name in enumerate(self.attribute_names): if item_name: main_info[f"item{index+1}"] = item_name son_infos = [] - for p_info in self.p_lists: + for p_info in self.attributes: item = { 'spu': self.spu, 'price': p_info.price, - 'old_price': p_info.old_price, + 'original_price': p_info.original_price, 'images': orjson.dumps(p_info.images), 'url': p_info.url, 'other_data': orjson.dumps(p_info.other_data) } - p_info.attr_items += [None, None] - for index, item_name in enumerate(p_info.attr_items[:3]): + p_info.attribute_values += [None, None] + for index, item_name in enumerate(p_info.attribute_values[:3]): if item_name: item[f"item{index+1}"] = item_name son_infos.append(item) @@ -92,18 +97,18 @@ class GoodsInfo(BaseModel): def to_shopyy_row_data(self): """转为shopyy行数据""" - prices = [self.price, self.old_price] - for p_info in self.p_lists: + prices = [self.price, self.original_price] + for p_info in self.attributes: prices.append(p_info.price) - prices.append(p_info.old_price) - old_price = max(prices) + prices.append(p_info.original_price) + original_price = max(prices) rows_data = [] rows_data.append([ "", # 商品ID（系统生成，留空） "", f"{self.brand} {self.title}" if self.brand != "" else self.title, - self.attr, + self.attribute_type, "", "", self.desc, @@ -122,9 +127,9 @@ class GoodsInfo(BaseModel): self.brand, # 专辑名称 "", "", - *(self.attr_items[:3] + [''] * (3 - len(self.attr_items)))[:3], # 款式1/2/3 + *(self.attribute_names[:3] + [''] * (3 - len(self.attribute_names)))[:3], # 款式1/2/3 self.price, - old_price, + original_price, "", "", "", @@ -133,14 +138,14 @@ class GoodsInfo(BaseModel): '', # 变体备注2 ','.join(self.images) if self.images else '', self.url, - ','.join(self.p_urls) if self.p_urls else '' + ','.join(self.attribute_urls) if self.attribute_urls else '' ]) p_index = [] - for p_info in self.p_lists: - if p_info.attr_items in p_index: + for p_info in self.attributes: + if p_info.attribute_values in p_index: continue - p_index.append(p_info.attr_items) + p_index.append(p_info.attribute_values) rows_data.append([ "", # 商品ID（系统生成，留空） "", @@ -164,9 +169,9 @@ class GoodsInfo(BaseModel): self.brand, # 专辑名称 "", "", - *(p_info.attr_items[:3] + [''] * (3 - len(p_info.attr_items)))[:3], # 款式1/2/3 + *(p_info.attribute_values[:3] + [''] * (3 - len(p_info.attribute_values)))[:3], # 款式1/2/3 p_info.price, - old_price, + original_price, "", "", "", @@ -181,7 +186,7 @@ class GoodsInfo(BaseModel): def to_woo_row_data(self, category_name): """转为woo行数据""" - def get_lists(attr = 'variable',spu = '', price = '', old_price = '', item_name1: str = '', item_value1: str = '', item_name2: str = '', item_value2: str = '', item_name3: str = '', item_value3: str = '', images: list = []): + def get_lists(attr = 'variable',spu = '', price = '', original_price = '', item_name1: str = '', item_value1: str = '', item_name2: str = '', item_value2: str = '', item_name3: str = '', item_value3: str = '', images: list = []): return [ attr, self.spu if attr == 'variable' or attr == 'simple' else "", @@ -191,7 +196,7 @@ class GoodsInfo(BaseModel): 1, 100, price, - old_price, + original_price, category_name if attr == 'variable' or attr == 'simple' else "", ','.join(images) if images else '', self.spu if attr == 'variation' else "", @@ -208,12 +213,12 @@ class GoodsInfo(BaseModel): desc = BeautifulSoup(self.desc, 'html.parser').get_text() rows_data = [] - if self.attr == 'S': + if self.attribute_type == 'S': rows_data.append(get_lists( attr='simple', spu=self.spu, price=self.price, - old_price=self.old_price, + original_price=self.original_price, images=self.images )) else: @@ -221,10 +226,10 @@ class GoodsInfo(BaseModel): item_values1 = set() item_values2 = set() item_values3 = set() - for p_info in self.p_lists: - item_value1 = p_info.attr_items[0] if len(p_info.attr_items) >= 1 else "" - item_value2 = p_info.attr_items[1] if len(p_info.attr_items) >= 2 else "" - item_value3 = p_info.attr_items[2] if len(p_info.attr_items) >= 3 else "" + for p_info in self.attributes: + item_value1 = p_info.attribute_values[0] if len(p_info.attribute_values) >= 1 else "" + item_value2 = p_info.attribute_values[1] if len(p_info.attribute_values) >= 2 else "" + item_value3 = p_info.attribute_values[2] if len(p_info.attribute_values) >= 3 else "" item_value1 = item_value1.replace(',', '，').replace('|', ' ') item_values1.add(item_value1) @@ -239,25 +244,25 @@ class GoodsInfo(BaseModel): attr='variation', spu='', price=p_info.price, - old_price=p_info.old_price, + original_price=p_info.original_price, images=p_info.images, - item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "", + item_name1=self.attribute_names[0] if len(self.attribute_names) >= 1 else "", item_value1=item_value1, - item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "", + item_name2=self.attribute_names[1] if len(self.attribute_names) >= 2 else "", item_value2=item_value2, - item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "", + item_name3=self.attribute_names[2] if len(self.attribute_names) >= 3 else "", item_value3=item_value3 )) rows_data.insert(0, get_lists( spu=self.spu, price='', - old_price='', + original_price='', images=self.images, - item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "", + item_name1=self.attribute_names[0] if len(self.attribute_names) >= 1 else "", item_value1='|'.join(item_values1), - item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "", + item_name2=self.attribute_names[1] if len(self.attribute_names) >= 2 else "", item_value2='|'.join(item_values2), - item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "", + item_name3=self.attribute_names[2] if len(self.attribute_names) >= 3 else "", item_value3='|'.join(item_values3) )) return rows_data diff --git a/lib/example.py b/lib/example.py index 90cfcf0..e053caf 100644 --- a/lib/example.py +++ b/lib/example.py @@ -7,71 +7,130 @@ class SpiderModule(spiders.Spiders): def __init__(self): super().__init__( self.project_name, - bitch = 10, # 运行批次大小 - worker_num = 1, # 线程数 + bitch = 10, + worker_num = 1, switch_clash = False, reflush_browser = False ) - # 获取导航入口 def get_category_urls(self) -> list[str]: category_urls = [] - self.tab.get('') - # 一级分类 - one_menu_eles = self.tab.eles('') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele.ele('') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-').replace(',', ' ') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - # 二级分类 - two_menu_eles = one_menu_ele.eles('') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-').replace(',', ' ') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - # 三级分类 - three_menu_eles = two_menu_ele.eles('') - for three_menu_ele in three_menu_eles: - three_link_ele = three_menu_ele.ele('') - three_url = three_link_ele.attr('href') - three_name = three_link_ele.text.replace('/', '-').replace(',', ' ') - category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + pass return category_urls - # 获取商品链接入口 - def get_goods_urls(self, category_url: str) -> list[str]: + def get_goods_urls(self, category, category_url: str) -> list[str]: self.tab.get(f"{category_url}") goods_urls = [] pass - return goods_urls + return {'category': category, 'urls': goods_urls} - # 获取商品详情入口 - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): + def get_goods_info(self, category, url) -> types.GoodsInfo: + def get_image_urls() -> list[str]: + """ + 获取商品 url 列表 + + :return list: [url: str] + """ image_urls = [] - pass + + # 获取图片列表元素 + image_eles = tab.eles('') + for image_ele in image_eles: + # 定位到 img 标签并提取src + image_url = image_ele.ele('').attr('src') + image_urls.append(image_url) + return image_urls + def get_prices(self) -> float: + """ + 获取 价格(price)/原价(original_price) + + :return price, original_price + """ + price = tab.ele('').text # 价格 + original_price = tab.ele('').text # 原价 + + ######################格式化处理####################### + + ###################################################### + + return price, original_price + + def get_attributes(self) -> list[dict]: + """ + 获取商品属性 + + attribute = { + 'url': '变体链接(如果有)', + 'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'], + 'price': 0.0, + 'original_price': 0.0, + 'images': [] + } + + :return list: [attribute1, attribute2, ...] + """ + def get_size_eles(): + pass + # ...更多子属性... + + attributes = [] + + #########################操作区域############################# + # 示例 + # color_eles = tab.eles('') + # for color_ele in color_eles: + # color_url = color_ele.ele('').attr('href') + # color_name = color_ele.text + + # color_images = get_image_urls() + # color_price, color_original_price = get_prices() + + # tab.get(color_url) # 链接访问或点击 + # # color_ele.click() + + # size_eles = get_size_eles() + # for size_ele in size_eles: + # size_name = size_ele.text + + # # ...item3... + + # attributes.append({ + # 'url': color_url, + # 'items': [color_name, size_name], + # 'price': color_price, + # 'original_price': color_original_price, + # 'images': [] + # }) + + # if len(size_eles) == 0: + # attributes.append({ + # 'url': color_url, + # 'items': [color_name], + # 'price': color_price, + # 'original_price': color_original_price, + # 'images': color_images + # }) + ############################################################### + + return attributes + tab = self.browser.new_tab(url) tab.set.window.max() - spu = formats.url_to_spu(url) + #####################基本信息######################## + spu = formats.url_to_spu(url) desc = "简介" brand = "品牌" title = "标题" - price = 0.00 - old_price = price + #################################################### - main_images = get_image_urls() + price, original_price = get_prices() goods_info = types.GoodsInfo( title=title, @@ -80,32 +139,31 @@ class SpiderModule(spiders.Spiders): url=url, spu=spu, price=price, - old_price=old_price, + original_price=original_price, + images=get_image_urls() ) m_data = [] - attr_items = [] + attributes = get_attributes() - if len(attr_items) == 0: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] + if len(attributes) == 0: + goods_info.attribute_type = 'S' + goods_info.attributes = [] + goods_info.attribute_names = [] else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'] + goods_info.attribute_type = 'M' + goods_info.attribute_names = m_data + for attribute in attributes: + goods_info.attributes.append(types.GoodsInfo.Attributes( + url=attribute['url'], + attribute_values=attribute['items'], + price=attribute['price'], + original_price=attribute['original_price'], + images=attribute['images'] )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - + for image_url in attribute['images']: + if image_url not in goods_info.images: + goods_info.images.append(image_url) try: tab.close() except: