172 lines
5.3 KiB
Python
172 lines
5.3 KiB
Python
from core import spiders, types
|
|
from utils import formats, messages
|
|
|
|
class SpiderModule(spiders.Spiders):
|
|
project_name = "example"
|
|
|
|
def __init__(self):
|
|
super().__init__(
|
|
self.project_name,
|
|
bitch = 10,
|
|
worker_num = 1,
|
|
switch_clash = False,
|
|
reflush_browser = False
|
|
)
|
|
|
|
def get_category_urls(self) -> list[str]:
|
|
category_urls = []
|
|
|
|
pass
|
|
|
|
return category_urls
|
|
|
|
def get_goods_urls(self, category, category_url: str) -> list[str]:
|
|
self.tab.get(f"{category_url}")
|
|
goods_urls = []
|
|
|
|
pass
|
|
|
|
return {'category': category, 'urls': goods_urls}
|
|
|
|
def get_goods_info(self, category, url) -> types.GoodsInfo:
|
|
def get_image_urls() -> list[str]:
|
|
"""
|
|
获取商品 url 列表
|
|
|
|
:return list: [url: str]
|
|
"""
|
|
image_urls = []
|
|
|
|
# 获取图片列表元素
|
|
image_eles = tab.eles('')
|
|
for image_ele in image_eles:
|
|
# 定位到 img 标签并提取src
|
|
image_url = image_ele.ele('').attr('src')
|
|
image_urls.append(image_url)
|
|
|
|
return image_urls
|
|
|
|
def get_prices(self) -> float:
|
|
"""
|
|
获取 价格(price)/原价(original_price)
|
|
|
|
:return price, original_price
|
|
"""
|
|
price = tab.ele('').text # 价格
|
|
original_price = tab.ele('').text # 原价
|
|
|
|
######################格式化处理#######################
|
|
|
|
######################################################
|
|
|
|
return price, original_price
|
|
|
|
def get_attributes(self) -> list[dict]:
|
|
"""
|
|
获取商品属性
|
|
|
|
attribute = {
|
|
'url': '变体链接(如果有)',
|
|
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
|
|
'price': 0.0,
|
|
'original_price': 0.0,
|
|
'images': []
|
|
}
|
|
|
|
:return list: [attribute1, attribute2, ...]
|
|
"""
|
|
def get_size_eles():
|
|
pass
|
|
# ...更多子属性...
|
|
|
|
attributes = []
|
|
|
|
#########################操作区域#############################
|
|
# 示例
|
|
# color_eles = tab.eles('')
|
|
# for color_ele in color_eles:
|
|
# color_url = color_ele.ele('').attr('href')
|
|
# color_name = color_ele.text
|
|
|
|
# color_images = get_image_urls()
|
|
# color_price, color_original_price = get_prices()
|
|
|
|
# tab.get(color_url) # 链接访问或点击
|
|
# # color_ele.click()
|
|
|
|
# size_eles = get_size_eles()
|
|
# for size_ele in size_eles:
|
|
# size_name = size_ele.text
|
|
|
|
# # ...item3...
|
|
|
|
# attributes.append({
|
|
# 'url': color_url,
|
|
# 'items': [color_name, size_name],
|
|
# 'price': color_price,
|
|
# 'original_price': color_original_price,
|
|
# 'images': []
|
|
# })
|
|
|
|
# if len(size_eles) == 0:
|
|
# attributes.append({
|
|
# 'url': color_url,
|
|
# 'items': [color_name],
|
|
# 'price': color_price,
|
|
# 'original_price': color_original_price,
|
|
# 'images': color_images
|
|
# })
|
|
###############################################################
|
|
|
|
return attributes
|
|
|
|
tab = self.browser.new_tab(url)
|
|
tab.set.window.max()
|
|
|
|
#####################基本信息########################
|
|
spu = formats.url_to_spu(url)
|
|
desc = "简介"
|
|
brand = "品牌"
|
|
title = "标题"
|
|
####################################################
|
|
|
|
price, original_price = get_prices()
|
|
|
|
goods_info = types.GoodsInfo(
|
|
title=title,
|
|
desc=desc,
|
|
brand=brand,
|
|
url=url,
|
|
spu=spu,
|
|
price=price,
|
|
original_price=original_price,
|
|
images=get_image_urls()
|
|
)
|
|
|
|
m_data = []
|
|
attributes = get_attributes()
|
|
|
|
if len(attributes) == 0:
|
|
goods_info.attribute_type = 'S'
|
|
goods_info.attributes = []
|
|
goods_info.attribute_names = []
|
|
else:
|
|
goods_info.attribute_type = 'M'
|
|
goods_info.attribute_names = m_data
|
|
for attribute in attributes:
|
|
goods_info.attributes.append(types.GoodsInfo.Attributes(
|
|
url=attribute['url'],
|
|
attribute_values=attribute['items'],
|
|
price=attribute['price'],
|
|
original_price=attribute['original_price'],
|
|
images=attribute['images']
|
|
))
|
|
for image_url in attribute['images']:
|
|
if image_url not in goods_info.images:
|
|
goods_info.images.append(image_url)
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
return goods_info
|