271 lines
7.7 KiB
Markdown
271 lines
7.7 KiB
Markdown
# spider
|
|
|
|
## 1.商品数据枚举类
|
|
```python
|
|
class GoodsInfo(BaseModel):
|
|
"""
|
|
商品类(包含处理)
|
|
|
|
:param url: 主商品链接 [必填]
|
|
:param spu: [必填]
|
|
:param title: 商品标题 [必填]
|
|
:param attribute_type: 属性类型 S|M [必填]
|
|
:param images: 图片列表 [必填]
|
|
:param brand: 品牌
|
|
:param desc: 简介
|
|
:param attribute_names: 属性[名称]列表 [Color, Size]
|
|
:param price: 价格
|
|
:param original_price: 原价
|
|
:param attributes: 变体类列表
|
|
:param attribute_urls: 变体链接列表
|
|
:param other_data: 其他参数
|
|
|
|
"""
|
|
class Attributes(BaseModel):
|
|
"""
|
|
变体类
|
|
|
|
:param attribute_values: 变体[值]列表 [red, 30] [必填]
|
|
:param price: 价格 [必填]
|
|
:param original_price: 原价 [必填]
|
|
:param url: 变体链接
|
|
:param images: 变体图片列表
|
|
:param other_data: 其他参数
|
|
|
|
"""
|
|
url: str = '' # 商品链接
|
|
attribute_values: list # 属性值1,2,3
|
|
price: float
|
|
original_price: float
|
|
images: list[str] = []
|
|
other_data: dict = {} # 其他参数
|
|
|
|
url: str
|
|
spu: str
|
|
title: str
|
|
brand: str = '' # 品牌
|
|
desc: str = '' # 简介
|
|
attribute_type: str # 属性类型 S M
|
|
images: list[str]
|
|
attribute_names: list = [] # 属性名称1,2,3
|
|
price: float = 0.00
|
|
original_price: float = 0.00
|
|
attributes: list[Attributes] = [] # 变体列表
|
|
attribute_urls: list[str] = [] # 变体链接列表(筛选用,可略)
|
|
other_data: dict = {} # 其他参数
|
|
```
|
|
|
|
## 2.统一操作方法
|
|
|
|
### 2.1 获取图片链接
|
|
```python
|
|
def get_image_urls() -> list[str]:
|
|
"""
|
|
获取商品 url 列表
|
|
|
|
:return list: [url: str]
|
|
"""
|
|
image_urls = []
|
|
|
|
# 获取图片列表元素
|
|
image_eles = tab.eles('')
|
|
for image_ele in image_eles:
|
|
# 定位到 img 标签并提取src
|
|
image_url = image_ele.ele('').attr('src')
|
|
image_urls.append(image_url)
|
|
|
|
return image_urls
|
|
```
|
|
|
|
### 2.2 获取价格
|
|
```python
|
|
def get_prices(self) -> float:
|
|
"""
|
|
获取 价格(price)/原价(original_price)
|
|
|
|
:return price, original_price
|
|
"""
|
|
price = tab.ele('').text # 价格
|
|
original_price = tab.ele('').text # 原价
|
|
|
|
######################格式化处理#######################
|
|
|
|
######################################################
|
|
|
|
return price, original_price
|
|
```
|
|
|
|
### 2.3 获取变体
|
|
```python
|
|
def get_attributes(self) -> list[dict]:
|
|
"""
|
|
获取商品属性
|
|
|
|
attribute = {
|
|
'url': '变体链接(如果有)',
|
|
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
|
|
'price': 0.0,
|
|
'original_price': 0.0,
|
|
'images': []
|
|
}
|
|
|
|
:return list: [attribute1, attribute2, ...]
|
|
"""
|
|
def get_size_eles():
|
|
pass
|
|
# ...更多子属性...
|
|
|
|
attributes = []
|
|
|
|
#########################操作区域#############################
|
|
|
|
###############################################################
|
|
|
|
return attributes
|
|
```
|
|
|
|
## 3.完整示例
|
|
```python
|
|
def get_goods_info(self, category, url) -> types.GoodsInfo:
|
|
def get_image_urls() -> list[str]:
|
|
"""
|
|
获取商品 url 列表
|
|
|
|
:return list: [url: str]
|
|
"""
|
|
image_urls = []
|
|
|
|
# 获取图片列表元素
|
|
image_eles = tab.eles('')
|
|
for image_ele in image_eles:
|
|
# 定位到 img 标签并提取src
|
|
image_url = image_ele.ele('').attr('src')
|
|
image_urls.append(image_url)
|
|
|
|
return image_urls
|
|
|
|
def get_prices(self) -> float:
|
|
"""
|
|
获取 价格(price)/原价(original_price)
|
|
|
|
:return price, original_price
|
|
"""
|
|
price = tab.ele('').text # 价格
|
|
original_price = tab.ele('').text # 原价
|
|
|
|
######################格式化处理#######################
|
|
|
|
######################################################
|
|
|
|
return price, original_price
|
|
|
|
def get_attributes(self) -> list[dict]:
|
|
"""
|
|
获取商品属性
|
|
|
|
attribute = {
|
|
'url': '变体链接(如果有)',
|
|
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
|
|
'price': 0.0,
|
|
'original_price': 0.0,
|
|
'images': []
|
|
}
|
|
|
|
:return list: [attribute1, attribute2, ...]
|
|
"""
|
|
def get_size_eles():
|
|
pass
|
|
# ...更多子属性...
|
|
|
|
attributes = []
|
|
|
|
#########################操作区域#############################
|
|
# 示例
|
|
# color_eles = tab.eles('')
|
|
# for color_ele in color_eles:
|
|
# color_url = color_ele.ele('').attr('href')
|
|
# color_name = color_ele.text
|
|
|
|
# color_images = get_image_urls()
|
|
# color_price, color_original_price = get_prices()
|
|
|
|
# tab.get(color_url) # 链接访问或点击
|
|
# # color_ele.click()
|
|
|
|
# size_eles = get_size_eles()
|
|
# for size_ele in size_eles:
|
|
# size_name = size_ele.text
|
|
|
|
# # ...item3...
|
|
|
|
# attributes.append({
|
|
# 'url': color_url,
|
|
# 'items': [color_name, size_name],
|
|
# 'price': color_price,
|
|
# 'original_price': color_original_price,
|
|
# 'images': []
|
|
# })
|
|
|
|
# if len(size_eles) == 0:
|
|
# attributes.append({
|
|
# 'url': color_url,
|
|
# 'items': [color_name],
|
|
# 'price': color_price,
|
|
# 'original_price': color_original_price,
|
|
# 'images': color_images
|
|
# })
|
|
###############################################################
|
|
|
|
return attributes
|
|
|
|
tab = self.browser.new_tab(url)
|
|
tab.set.window.max()
|
|
|
|
#####################基本信息########################
|
|
spu = formats.url_to_spu(url)
|
|
desc = "简介"
|
|
brand = "品牌"
|
|
title = "标题"
|
|
####################################################
|
|
|
|
price, original_price = get_prices()
|
|
|
|
goods_info = types.GoodsInfo(
|
|
title=title,
|
|
desc=desc,
|
|
brand=brand,
|
|
url=url,
|
|
spu=spu,
|
|
price=price,
|
|
original_price=original_price,
|
|
images=get_image_urls()
|
|
)
|
|
|
|
m_data = []
|
|
attributes = get_attributes()
|
|
|
|
if len(attributes) == 0:
|
|
goods_info.attribute_type = 'S'
|
|
goods_info.attributes = []
|
|
goods_info.attribute_names = []
|
|
else:
|
|
goods_info.attribute_type = 'M'
|
|
goods_info.attribute_names = m_data
|
|
for attribute in attributes:
|
|
goods_info.attributes.append(types.GoodsInfo.Attributes(
|
|
url=attribute['url'],
|
|
attribute_values=attribute['items'],
|
|
price=attribute['price'],
|
|
original_price=attribute['original_price'],
|
|
images=attribute['images']
|
|
))
|
|
for image_url in attribute['images']:
|
|
if image_url not in goods_info.images:
|
|
goods_info.images.append(image_url)
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
return goods_info
|
|
```
|