Cerys/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整数据结构

Browse Source
This commit is contained in:
Cerys
2026-05-22 11:53:25 +08:00
parent c360413981
commit 73287d8fe1
4 changed files with 448 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
README.md
View File

@@ -1,2 +1,270 @@
# spider
## 1.商品数据枚举类
```python
class GoodsInfo(BaseModel):
"""
商品类(包含处理)
:param url: 主商品链接 [必填]
:param spu: [必填]
:param title: 商品标题 [必填]
:param attribute_type: 属性类型 S|M [必填]
:param images: 图片列表 [必填]
:param brand: 品牌
:param desc: 简介
:param attribute_names: 属性[名称]列表 [Color, Size]
:param price: 价格
:param original_price: 原价
:param attributes: 变体类列表
:param attribute_urls: 变体链接列表
:param other_data: 其他参数
"""
class Attributes(BaseModel):
"""
变体类
:param attribute_values: 变体[值]列表 [red, 30] [必填]
:param price: 价格 [必填]
:param original_price: 原价 [必填]
:param url: 变体链接
:param images: 变体图片列表
:param other_data: 其他参数
"""
url: str = '' # 商品链接
attribute_values: list # 属性值1,2,3
price: float
original_price: float
images: list[str] = []
other_data: dict = {} # 其他参数
url: str
spu: str
title: str
brand: str = '' # 品牌
desc: str = '' # 简介
attribute_type: str # 属性类型 S M
images: list[str]
attribute_names: list = [] # 属性名称1,2,3
price: float = 0.00
original_price: float = 0.00
attributes: list[Attributes] = [] # 变体列表
attribute_urls: list[str] = [] # 变体链接列表(筛选用,可略)
other_data: dict = {} # 其他参数
```
## 2.统一操作方法
### 2.1 获取图片链接
```python
def get_image_urls() -> list[str]:
"""
获取商品 url 列表
:return list: [url: str]
"""
image_urls = []
# 获取图片列表元素
image_eles = tab.eles('')
for image_ele in image_eles:
# 定位到 img 标签并提取src
image_url = image_ele.ele('').attr('src')
image_urls.append(image_url)
return image_urls
```
### 2.2 获取价格
```python
def get_prices(self) -> float:
"""
获取 价格(price)/原价(original_price)
:return price, original_price
"""
price = tab.ele('').text # 价格
original_price = tab.ele('').text # 原价
######################格式化处理#######################
######################################################
return price, original_price
```
### 2.3 获取变体
```python
def get_attributes(self) -> list[dict]:
"""
获取商品属性
attribute = {
'url': '变体链接(如果有)',
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
'price': 0.0,
'original_price': 0.0,
'images': []
}
:return list: [attribute1, attribute2, ...]
"""
def get_size_eles():
pass
# ...更多子属性...
attributes = []
#########################操作区域#############################
###############################################################
return attributes
```
## 3.完整示例
```python
def get_goods_info(self, category, url) -> types.GoodsInfo:
def get_image_urls() -> list[str]:
"""
获取商品 url 列表
:return list: [url: str]
"""
image_urls = []
# 获取图片列表元素
image_eles = tab.eles('')
for image_ele in image_eles:
# 定位到 img 标签并提取src
image_url = image_ele.ele('').attr('src')
image_urls.append(image_url)
return image_urls
def get_prices(self) -> float:
"""
获取 价格(price)/原价(original_price)
:return price, original_price
"""
price = tab.ele('').text # 价格
original_price = tab.ele('').text # 原价
######################格式化处理#######################
######################################################
return price, original_price
def get_attributes(self) -> list[dict]:
"""
获取商品属性
attribute = {
'url': '变体链接(如果有)',
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
'price': 0.0,
'original_price': 0.0,
'images': []
}
:return list: [attribute1, attribute2, ...]
"""
def get_size_eles():
pass
# ...更多子属性...
attributes = []
#########################操作区域#############################
# 示例
# color_eles = tab.eles('')
# for color_ele in color_eles:
# color_url = color_ele.ele('').attr('href')
# color_name = color_ele.text
# color_images = get_image_urls()
# color_price, color_original_price = get_prices()
# tab.get(color_url) # 链接访问或点击
# # color_ele.click()
# size_eles = get_size_eles()
# for size_ele in size_eles:
# size_name = size_ele.text
# # ...item3...
# attributes.append({
# 'url': color_url,
# 'items': [color_name, size_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': []
# })
# if len(size_eles) == 0:
# attributes.append({
# 'url': color_url,
# 'items': [color_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': color_images
# })
###############################################################
return attributes
tab = self.browser.new_tab(url)
tab.set.window.max()
#####################基本信息########################
spu = formats.url_to_spu(url)
desc = "简介"
brand = "品牌"
title = "标题"
####################################################
price, original_price = get_prices()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
original_price=original_price,
images=get_image_urls()
)
m_data = []
attributes = get_attributes()
if len(attributes) == 0:
goods_info.attribute_type = 'S'
goods_info.attributes = []
goods_info.attribute_names = []
else:
goods_info.attribute_type = 'M'
goods_info.attribute_names = m_data
for attribute in attributes:
goods_info.attributes.append(types.GoodsInfo.Attributes(
url=attribute['url'],
attribute_values=attribute['items'],
price=attribute['price'],
original_price=attribute['original_price'],
images=attribute['images']
))
for image_url in attribute['images']:
if image_url not in goods_info.images:
goods_info.images.append(image_url)
try:
tab.close()
except:
pass
return goods_info
```