Cerys/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整数据结构

Browse Source
This commit is contained in:
Cerys
2026-05-22 11:53:25 +08:00
parent c360413981
commit 73287d8fe1
4 changed files with 448 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
README.md
View File

@@ -1,2 +1,270 @@
# spider
## 1.商品数据枚举类
```python
class GoodsInfo(BaseModel):
"""
商品类(包含处理)
:param url: 主商品链接 [必填]
:param spu: [必填]
:param title: 商品标题 [必填]
:param attribute_type: 属性类型 S|M [必填]
:param images: 图片列表 [必填]
:param brand: 品牌
:param desc: 简介
:param attribute_names: 属性[名称]列表 [Color, Size]
:param price: 价格
:param original_price: 原价
:param attributes: 变体类列表
:param attribute_urls: 变体链接列表
:param other_data: 其他参数
"""
class Attributes(BaseModel):
"""
变体类
:param attribute_values: 变体[值]列表 [red, 30] [必填]
:param price: 价格 [必填]
:param original_price: 原价 [必填]
:param url: 变体链接
:param images: 变体图片列表
:param other_data: 其他参数
"""
url: str = '' # 商品链接
attribute_values: list # 属性值1,2,3
price: float
original_price: float
images: list[str] = []
other_data: dict = {} # 其他参数
url: str
spu: str
title: str
brand: str = '' # 品牌
desc: str = '' # 简介
attribute_type: str # 属性类型 S M
images: list[str]
attribute_names: list = [] # 属性名称1,2,3
price: float = 0.00
original_price: float = 0.00
attributes: list[Attributes] = [] # 变体列表
attribute_urls: list[str] = [] # 变体链接列表(筛选用,可略)
other_data: dict = {} # 其他参数
```
## 2.统一操作方法
### 2.1 获取图片链接
```python
def get_image_urls() -> list[str]:
"""
获取商品 url 列表
:return list: [url: str]
"""
image_urls = []
# 获取图片列表元素
image_eles = tab.eles('')
for image_ele in image_eles:
# 定位到 img 标签并提取src
image_url = image_ele.ele('').attr('src')
image_urls.append(image_url)
return image_urls
```
### 2.2 获取价格
```python
def get_prices(self) -> float:
"""
获取 价格(price)/原价(original_price)
:return price, original_price
"""
price = tab.ele('').text # 价格
original_price = tab.ele('').text # 原价
######################格式化处理#######################
######################################################
return price, original_price
```
### 2.3 获取变体
```python
def get_attributes(self) -> list[dict]:
"""
获取商品属性
attribute = {
'url': '变体链接(如果有)',
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
'price': 0.0,
'original_price': 0.0,
'images': []
}
:return list: [attribute1, attribute2, ...]
"""
def get_size_eles():
pass
# ...更多子属性...
attributes = []
#########################操作区域#############################
###############################################################
return attributes
```
## 3.完整示例
```python
def get_goods_info(self, category, url) -> types.GoodsInfo:
def get_image_urls() -> list[str]:
"""
获取商品 url 列表
:return list: [url: str]
"""
image_urls = []
# 获取图片列表元素
image_eles = tab.eles('')
for image_ele in image_eles:
# 定位到 img 标签并提取src
image_url = image_ele.ele('').attr('src')
image_urls.append(image_url)
return image_urls
def get_prices(self) -> float:
"""
获取 价格(price)/原价(original_price)
:return price, original_price
"""
price = tab.ele('').text # 价格
original_price = tab.ele('').text # 原价
######################格式化处理#######################
######################################################
return price, original_price
def get_attributes(self) -> list[dict]:
"""
获取商品属性
attribute = {
'url': '变体链接(如果有)',
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
'price': 0.0,
'original_price': 0.0,
'images': []
}
:return list: [attribute1, attribute2, ...]
"""
def get_size_eles():
pass
# ...更多子属性...
attributes = []
#########################操作区域#############################
# 示例
# color_eles = tab.eles('')
# for color_ele in color_eles:
# color_url = color_ele.ele('').attr('href')
# color_name = color_ele.text
# color_images = get_image_urls()
# color_price, color_original_price = get_prices()
# tab.get(color_url) # 链接访问或点击
# # color_ele.click()
# size_eles = get_size_eles()
# for size_ele in size_eles:
# size_name = size_ele.text
# # ...item3...
# attributes.append({
# 'url': color_url,
# 'items': [color_name, size_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': []
# })
# if len(size_eles) == 0:
# attributes.append({
# 'url': color_url,
# 'items': [color_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': color_images
# })
###############################################################
return attributes
tab = self.browser.new_tab(url)
tab.set.window.max()
#####################基本信息########################
spu = formats.url_to_spu(url)
desc = "简介"
brand = "品牌"
title = "标题"
####################################################
price, original_price = get_prices()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
original_price=original_price,
images=get_image_urls()
)
m_data = []
attributes = get_attributes()
if len(attributes) == 0:
goods_info.attribute_type = 'S'
goods_info.attributes = []
goods_info.attribute_names = []
else:
goods_info.attribute_type = 'M'
goods_info.attribute_names = m_data
for attribute in attributes:
goods_info.attributes.append(types.GoodsInfo.Attributes(
url=attribute['url'],
attribute_values=attribute['items'],
price=attribute['price'],
original_price=attribute['original_price'],
images=attribute['images']
))
for image_url in attribute['images']:
if image_url not in goods_info.images:
goods_info.images.append(image_url)
try:
tab.close()
except:
pass
return goods_info
```

4
core/goods.py
View File

@@ -186,7 +186,7 @@ class GoodsModel:
for item in db_data:
if item['spu'] in index_data:
index_data[item['spu']].p_lists.append(
types.GoodsInfo.GoodsInfoP(
types.GoodsInfo.Attributes(
attr_items = [item['son_item1'], item['son_item2'], item['son_item3']],
price = item['son_price'],
old_price = item['son_old_price'],
@@ -212,7 +212,7 @@ class GoodsModel:
)
if item['attr'] == 'M':
index_data[item['spu']].p_lists.append(
types.GoodsInfo.GoodsInfoP(
types.GoodsInfo.Attributes(
attr_items = [item['son_item1'], item['son_item2'], item['son_item3']],
price = item['son_price'],
old_price = item['son_old_price'],

117
core/types.py
View File

@@ -10,47 +10,52 @@ class GoodsInfo(BaseModel):
"""
商品类(包含处理)
:param url: 主商品链接 [必填]
:param spu: [必填]
:param title: 商品标题 [必填]
:param attribute_type: 属性类型 S|M [必填]
:param images: 图片列表 [必填]
:param brand: 品牌
:param desc: 简介
:param attr: 属性 S|M
:param attr_items: 变体[名称]列表 [Color, Size]
:param attribute_names: 属性[名称]列表 [Color, Size]
:param price: 价格
:param old_price: 原价
:param images: 图片列表
:param url: 主商品链接
:param p_urls: 变体链接列表
:param p_lists: 变体类列表
:param original_price: 原价
:param attributes: 变体类列表
:param attribute_urls: 变体链接列表
:param other_data: 其他参数
"""
class GoodsInfoP(BaseModel):
class Attributes(BaseModel):
"""
变体类
:param attr_items: 变体[值]列表 [red, 30]
:param attribute_values: 变体[值]列表 [red, 30] [必填]
:param price: 价格 [必填]
:param original_price: 原价 [必填]
:param url: 变体链接
:param images: 变体图片列表
:param other_data: 其他参数
"""
attr_items: list = [] # 变体名称1,2,3
price: float = 0.00
old_price: float = 0.00
images: list[str] = []
url: str = '' # 商品链接
attribute_values: list # 属性值1,2,3
price: float
original_price: float
images: list[str] = []
other_data: dict = {} # 其他参数
url: str
spu: str
title: str
brand: str = '' # 品牌
desc: str = '' # 简介
attr: str = 'S' # 属性 S M
attr_items: list = [] # 变体名称1,2,3
attribute_type: str # 属性类型 S M
images: list[str]
attribute_names: list = [] # 属性名称1,2,3
price: float = 0.00
old_price: float = 0.00
images: list[str] = []
url: str = '' # 商品链接
p_urls: list[str] = [] # 变体链接列表(筛选用,可略)
p_lists: list[GoodsInfoP] = [] # 变体列表
original_price: float = 0.00
attributes: list[Attributes] = [] # 变体列表
attribute_urls: list[str] = [] # 变体链接列表(筛选用,可略)
other_data: dict = {} # 其他参数
def to_db_data(self):
@@ -60,30 +65,30 @@ class GoodsInfo(BaseModel):
'spu': self.spu,
'title': self.title,
'brand': self.brand,
'attr': self.attr,
'attr': self.attribute_type,
'desc': self.desc,
'price': self.price,
'old_price': self.old_price,
'original_price': self.original_price,
'images': orjson.dumps(self.images),
'url': self.url,
'other_data': orjson.dumps(self.other_data)
}
for index, item_name in enumerate(self.attr_items):
for index, item_name in enumerate(self.attribute_names):
if item_name:
main_info[f"item{index+1}"] = item_name
son_infos = []
for p_info in self.p_lists:
for p_info in self.attributes:
item = {
'spu': self.spu,
'price': p_info.price,
'old_price': p_info.old_price,
'original_price': p_info.original_price,
'images': orjson.dumps(p_info.images),
'url': p_info.url,
'other_data': orjson.dumps(p_info.other_data)
}
p_info.attr_items += [None, None]
for index, item_name in enumerate(p_info.attr_items[:3]):
p_info.attribute_values += [None, None]
for index, item_name in enumerate(p_info.attribute_values[:3]):
if item_name:
item[f"item{index+1}"] = item_name
son_infos.append(item)
@@ -92,18 +97,18 @@ class GoodsInfo(BaseModel):
def to_shopyy_row_data(self):
"""转为shopyy行数据"""
prices = [self.price, self.old_price]
for p_info in self.p_lists:
prices = [self.price, self.original_price]
for p_info in self.attributes:
prices.append(p_info.price)
prices.append(p_info.old_price)
old_price = max(prices)
prices.append(p_info.original_price)
original_price = max(prices)
rows_data = []
rows_data.append([
"", # 商品ID（系统生成，留空）
"",
f"{self.brand} {self.title}" if self.brand != "" else self.title,
self.attr,
self.attribute_type,
"",
"",
self.desc,
@@ -122,9 +127,9 @@ class GoodsInfo(BaseModel):
self.brand, # 专辑名称
"",
"",
*(self.attr_items[:3] + [''] * (3 - len(self.attr_items)))[:3], # 款式1/2/3
*(self.attribute_names[:3] + [''] * (3 - len(self.attribute_names)))[:3], # 款式1/2/3
self.price,
old_price,
original_price,
"",
"",
"",
@@ -133,14 +138,14 @@ class GoodsInfo(BaseModel):
'', # 变体备注2
','.join(self.images) if self.images else '',
self.url,
','.join(self.p_urls) if self.p_urls else ''
','.join(self.attribute_urls) if self.attribute_urls else ''
])
p_index = []
for p_info in self.p_lists:
if p_info.attr_items in p_index:
for p_info in self.attributes:
if p_info.attribute_values in p_index:
continue
p_index.append(p_info.attr_items)
p_index.append(p_info.attribute_values)
rows_data.append([
"", # 商品ID（系统生成，留空）
"",
@@ -164,9 +169,9 @@ class GoodsInfo(BaseModel):
self.brand, # 专辑名称
"",
"",
*(p_info.attr_items[:3] + [''] * (3 - len(p_info.attr_items)))[:3], # 款式1/2/3
*(p_info.attribute_values[:3] + [''] * (3 - len(p_info.attribute_values)))[:3], # 款式1/2/3
p_info.price,
old_price,
original_price,
"",
"",
"",
@@ -181,7 +186,7 @@ class GoodsInfo(BaseModel):
def to_woo_row_data(self, category_name):
"""转为woo行数据"""
def get_lists(attr = 'variable',spu = '', price = '', old_price = '', item_name1: str = '', item_value1: str = '', item_name2: str = '', item_value2: str = '', item_name3: str = '', item_value3: str = '', images: list = []):
def get_lists(attr = 'variable',spu = '', price = '', original_price = '', item_name1: str = '', item_value1: str = '', item_name2: str = '', item_value2: str = '', item_name3: str = '', item_value3: str = '', images: list = []):
return [
attr,
self.spu if attr == 'variable' or attr == 'simple' else "",
@@ -191,7 +196,7 @@ class GoodsInfo(BaseModel):
1,
100,
price,
old_price,
original_price,
category_name if attr == 'variable' or attr == 'simple' else "",
','.join(images) if images else '',
self.spu if attr == 'variation' else "",
@@ -208,12 +213,12 @@ class GoodsInfo(BaseModel):
desc = BeautifulSoup(self.desc, 'html.parser').get_text()
rows_data = []
if self.attr == 'S':
if self.attribute_type == 'S':
rows_data.append(get_lists(
attr='simple',
spu=self.spu,
price=self.price,
old_price=self.old_price,
original_price=self.original_price,
images=self.images
))
else:
@@ -221,10 +226,10 @@ class GoodsInfo(BaseModel):
item_values1 = set()
item_values2 = set()
item_values3 = set()
for p_info in self.p_lists:
item_value1 = p_info.attr_items[0] if len(p_info.attr_items) >= 1 else ""
item_value2 = p_info.attr_items[1] if len(p_info.attr_items) >= 2 else ""
item_value3 = p_info.attr_items[2] if len(p_info.attr_items) >= 3 else ""
for p_info in self.attributes:
item_value1 = p_info.attribute_values[0] if len(p_info.attribute_values) >= 1 else ""
item_value2 = p_info.attribute_values[1] if len(p_info.attribute_values) >= 2 else ""
item_value3 = p_info.attribute_values[2] if len(p_info.attribute_values) >= 3 else ""
item_value1 = item_value1.replace(',', '，').replace('|', ' ')
item_values1.add(item_value1)
@@ -239,25 +244,25 @@ class GoodsInfo(BaseModel):
attr='variation',
spu='',
price=p_info.price,
old_price=p_info.old_price,
original_price=p_info.original_price,
images=p_info.images,
item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "",
item_name1=self.attribute_names[0] if len(self.attribute_names) >= 1 else "",
item_value1=item_value1,
item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "",
item_name2=self.attribute_names[1] if len(self.attribute_names) >= 2 else "",
item_value2=item_value2,
item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "",
item_name3=self.attribute_names[2] if len(self.attribute_names) >= 3 else "",
item_value3=item_value3
))
rows_data.insert(0, get_lists(
spu=self.spu,
price='',
old_price='',
original_price='',
images=self.images,
item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "",
item_name1=self.attribute_names[0] if len(self.attribute_names) >= 1 else "",
item_value1='|'.join(item_values1),
item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "",
item_name2=self.attribute_names[1] if len(self.attribute_names) >= 2 else "",
item_value2='|'.join(item_values2),
item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "",
item_name3=self.attribute_names[2] if len(self.attribute_names) >= 3 else "",
item_value3='|'.join(item_values3)
))
return rows_data

176
lib/example.py
View File

@@ -7,71 +7,130 @@ class SpiderModule(spiders.Spiders):
def __init__(self):
super().__init__(
self.project_name,
bitch = 10, # 运行批次大小
worker_num = 1, # 线程数
bitch = 10,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
# 获取导航入口
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('')
# 一级分类
one_menu_eles = self.tab.eles('')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-').replace(',', ' ')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
# 二级分类
two_menu_eles = one_menu_ele.eles('')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-').replace(',', ' ')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
# 三级分类
three_menu_eles = two_menu_ele.eles('')
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-').replace(',', ' ')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
pass
return category_urls
# 获取商品链接入口
def get_goods_urls(self, category_url: str) -> list[str]:
def get_goods_urls(self, category, category_url: str) -> list[str]:
self.tab.get(f"{category_url}")
goods_urls = []
pass
return goods_urls
return {'category': category, 'urls': goods_urls}
# 获取商品详情入口
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
def get_goods_info(self, category, url) -> types.GoodsInfo:
def get_image_urls() -> list[str]:
"""
获取商品 url 列表
:return list: [url: str]
"""
image_urls = []
pass
# 获取图片列表元素
image_eles = tab.eles('')
for image_ele in image_eles:
# 定位到 img 标签并提取src
image_url = image_ele.ele('').attr('src')
image_urls.append(image_url)
return image_urls
def get_prices(self) -> float:
"""
获取 价格(price)/原价(original_price)
:return price, original_price
"""
price = tab.ele('').text # 价格
original_price = tab.ele('').text # 原价
######################格式化处理#######################
######################################################
return price, original_price
def get_attributes(self) -> list[dict]:
"""
获取商品属性
attribute = {
'url': '变体链接(如果有)',
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
'price': 0.0,
'original_price': 0.0,
'images': []
}
:return list: [attribute1, attribute2, ...]
"""
def get_size_eles():
pass
# ...更多子属性...
attributes = []
#########################操作区域#############################
# 示例
# color_eles = tab.eles('')
# for color_ele in color_eles:
# color_url = color_ele.ele('').attr('href')
# color_name = color_ele.text
# color_images = get_image_urls()
# color_price, color_original_price = get_prices()
# tab.get(color_url) # 链接访问或点击
# # color_ele.click()
# size_eles = get_size_eles()
# for size_ele in size_eles:
# size_name = size_ele.text
# # ...item3...
# attributes.append({
# 'url': color_url,
# 'items': [color_name, size_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': []
# })
# if len(size_eles) == 0:
# attributes.append({
# 'url': color_url,
# 'items': [color_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': color_images
# })
###############################################################
return attributes
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
#####################基本信息########################
spu = formats.url_to_spu(url)
desc = "简介"
brand = "品牌"
title = "标题"
price = 0.00
old_price = price
####################################################
main_images = get_image_urls()
price, original_price = get_prices()
goods_info = types.GoodsInfo(
title=title,
@@ -80,32 +139,31 @@ class SpiderModule(spiders.Spiders):
url=url,
spu=spu,
price=price,
old_price=old_price,
original_price=original_price,
images=get_image_urls()
)
m_data = []
attr_items = []
attributes = get_attributes()
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
if len(attributes) == 0:
goods_info.attribute_type = 'S'
goods_info.attributes = []
goods_info.attribute_names = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images']
goods_info.attribute_type = 'M'
goods_info.attribute_names = m_data
for attribute in attributes:
goods_info.attributes.append(types.GoodsInfo.Attributes(
url=attribute['url'],
attribute_values=attribute['items'],
price=attribute['price'],
original_price=attribute['original_price'],
images=attribute['images']
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
for image_url in attribute['images']:
if image_url not in goods_info.images:
goods_info.images.append(image_url)
try:
tab.close()
except: