Cerys/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

调整数据结构

Browse Source
This commit is contained in:
Cerys
2026-05-22 11:53:25 +08:00
parent c360413981
commit 73287d8fe1
4 changed files with 448 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
lib/example.py
View File

@@ -7,71 +7,130 @@ class SpiderModule(spiders.Spiders):
def __init__(self):
super().__init__(
self.project_name,
bitch = 10, # 运行批次大小
worker_num = 1, # 线程数
bitch = 10,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
# 获取导航入口
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('')
# 一级分类
one_menu_eles = self.tab.eles('')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-').replace(',', ' ')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
# 二级分类
two_menu_eles = one_menu_ele.eles('')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-').replace(',', ' ')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
# 三级分类
three_menu_eles = two_menu_ele.eles('')
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-').replace(',', ' ')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
pass
return category_urls
# 获取商品链接入口
def get_goods_urls(self, category_url: str) -> list[str]:
def get_goods_urls(self, category, category_url: str) -> list[str]:
self.tab.get(f"{category_url}")
goods_urls = []
pass
return goods_urls
return {'category': category, 'urls': goods_urls}
# 获取商品详情入口
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
def get_goods_info(self, category, url) -> types.GoodsInfo:
def get_image_urls() -> list[str]:
"""
获取商品 url 列表
:return list: [url: str]
"""
image_urls = []
pass
# 获取图片列表元素
image_eles = tab.eles('')
for image_ele in image_eles:
# 定位到 img 标签并提取src
image_url = image_ele.ele('').attr('src')
image_urls.append(image_url)
return image_urls
def get_prices(self) -> float:
"""
获取 价格(price)/原价(original_price)
:return price, original_price
"""
price = tab.ele('').text # 价格
original_price = tab.ele('').text # 原价
######################格式化处理#######################
######################################################
return price, original_price
def get_attributes(self) -> list[dict]:
"""
获取商品属性
attribute = {
'url': '变体链接(如果有)',
'items': ['变体名称1(color_name/size_name/...)', '变体名称2', '...'],
'price': 0.0,
'original_price': 0.0,
'images': []
}
:return list: [attribute1, attribute2, ...]
"""
def get_size_eles():
pass
# ...更多子属性...
attributes = []
#########################操作区域#############################
# 示例
# color_eles = tab.eles('')
# for color_ele in color_eles:
# color_url = color_ele.ele('').attr('href')
# color_name = color_ele.text
# color_images = get_image_urls()
# color_price, color_original_price = get_prices()
# tab.get(color_url) # 链接访问或点击
# # color_ele.click()
# size_eles = get_size_eles()
# for size_ele in size_eles:
# size_name = size_ele.text
# # ...item3...
# attributes.append({
# 'url': color_url,
# 'items': [color_name, size_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': []
# })
# if len(size_eles) == 0:
# attributes.append({
# 'url': color_url,
# 'items': [color_name],
# 'price': color_price,
# 'original_price': color_original_price,
# 'images': color_images
# })
###############################################################
return attributes
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
#####################基本信息########################
spu = formats.url_to_spu(url)
desc = "简介"
brand = "品牌"
title = "标题"
price = 0.00
old_price = price
####################################################
main_images = get_image_urls()
price, original_price = get_prices()
goods_info = types.GoodsInfo(
title=title,
@@ -80,32 +139,31 @@ class SpiderModule(spiders.Spiders):
url=url,
spu=spu,
price=price,
old_price=old_price,
original_price=original_price,
images=get_image_urls()
)
m_data = []
attr_items = []
attributes = get_attributes()
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
if len(attributes) == 0:
goods_info.attribute_type = 'S'
goods_info.attributes = []
goods_info.attribute_names = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images']
goods_info.attribute_type = 'M'
goods_info.attribute_names = m_data
for attribute in attributes:
goods_info.attributes.append(types.GoodsInfo.Attributes(
url=attribute['url'],
attribute_values=attribute['items'],
price=attribute['price'],
original_price=attribute['original_price'],
images=attribute['images']
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
for image_url in attribute['images']:
if image_url not in goods_info.images:
goods_info.images.append(image_url)
try:
tab.close()
except: