218 lines
8.1 KiB
Python
218 lines
8.1 KiB
Python
from core import spiders, types
|
|
from utils import formats, messages
|
|
|
|
class SpiderModule(spiders.Spiders):
|
|
project_name = "peek-cloppenburg"
|
|
|
|
def __init__(self):
|
|
super().__init__(
|
|
self.project_name,
|
|
bitch = 30,
|
|
worker_num = 5,
|
|
switch_clash = False,
|
|
reflush_browser = False
|
|
)
|
|
|
|
def get_category_urls(self) -> list[str]:
|
|
category_urls = []
|
|
|
|
self.tab.get('https://www.peek-cloppenburg.at/')
|
|
one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a')
|
|
for one_menu_ele in one_menu_eles:
|
|
one_link_ele = one_menu_ele
|
|
one_url = one_link_ele.attr('href')
|
|
one_name = one_link_ele.text.replace('/', '-')
|
|
category_urls.append(f"{one_url}#{one_name}")
|
|
messages.sendInfo(one_name)
|
|
one_link_ele.click()
|
|
self.tab.wait(3)
|
|
|
|
two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/a', timeout=1)
|
|
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
|
two_link_ele = two_menu_ele
|
|
two_url = two_link_ele.attr('href')
|
|
two_name = two_link_ele.text.replace('/', '-')
|
|
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
|
|
|
try:
|
|
three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
|
|
except:
|
|
continue
|
|
for three_menu_ele in three_menu_eles:
|
|
three_link_ele = three_menu_ele.ele('@tag()=a')
|
|
three_url = three_link_ele.attr('href')
|
|
three_name = three_link_ele.text.replace('/', '-')
|
|
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
|
|
|
return category_urls
|
|
|
|
def get_goods_urls(self, category_url: str) -> list[str]:
|
|
self.tab.get(f"{category_url}?page=1")
|
|
goods_urls = []
|
|
|
|
max_page = 1
|
|
#####获取页数####
|
|
try:
|
|
max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text)
|
|
except:
|
|
pass
|
|
#################
|
|
|
|
goods_urls = []
|
|
if max_page > 1:
|
|
for page in range(1, max_page+1):
|
|
if page > 1:
|
|
self.tab.get(f"{category_url}?page={page}&userId=123456")
|
|
|
|
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
|
|
for goods_ele in goods_eles:
|
|
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
|
goods_urls.append(goods_url)
|
|
messages.sendInfo(f"{page}/{max_page}")
|
|
else:
|
|
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
|
|
for goods_ele in goods_eles:
|
|
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
|
goods_urls.append(goods_url)
|
|
|
|
return goods_urls
|
|
|
|
def get_goods_info(self, url) -> types.GoodsInfo:
|
|
def get_image_urls():
|
|
image_urls = []
|
|
image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a')
|
|
|
|
for image_ele in image_eles:
|
|
image_url = image_ele.attr('href')
|
|
image_urls.append(image_url)
|
|
return image_urls
|
|
|
|
def get_size_eles():
|
|
try:
|
|
eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li')
|
|
except:
|
|
pass
|
|
try:
|
|
eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li')
|
|
except:
|
|
pass
|
|
try:
|
|
eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li')
|
|
except:
|
|
pass
|
|
try:
|
|
eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li')
|
|
except:
|
|
pass
|
|
return eles
|
|
|
|
tab = self.browser.new_tab(url)
|
|
tab.set.window.max()
|
|
spu = formats.url_to_spu(url)
|
|
|
|
try:
|
|
h1_eles = tab.ele('@tag()=h1', timeout=10).eles('@tag()=span')
|
|
brand = ''
|
|
try:
|
|
brand = h1_eles[0].text
|
|
title = h1_eles[1].text
|
|
except:
|
|
title = tab.ele('@tag()=h1').text
|
|
|
|
desc = ""
|
|
try:
|
|
desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}")
|
|
except:
|
|
pass
|
|
|
|
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '.')
|
|
try:
|
|
old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '.')
|
|
except:
|
|
old_price = price
|
|
|
|
main_images = get_image_urls()
|
|
|
|
goods_info = types.GoodsInfo(
|
|
title=title,
|
|
desc=desc,
|
|
brand=brand,
|
|
url=url,
|
|
spu=spu,
|
|
price=price,
|
|
old_price=old_price,
|
|
)
|
|
|
|
m_data = []
|
|
attr_items = []
|
|
color_urls = []
|
|
try:
|
|
color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li')
|
|
m_data.append('Farbe')
|
|
for color_ele in color_eles:
|
|
color_url = color_ele.ele('@tag()=a').attr('href')
|
|
color_urls.append(color_url)
|
|
except:
|
|
pass
|
|
m_data.append('Größe')
|
|
|
|
if len(color_urls) > 0:
|
|
for color_url in color_urls:
|
|
color_images = main_images
|
|
if color_url != url:
|
|
tab.get(color_url)
|
|
color_images = get_image_urls()
|
|
|
|
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '')
|
|
try:
|
|
old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '')
|
|
except:
|
|
old_price = price
|
|
|
|
color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]')
|
|
|
|
size_eles = get_size_eles()
|
|
for size_ele in size_eles:
|
|
size_name = size_ele.ele('.text-cp-md truncate').text
|
|
|
|
attr_items.append({
|
|
'url': color_url,
|
|
'items': [color_name, size_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': color_images
|
|
})
|
|
|
|
if len(attr_items) == 0:
|
|
goods_info.attr = 'S'
|
|
goods_info.attr_items = []
|
|
goods_info.images = main_images
|
|
goods_info.p_lists = []
|
|
else:
|
|
goods_info.attr = 'M'
|
|
goods_info.attr_items = m_data
|
|
for attr_item in attr_items:
|
|
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
|
url=attr_item['url'],
|
|
attr_items=attr_item['items'],
|
|
price=attr_item['price'],
|
|
old_price=attr_item['old_price'],
|
|
images=attr_item['images'],
|
|
))
|
|
for image_url in attr_item['images']:
|
|
if image_url not in main_images:
|
|
main_images.append(image_url)
|
|
goods_info.images = main_images
|
|
goods_info.p_urls = color_urls
|
|
except Exception as e:
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
raise Exception(e)
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
return goods_info
|