137 lines
4.9 KiB
Python
137 lines
4.9 KiB
Python
from core import spiders, types
|
||
from utils import formats, messages
|
||
|
||
class SpiderModule(spiders.Spiders):
|
||
project_name = "condom69"
|
||
|
||
def __init__(self):
|
||
super().__init__(
|
||
self.project_name,
|
||
bitch = 30,
|
||
worker_num = 5,
|
||
switch_clash = False,
|
||
reflush_browser = False
|
||
)
|
||
self.tab.get('https://www.condom69.net/')
|
||
# input('wait:')
|
||
|
||
def get_category_urls(self) -> list[str]:
|
||
category_urls = []
|
||
|
||
one_menu_eles = self.tab.eles('xpath=//*[@id="nav-one"]/li')
|
||
for one_menu_ele in one_menu_eles[:-1]:
|
||
one_link_ele = one_menu_ele.ele('xpath=a')
|
||
one_url = one_link_ele.attr('href')
|
||
one_name = one_link_ele.text.replace('/', '-')
|
||
category_urls.append(f"{one_url}#{one_name}")
|
||
messages.sendInfo(one_name)
|
||
|
||
two_menu_eles = one_menu_ele.eles('xpath=div/ul/li', timeout=1)
|
||
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
||
two_link_ele = two_menu_ele.ele('xpath=div/a')
|
||
two_url = two_link_ele.attr('href')
|
||
two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ')
|
||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||
|
||
# try:
|
||
# three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
|
||
# except:
|
||
# continue
|
||
# for three_menu_ele in three_menu_eles:
|
||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||
# three_url = three_link_ele.attr('href')
|
||
# three_name = three_link_ele.text.replace('/', '-')
|
||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||
|
||
return category_urls
|
||
|
||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||
page = 1
|
||
self.tab.get(f"{category_url}&pg={page}")
|
||
|
||
#####获取页数####
|
||
|
||
#################
|
||
|
||
goods_urls = []
|
||
url_index = {}
|
||
while True:
|
||
self.tab.scroll.to_bottom()
|
||
goods_eles = self.tab.eles('.divProdItemImg2Inner')
|
||
page_urls = []
|
||
for goods_ele in goods_eles:
|
||
goods_url = f"{goods_ele.ele('xpath=a').attr('href')}"
|
||
if goods_url not in url_index:
|
||
page_urls.append(goods_url)
|
||
url_index[goods_url] = None
|
||
if len(page_urls) > 0:
|
||
goods_urls += page_urls
|
||
else:
|
||
break
|
||
page += 1
|
||
self.tab.get(f"{category_url}&pg={page}")
|
||
|
||
messages.sendInfo(f"{len(goods_urls)} 个")
|
||
|
||
return goods_urls
|
||
|
||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||
def get_image_urls():
|
||
image_urls = []
|
||
image_eles = tab.eles('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[1]/div[2]/div')
|
||
|
||
for image_ele in image_eles:
|
||
image_url = image_ele.ele('xpath=div/div/a/img').attr('src')
|
||
image_urls.append(image_url)
|
||
return image_urls
|
||
|
||
tab = self.browser.new_tab(url)
|
||
tab.set.window.max()
|
||
spu = formats.url_to_spu(url)
|
||
|
||
try:
|
||
h1_ele = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[1]')
|
||
brand = ''
|
||
try:
|
||
brand = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/table/tbody/tr/td[3]/a').text
|
||
except:
|
||
pass
|
||
title = h1_ele.text
|
||
|
||
desc = ""
|
||
try:
|
||
desc = formats.clean_html(tab.ele('.ctl00_cphContent_ucUsrIndProduct_pnlIndProdDesc').html)
|
||
except:
|
||
pass
|
||
|
||
price = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[3]/div[1]/div[2]/span').text.replace(',', '')
|
||
old_price = price
|
||
|
||
main_images = get_image_urls()
|
||
|
||
goods_info = types.GoodsInfo(
|
||
title=title,
|
||
desc=desc,
|
||
brand=brand,
|
||
url=url,
|
||
spu=spu,
|
||
price=price,
|
||
old_price=old_price,
|
||
)
|
||
|
||
goods_info.attr = 'S'
|
||
goods_info.attr_items = []
|
||
goods_info.images = main_images
|
||
goods_info.p_lists = []
|
||
except Exception as e:
|
||
try:
|
||
tab.close()
|
||
except:
|
||
pass
|
||
raise Exception(e)
|
||
try:
|
||
tab.close()
|
||
except:
|
||
pass
|
||
return goods_info
|