初始化
This commit is contained in:
325
lib/module/ancientnutrition.py
Normal file
325
lib/module/ancientnutrition.py
Normal file
@@ -0,0 +1,325 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "ancientnutrition"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 1,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://ancientnutrition.com/')
|
||||
one_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[4]/div[1]/div[1]/div')
|
||||
for index, one_menu_ele in enumerate(one_menu_eles[1:]):
|
||||
one_link_ele = one_menu_ele.ele('xpath=div/a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
print(index+2)
|
||||
two_menu_eles = self.tab.eles(f'xpath=//*[@id="__next"]/div[4]/div[{index+2}]/div/div/div[2]/div/a')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(f"{category_url}")
|
||||
goods_urls = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.tab.scroll.to_bottom()
|
||||
self.tab.ele('text=Load More').click()
|
||||
except:
|
||||
break
|
||||
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[9]/div/div[2]/div[2]/div')
|
||||
for goods_ele in goods_eles:
|
||||
try:
|
||||
goods_link_ele = goods_ele.ele('@tag()=a')
|
||||
except Exception as e:
|
||||
continue
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
image_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[1]/div/div[2]/div/div[1]/div')
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele.ele('xpath=div/span/img')
|
||||
image_urls.append(image_link_ele.attr('src').replace('_150x150', '_1450x1450'))
|
||||
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
# try:
|
||||
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
|
||||
# except:
|
||||
# pass
|
||||
# if brand == '':
|
||||
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
# else:
|
||||
title = tab.ele('@tag()=h1').text
|
||||
|
||||
desc = formats.clean_html(tab.ele('#productDescription').html)
|
||||
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
try:
|
||||
tab.ele('text=Select Flavor').click()
|
||||
m_data.append('Flavor')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
tab.ele('text=Select Size').click()
|
||||
m_data.append('Size')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
tab.ele('text=Select Quantity').click()
|
||||
m_data.append('Quantity')
|
||||
except:
|
||||
pass
|
||||
|
||||
if 'Flavor' in m_data:
|
||||
tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click()
|
||||
color_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div[2]/div')
|
||||
color_xpaths = []
|
||||
for color_ele in color_eles:
|
||||
color_xpaths.append(color_ele.xpath)
|
||||
|
||||
for color_xpath in color_xpaths:
|
||||
while True:
|
||||
tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click()
|
||||
try:
|
||||
color_ele = tab.ele(f"xpath={color_xpath}")
|
||||
color_name = color_ele.text
|
||||
break
|
||||
except:
|
||||
tab.get(url)
|
||||
color_ele.click()
|
||||
tab.wait(3)
|
||||
color_images = get_image_urls()
|
||||
if 'Size' in m_data:
|
||||
try:
|
||||
size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}")
|
||||
except:
|
||||
size_name = 'Default'
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name, quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
size_xpaths = []
|
||||
for size_ele in size_eles:
|
||||
try:
|
||||
size_xpaths.append(size_ele.xpath)
|
||||
except:
|
||||
continue
|
||||
|
||||
for size_xpath in size_xpaths:
|
||||
try:
|
||||
size_ele = tab.ele(f"xpath={size_xpath}")
|
||||
size_name = size_ele.text
|
||||
except:
|
||||
continue
|
||||
size_ele.click()
|
||||
tab.wait(3)
|
||||
color_images = get_image_urls()
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name, quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
color_images = []
|
||||
if 'Size' in m_data:
|
||||
size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}")
|
||||
size_xpaths = []
|
||||
for size_ele in size_eles:
|
||||
try:
|
||||
size_xpaths.append(size_ele.xpath)
|
||||
except:
|
||||
continue
|
||||
|
||||
for size_xpath in size_xpaths:
|
||||
try:
|
||||
size_ele = tab.ele(f"xpath={size_xpath}")
|
||||
size_name = size_ele.text
|
||||
except:
|
||||
continue
|
||||
size_ele.click()
|
||||
tab.wait(2)
|
||||
color_images = get_image_urls()
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [size_name, quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
Reference in New Issue
Block a user