删除旧文件
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -1,176 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "agatameble"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 1,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.agatameble.pl/')
|
||||
one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
for three_menu_ele in three_menu_eles:
|
||||
three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
three_url = three_link_ele.attr('href')
|
||||
three_name = three_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(f"{category_url}")
|
||||
goods_urls = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.tab.scroll.to_bottom()
|
||||
self.tab.ele('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/div/div/a').click()
|
||||
except:
|
||||
break
|
||||
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/section/div/div/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('xpath=a')
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
tab.ele('xpath=//*[@id="contentContainer"]/form/div[1]/section/div/div[2]/button[1]').click()
|
||||
image_eles = tab.eles('xpath=//*[@id="root"]/div/section/div[2]/div/aside/div[2]/div/div/div/div[1]/div/div/div/div')
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele.ele('xpath=div/img[2]')
|
||||
image_urls.append(image_link_ele.attr('src').replace('width=', '').replace('height=', ''))
|
||||
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
try:
|
||||
brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
|
||||
except:
|
||||
pass
|
||||
if brand == '':
|
||||
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
else:
|
||||
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text
|
||||
|
||||
desc = ""
|
||||
desc_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[1]/div/div')[:3]
|
||||
for desc_ele in desc_eles:
|
||||
desc += formats.clean_html(desc_ele.html)
|
||||
|
||||
price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}")
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
color_urls = [url]
|
||||
color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li')
|
||||
for color_ele in color_eles[1:]:
|
||||
color_link_ele = color_ele.ele('xpath=a')
|
||||
color_urls.append(color_link_ele.attr('href'))
|
||||
if len(color_urls) > 1:
|
||||
m_data = ["Color"]
|
||||
|
||||
if len(color_urls) > 1:
|
||||
for color_url in color_urls:
|
||||
color_images = main_images
|
||||
if color_url != url:
|
||||
tab.get(color_url)
|
||||
if brand == '':
|
||||
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
else:
|
||||
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text
|
||||
color_images = get_image_urls()
|
||||
|
||||
price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}")
|
||||
old_price = price
|
||||
|
||||
try:
|
||||
item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text
|
||||
except:
|
||||
continue
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [item_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
goods_info.p_urls = color_urls
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,325 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "ancientnutrition"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 1,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://ancientnutrition.com/')
|
||||
one_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[4]/div[1]/div[1]/div')
|
||||
for index, one_menu_ele in enumerate(one_menu_eles[1:]):
|
||||
one_link_ele = one_menu_ele.ele('xpath=div/a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
print(index+2)
|
||||
two_menu_eles = self.tab.eles(f'xpath=//*[@id="__next"]/div[4]/div[{index+2}]/div/div/div[2]/div/a')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(f"{category_url}")
|
||||
goods_urls = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.tab.scroll.to_bottom()
|
||||
self.tab.ele('text=Load More').click()
|
||||
except:
|
||||
break
|
||||
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[9]/div/div[2]/div[2]/div')
|
||||
for goods_ele in goods_eles:
|
||||
try:
|
||||
goods_link_ele = goods_ele.ele('@tag()=a')
|
||||
except Exception as e:
|
||||
continue
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
image_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[1]/div/div[2]/div/div[1]/div')
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele.ele('xpath=div/span/img')
|
||||
image_urls.append(image_link_ele.attr('src').replace('_150x150', '_1450x1450'))
|
||||
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
# try:
|
||||
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
|
||||
# except:
|
||||
# pass
|
||||
# if brand == '':
|
||||
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
# else:
|
||||
title = tab.ele('@tag()=h1').text
|
||||
|
||||
desc = formats.clean_html(tab.ele('#productDescription').html)
|
||||
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
try:
|
||||
tab.ele('text=Select Flavor').click()
|
||||
m_data.append('Flavor')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
tab.ele('text=Select Size').click()
|
||||
m_data.append('Size')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
tab.ele('text=Select Quantity').click()
|
||||
m_data.append('Quantity')
|
||||
except:
|
||||
pass
|
||||
|
||||
if 'Flavor' in m_data:
|
||||
tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click()
|
||||
color_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div[2]/div')
|
||||
color_xpaths = []
|
||||
for color_ele in color_eles:
|
||||
color_xpaths.append(color_ele.xpath)
|
||||
|
||||
for color_xpath in color_xpaths:
|
||||
while True:
|
||||
tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click()
|
||||
try:
|
||||
color_ele = tab.ele(f"xpath={color_xpath}")
|
||||
color_name = color_ele.text
|
||||
break
|
||||
except:
|
||||
tab.get(url)
|
||||
color_ele.click()
|
||||
tab.wait(3)
|
||||
color_images = get_image_urls()
|
||||
if 'Size' in m_data:
|
||||
try:
|
||||
size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}")
|
||||
except:
|
||||
size_name = 'Default'
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name, quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
size_xpaths = []
|
||||
for size_ele in size_eles:
|
||||
try:
|
||||
size_xpaths.append(size_ele.xpath)
|
||||
except:
|
||||
continue
|
||||
|
||||
for size_xpath in size_xpaths:
|
||||
try:
|
||||
size_ele = tab.ele(f"xpath={size_xpath}")
|
||||
size_name = size_ele.text
|
||||
except:
|
||||
continue
|
||||
size_ele.click()
|
||||
tab.wait(3)
|
||||
color_images = get_image_urls()
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name, quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name, size_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [color_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
color_images = []
|
||||
if 'Size' in m_data:
|
||||
size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}")
|
||||
size_xpaths = []
|
||||
for size_ele in size_eles:
|
||||
try:
|
||||
size_xpaths.append(size_ele.xpath)
|
||||
except:
|
||||
continue
|
||||
|
||||
for size_xpath in size_xpaths:
|
||||
try:
|
||||
size_ele = tab.ele(f"xpath={size_xpath}")
|
||||
size_name = size_ele.text
|
||||
except:
|
||||
continue
|
||||
size_ele.click()
|
||||
tab.wait(2)
|
||||
color_images = get_image_urls()
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [size_name, quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
if 'Quantity' in m_data:
|
||||
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
|
||||
for quantity_index, quantity_ele in enumerate(quantity_eles):
|
||||
try:
|
||||
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
|
||||
quantity_name = quantity_ele.text
|
||||
except:
|
||||
continue
|
||||
quantity_ele.click()
|
||||
tab.wait(2)
|
||||
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
|
||||
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
|
||||
attr_items.append({
|
||||
'url': tab.url,
|
||||
'items': [quantity_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,201 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
from DrissionPage.common import Keys
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "bricomarche"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 5,
|
||||
worker_num = 1,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
self.tab.get('https://www.bricomarche.pl/')
|
||||
# input('wait:')
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.bricomarche.pl/')
|
||||
one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
for three_menu_ele in three_menu_eles:
|
||||
three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
three_url = three_link_ele.attr('href')
|
||||
three_name = three_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
page = 1
|
||||
limit = 100
|
||||
self.tab.get(f"{category_url}?page={page}&limit={limit}")
|
||||
if self.tab.ele('@tag()=h2').text == '执行安全验证':
|
||||
self.tab.wait(10)
|
||||
self.tab.actions.type((Keys.TAB, Keys.SPACE))
|
||||
goods_urls = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.tab.scroll.to_bottom()
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="spark"]/div[2]/div/section/div/div/main/div[2]/section/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('@tag()=a')
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
if len(goods_eles) == 0:
|
||||
break
|
||||
except:
|
||||
break
|
||||
page += 1
|
||||
self.tab.get(f"{category_url}?page={page}&limit={limit}")
|
||||
if self.tab.ele('@tag()=h2').text == '执行安全验证':
|
||||
self.tab.wait(10)
|
||||
self.tab.actions.type((Keys.TAB, Keys.SPACE))
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
image_eles = tab.eles('xpath=//*[@id="spark"]/div[2]/div/main/div/div[2]/section[1]/div/div[1]/div/div/div/div/div')
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele.ele('xpath=button/div/img')
|
||||
image_urls.append(image_link_ele.attr('src').replace('gallery_100_91', 'gallery'))
|
||||
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
while True:
|
||||
try:
|
||||
if tab.ele('@tag()=h2').text == '执行安全验证':
|
||||
tab.wait(10)
|
||||
tab.actions.type((Keys.TAB, Keys.SPACE))
|
||||
tab.wait(5)
|
||||
else:
|
||||
break
|
||||
except:
|
||||
break
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
# try:
|
||||
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
|
||||
# except:
|
||||
# pass
|
||||
|
||||
title = tab.ele('@tag()=h1', timeout=10).text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('.attributes').html)
|
||||
except:
|
||||
pass
|
||||
|
||||
# try:
|
||||
price_ele = tab.ele('.main-price')
|
||||
price = float(f"{price_ele.ele('.whole').text}{price_ele.ele('.cents').text.replace(' zł', '')}")
|
||||
# except:
|
||||
# price = 0.0
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
# color_urls = [url]
|
||||
# color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li')
|
||||
# for color_ele in color_eles[1:]:
|
||||
# color_link_ele = color_ele.ele('xpath=a')
|
||||
# color_urls.append(color_link_ele.attr('href'))
|
||||
# if len(color_urls) > 1:
|
||||
# m_data = ["Color"]
|
||||
|
||||
# if len(color_urls) > 1:
|
||||
# for color_url in color_urls:
|
||||
# color_images = main_images
|
||||
# if color_url != url:
|
||||
# tab.get(color_url)
|
||||
# if brand == '':
|
||||
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
# else:
|
||||
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text
|
||||
# color_images = get_image_urls()
|
||||
|
||||
# price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}")
|
||||
# old_price = price
|
||||
|
||||
# try:
|
||||
# item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text
|
||||
# except:
|
||||
# continue
|
||||
# attr_items.append({
|
||||
# 'url': color_url,
|
||||
# 'items': [item_name],
|
||||
# 'price': price,
|
||||
# 'old_price': old_price,
|
||||
# 'images': color_images
|
||||
# })
|
||||
|
||||
if len(attr_items) == 0:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
# goods_info.p_urls = color_urls
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,136 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "condom69"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 5,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
self.tab.get('https://www.condom69.net/')
|
||||
# input('wait:')
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
one_menu_eles = self.tab.eles('xpath=//*[@id="nav-one"]/li')
|
||||
for one_menu_ele in one_menu_eles[:-1]:
|
||||
one_link_ele = one_menu_ele.ele('xpath=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/ul/li', timeout=1)
|
||||
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
||||
two_link_ele = two_menu_ele.ele('xpath=div/a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
# try:
|
||||
# three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
|
||||
# except:
|
||||
# continue
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
page = 1
|
||||
self.tab.get(f"{category_url}&pg={page}")
|
||||
|
||||
#####获取页数####
|
||||
|
||||
#################
|
||||
|
||||
goods_urls = []
|
||||
url_index = {}
|
||||
while True:
|
||||
self.tab.scroll.to_bottom()
|
||||
goods_eles = self.tab.eles('.divProdItemImg2Inner')
|
||||
page_urls = []
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('xpath=a').attr('href')}"
|
||||
if goods_url not in url_index:
|
||||
page_urls.append(goods_url)
|
||||
url_index[goods_url] = None
|
||||
if len(page_urls) > 0:
|
||||
goods_urls += page_urls
|
||||
else:
|
||||
break
|
||||
page += 1
|
||||
self.tab.get(f"{category_url}&pg={page}")
|
||||
|
||||
messages.sendInfo(f"{len(goods_urls)} 个")
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
image_eles = tab.eles('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[1]/div[2]/div')
|
||||
|
||||
for image_ele in image_eles:
|
||||
image_url = image_ele.ele('xpath=div/div/a/img').attr('src')
|
||||
image_urls.append(image_url)
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
h1_ele = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[1]')
|
||||
brand = ''
|
||||
try:
|
||||
brand = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/table/tbody/tr/td[3]/a').text
|
||||
except:
|
||||
pass
|
||||
title = h1_ele.text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('.ctl00_cphContent_ucUsrIndProduct_pnlIndProdDesc').html)
|
||||
except:
|
||||
pass
|
||||
|
||||
price = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[3]/div[1]/div[2]/span').text.replace(',', '')
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,210 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "dk_one"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 1,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.dk_one.com/')
|
||||
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
if one_name == '라이프':
|
||||
continue
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(category_url)
|
||||
goods_urls = []
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="prodList"]/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('.prod-link')
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls(tab):
|
||||
image_urls = []
|
||||
image_eles = tab.ele('#prod-img').eles('@tag()=img')
|
||||
for image_ele in image_eles:
|
||||
image_urls.append(image_ele.attr('src'))
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
# try:
|
||||
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
|
||||
# except:
|
||||
# pass
|
||||
# if brand == '':
|
||||
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
# else:
|
||||
title = tab.ele('.page-title').text
|
||||
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('@@class=tab-content@@id=tabContent0202').html)
|
||||
except:
|
||||
desc = ''
|
||||
|
||||
price = float(tab.ele('.price-group').ele('.val').text.replace(',', ''))
|
||||
try:
|
||||
old_price = float(tab.ele('.price-group origin').text.replace(',', ''))
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls(tab)
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
images=main_images
|
||||
)
|
||||
|
||||
m_data = []
|
||||
color_data = {}
|
||||
color_eles = []
|
||||
# 颜色
|
||||
tab.ele('.opt-color')
|
||||
m_data.append('컬러')
|
||||
color_eles = tab.ele('.color-list-wrap').eles('.prod-color-rdo')
|
||||
|
||||
m_data.append('사이즈')
|
||||
|
||||
for color_ele in color_eles:
|
||||
color_input_onclick = color_ele.ele('@tag()=input').attr('onclick')
|
||||
have_url = False
|
||||
if 'location.href=' in color_input_onclick:
|
||||
color_url = 'https://dk-on.com' + formats.re_search(color_input_onclick, r"location\.href\s*=\s*'([^']*)'")
|
||||
color_name = color_url.split('/')[-1]
|
||||
have_url = True
|
||||
elif 'fnColorClicked' in color_input_onclick:
|
||||
color_url = goods_info.url
|
||||
color_name = color_ele.ele('@tag()=input').attr('value')
|
||||
color_ele.click()
|
||||
else:
|
||||
color_url = goods_info.url
|
||||
color_name = goods_info.title.split(' / ')[-1]
|
||||
|
||||
color_data[color_name] = {'url': color_url, 'have_url': have_url, 'ele': color_ele}
|
||||
goods_info.p_urls.append(color_url)
|
||||
|
||||
real_color_data = {}
|
||||
|
||||
|
||||
for color_name, color_item in color_data.items():
|
||||
color_tab = tab
|
||||
if color_item['have_url']:
|
||||
color_tab = self.browser.new_tab()
|
||||
|
||||
size_data = []
|
||||
if color_item['have_url']:
|
||||
color_tab.get(color_item['url'])
|
||||
else:
|
||||
color_item['ele'].click()
|
||||
color_tab.wait(1)
|
||||
color_item['price'] = str(color_tab.ele('.price-group').ele('.val').text.replace(',', ''))
|
||||
try:
|
||||
color_item['old_price'] = str(color_tab.ele('.price-group origin').text.replace(',', ''))
|
||||
except:
|
||||
color_item['old_price'] = color_item['price']
|
||||
|
||||
# 颜色对应商品图
|
||||
color_item['images'] = get_image_urls(tab = color_tab)
|
||||
|
||||
# 颜色对应尺码
|
||||
radio_eles = color_tab.ele('.prod-size-list').eles('.prod-size')
|
||||
for radio_ele in radio_eles:
|
||||
value = radio_ele.ele('@tag()=label').text
|
||||
if value == None:
|
||||
continue
|
||||
size_data.append(value)
|
||||
color_item['size'] = size_data
|
||||
|
||||
real_color_data[color_tab.ele('.page-title').text.split(' / ')[-1]] = color_item
|
||||
color_data = real_color_data
|
||||
|
||||
goods_info.attr = 'M'
|
||||
goods_info.title = goods_info.title.split(' / ')[0]
|
||||
goods_info.attr_items = m_data
|
||||
for color_name, color_item in color_data.items():
|
||||
for color_image in color_item['images']:
|
||||
if color_image in goods_info.images:
|
||||
continue
|
||||
goods_info.images.append(color_image)
|
||||
|
||||
# if color_data:
|
||||
item_data = []
|
||||
for color_name, color_item in color_data.items():
|
||||
for size in color_item['size']:
|
||||
item_data.append([color_name, size])
|
||||
for attribute_items in item_data:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
attr_items=attribute_items,
|
||||
price=color_data[attribute_items[0]]['price'],
|
||||
old_price=color_data[attribute_items[0]]['price'],
|
||||
images=color_data[attribute_items[0]]['images'],
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
color_tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
color_tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,141 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "friskybusiness"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 5,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
self.tab.get('https://friskybusiness.sg/')
|
||||
# input('wait:')
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
one_menu_eles = self.tab.eles('xpath=/html/body/header/height-observer/x-header/nav[2]/ul/li')
|
||||
for one_index, one_menu_ele in enumerate(one_menu_eles[:-1]):
|
||||
|
||||
one_link_ele = one_menu_ele.ele('@tag()=summary')
|
||||
one_url = one_link_ele.attr('data-follow-link')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
if one_index == 0 or one_index == 6:
|
||||
two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/div/a')
|
||||
else:
|
||||
two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/ul/li', timeout=1)
|
||||
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
||||
if one_index == 0 or one_index == 6:
|
||||
two_link_ele = two_menu_ele
|
||||
else:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
try:
|
||||
three_menu_eles = two_menu_ele.eles(f'xpath=div/ul/li')
|
||||
except:
|
||||
continue
|
||||
for three_menu_ele in three_menu_eles:
|
||||
three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
three_url = three_link_ele.attr('href')
|
||||
three_name = three_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ')
|
||||
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
page = 1
|
||||
self.tab.get(f"{category_url}?page={page}")
|
||||
goods_urls = []
|
||||
|
||||
#####获取页数####
|
||||
|
||||
#################
|
||||
|
||||
goods_urls = []
|
||||
while True:
|
||||
self.tab.scroll.to_bottom()
|
||||
try:
|
||||
goods_eles = self.tab.eles('#:snize-product-', timeout=5)
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
||||
goods_urls.append(goods_url)
|
||||
if len(goods_eles) == 0:
|
||||
break
|
||||
except:
|
||||
break
|
||||
page += 1
|
||||
self.tab.get(f"{category_url}?page={page}")
|
||||
|
||||
messages.sendInfo(f"{len(goods_urls)} 个")
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
image_eles = tab.eles('xpath=/html/body/main/section[1]/div/div/product-rerender/div/product-gallery/safe-sticky/product-gallery-navigation/button')
|
||||
|
||||
for image_ele in image_eles[-1:]:
|
||||
image_url = image_ele.ele('xpath=img').attr('src')
|
||||
image_urls.append(image_url)
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
h1_ele = tab.ele('@tag()=h1')
|
||||
brand = ''
|
||||
title = h1_ele.text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('.liquid').html)
|
||||
except:
|
||||
pass
|
||||
|
||||
price_list = tab.ele('.price-list price-list--product')
|
||||
price = price_list.ele('@tag()=sale-price').ele('.money').text.replace('$', '').replace(',', '')
|
||||
try:
|
||||
old_price = price_list.ele('@tag()=compare-at-price').ele('.money').text.replace('$', '').replace(',', '')
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,217 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "peek-cloppenburg"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 5,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.peek-cloppenburg.at/')
|
||||
one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
one_link_ele.click()
|
||||
self.tab.wait(3)
|
||||
|
||||
two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/a', timeout=1)
|
||||
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
||||
two_link_ele = two_menu_ele
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
try:
|
||||
three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
|
||||
except:
|
||||
continue
|
||||
for three_menu_ele in three_menu_eles:
|
||||
three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
three_url = three_link_ele.attr('href')
|
||||
three_name = three_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(f"{category_url}?page=1")
|
||||
goods_urls = []
|
||||
|
||||
max_page = 1
|
||||
#####获取页数####
|
||||
try:
|
||||
max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text)
|
||||
except:
|
||||
pass
|
||||
#################
|
||||
|
||||
goods_urls = []
|
||||
if max_page > 1:
|
||||
for page in range(1, max_page+1):
|
||||
if page > 1:
|
||||
self.tab.get(f"{category_url}?page={page}&userId=123456")
|
||||
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
||||
goods_urls.append(goods_url)
|
||||
messages.sendInfo(f"{page}/{max_page}")
|
||||
else:
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
||||
goods_urls.append(goods_url)
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a')
|
||||
|
||||
for image_ele in image_eles:
|
||||
image_url = image_ele.attr('href')
|
||||
image_urls.append(image_url)
|
||||
return image_urls
|
||||
|
||||
def get_size_eles():
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
return eles
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
h1_eles = tab.ele('@tag()=h1', timeout=10).eles('@tag()=span')
|
||||
brand = ''
|
||||
try:
|
||||
brand = h1_eles[0].text
|
||||
title = h1_eles[1].text
|
||||
except:
|
||||
title = tab.ele('@tag()=h1').text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}")
|
||||
except:
|
||||
pass
|
||||
|
||||
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '.')
|
||||
try:
|
||||
old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '.')
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
color_urls = []
|
||||
try:
|
||||
color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li')
|
||||
m_data.append('Farbe')
|
||||
for color_ele in color_eles:
|
||||
color_url = color_ele.ele('@tag()=a').attr('href')
|
||||
color_urls.append(color_url)
|
||||
except:
|
||||
pass
|
||||
m_data.append('Größe')
|
||||
|
||||
if len(color_urls) > 0:
|
||||
for color_url in color_urls:
|
||||
color_images = main_images
|
||||
if color_url != url:
|
||||
tab.get(color_url)
|
||||
color_images = get_image_urls()
|
||||
|
||||
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '')
|
||||
try:
|
||||
old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '')
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]')
|
||||
|
||||
size_eles = get_size_eles()
|
||||
for size_ele in size_eles:
|
||||
size_name = size_ele.ele('.text-cp-md truncate').text
|
||||
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name, size_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
goods_info.p_urls = color_urls
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,213 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "peek-cloppenburg-nl"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 5,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.peek-cloppenburg.nl/')
|
||||
one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
one_link_ele.click()
|
||||
self.tab.wait(3)
|
||||
|
||||
two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/a', timeout=1)
|
||||
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
||||
two_link_ele = two_menu_ele
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
try:
|
||||
three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
|
||||
except:
|
||||
continue
|
||||
for three_menu_ele in three_menu_eles:
|
||||
three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
three_url = three_link_ele.attr('href')
|
||||
three_name = three_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(f"{category_url}?page=1")
|
||||
goods_urls = []
|
||||
|
||||
max_page = 1
|
||||
#####获取页数####
|
||||
try:
|
||||
max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text)
|
||||
except:
|
||||
pass
|
||||
#################
|
||||
|
||||
goods_urls = []
|
||||
if max_page > 1:
|
||||
for page in range(1, max_page+1):
|
||||
if page > 1:
|
||||
self.tab.get(f"{category_url}?page={page}&userId=123456")
|
||||
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
||||
goods_urls.append(goods_url)
|
||||
messages.sendInfo(f"{page}/{max_page}")
|
||||
else:
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
||||
goods_urls.append(goods_url)
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a')
|
||||
|
||||
for image_ele in image_eles:
|
||||
image_url = image_ele.attr('href')
|
||||
image_urls.append(image_url)
|
||||
return image_urls
|
||||
|
||||
def get_size_eles():
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
try:
|
||||
eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li')
|
||||
except:
|
||||
pass
|
||||
return eles
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
h1_eles = tab.ele('@tag()=h1').eles('@tag()=span')
|
||||
brand = h1_eles[0].text
|
||||
title = h1_eles[1].text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}")
|
||||
except:
|
||||
pass
|
||||
|
||||
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.')
|
||||
try:
|
||||
old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.')
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
color_urls = []
|
||||
try:
|
||||
color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li')
|
||||
m_data.append('Kleur')
|
||||
for color_ele in color_eles:
|
||||
color_url = color_ele.ele('@tag()=a').attr('href')
|
||||
color_urls.append(color_url)
|
||||
except:
|
||||
pass
|
||||
m_data.append('Maat')
|
||||
|
||||
if len(color_urls) > 0:
|
||||
for color_url in color_urls:
|
||||
color_images = main_images
|
||||
if color_url != url:
|
||||
tab.get(color_url)
|
||||
color_images = get_image_urls()
|
||||
|
||||
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.')
|
||||
try:
|
||||
old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.')
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]')
|
||||
|
||||
size_eles = get_size_eles()
|
||||
for size_ele in size_eles:
|
||||
size_name = size_ele.ele('.text-cp-md truncate').text
|
||||
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name, size_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
goods_info.p_urls = color_urls
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,371 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages, files
|
||||
import requests
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "ssfshop"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 300,
|
||||
worker_num = 24,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.ssfshop.com/')
|
||||
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
if one_name == '라이프':
|
||||
continue
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
page = 1
|
||||
self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
|
||||
goods_urls = []
|
||||
|
||||
url_index = {}
|
||||
while True:
|
||||
self.tab.scroll.to_bottom()
|
||||
page_urls = []
|
||||
goods_eles = self.tab.eles('.god-item')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('xpath=a')
|
||||
if goods_link_ele.attr('href') in url_index:
|
||||
continue
|
||||
page_urls.append(goods_link_ele.attr('href'))
|
||||
url_index[goods_link_ele.attr('href')] = None
|
||||
|
||||
if len(page_urls) == 0:
|
||||
break
|
||||
if len(goods_urls) > 5000:
|
||||
break
|
||||
goods_urls += page_urls
|
||||
page += 1
|
||||
self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
|
||||
self.tab.wait(1)
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url: str) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
for _ in range(10):
|
||||
image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div')
|
||||
if len(image_eles) > 0:
|
||||
break
|
||||
tab.wait(3)
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele
|
||||
image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data'))
|
||||
|
||||
return image_urls
|
||||
|
||||
def get_options(godNo, optValCd1, optValCd2 = ''):
|
||||
headers = {
|
||||
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Connection': 'keep-alive',
|
||||
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
||||
'Origin': 'https://www.ssfshop.com',
|
||||
'Referer': url,
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
|
||||
'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
# 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950',
|
||||
}
|
||||
|
||||
data = {
|
||||
'optValCd1': optValCd1,
|
||||
'optValCd2': optValCd2,
|
||||
'optValCd3': '',
|
||||
'godNo': godNo,
|
||||
}
|
||||
|
||||
response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'})
|
||||
json_data = response.json()
|
||||
|
||||
options = {}
|
||||
if 'opt' not in json_data:
|
||||
return options
|
||||
|
||||
for option in json_data['opt']:
|
||||
for i in range(1, 6):
|
||||
key = f"optValNm{i}"
|
||||
if option[key]:
|
||||
options[option[key]] = {}
|
||||
break
|
||||
return options
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = url.split('/')[3]
|
||||
godNo = url.split('/')[4]
|
||||
|
||||
title = tab.ele('#goodDtlTitle').text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = tab.ele('#godsTabView').html
|
||||
except:
|
||||
pass
|
||||
|
||||
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
|
||||
try:
|
||||
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
color_urls = []
|
||||
try:
|
||||
tab.ele('.gods-option').ele('text=색상').click()
|
||||
color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label')
|
||||
for color_ele in color_eles:
|
||||
color_url = color_ele.attr('onclick')
|
||||
if color_url:
|
||||
color_url = color_url.split("'")[1]
|
||||
else:
|
||||
color_url = url
|
||||
if 'https://' not in color_url:
|
||||
color_url = 'https://www.ssfshop.com'+color_url
|
||||
color_urls.append(color_url)
|
||||
except:
|
||||
pass
|
||||
|
||||
if len(color_urls) > 1:
|
||||
m_data = ["색상"]
|
||||
|
||||
try:
|
||||
option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv')
|
||||
except:
|
||||
option_eles = []
|
||||
for option_ele in option_eles:
|
||||
m_data.append(option_ele.ele('@tag()=input').attr('opt-nm'))
|
||||
m_data = m_data[:3]
|
||||
|
||||
option_data = {}
|
||||
if len(option_eles) > 0:
|
||||
try:
|
||||
option_eles[0].click()
|
||||
except:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
tab.wait(0.5)
|
||||
option1_eles = option_eles[0].eles('@tag()=li')
|
||||
for option1_ele in option1_eles:
|
||||
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
|
||||
for option2_name, items in option_data[option1_ele.text].items():
|
||||
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
|
||||
|
||||
if len(color_urls) > 1:
|
||||
for color_url in color_urls:
|
||||
color_images = main_images
|
||||
if color_url != url:
|
||||
tab.get(color_url)
|
||||
title = tab.ele('#goodDtlTitle').text
|
||||
color_images = get_image_urls()
|
||||
|
||||
godNo = color_url.split('/')[4]
|
||||
try:
|
||||
option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied')
|
||||
except:
|
||||
option_eles = []
|
||||
option_data = {}
|
||||
if len(option_eles) > 0:
|
||||
option1_eles = option_eles[0].eles('xpath=div/ul/li')
|
||||
for option1_ele in option1_eles:
|
||||
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
|
||||
for option2_name, items in option_data[option1_ele.text].items():
|
||||
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
|
||||
|
||||
try:
|
||||
color_name = title.split('-')[1]
|
||||
except:
|
||||
color_name = tab.ele('.buy-txts').text
|
||||
|
||||
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
|
||||
try:
|
||||
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
if len(option_data) > 0:
|
||||
for option1_name, items1 in option_data.items():
|
||||
if len(items1) > 0:
|
||||
for option2_name, items2 in items1.items():
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name, option1_name, option2_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name, option1_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
for option1_name, items1 in option_data.items():
|
||||
if len(items1) > 0:
|
||||
for option2_name, items2 in items1.items():
|
||||
if len(items2) > 0:
|
||||
for option3_name, items3 in items2.items():
|
||||
attr_items.append({
|
||||
'url': url,
|
||||
'items': [option1_name, option2_name, option3_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': []
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': url,
|
||||
'items': [option1_name, option2_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': []
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': url,
|
||||
'items': [option1_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': []
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
try:
|
||||
tab.ele('.gods-option').ele('text=사이즈').click()
|
||||
size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li')
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = ['사이즈']
|
||||
goods_info.images = main_images
|
||||
for size_ele in size_eles:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
attr_items=[size_ele.text],
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
images=[],
|
||||
))
|
||||
except:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
if len(attr_item['items']) != len(m_data):
|
||||
attr_item['items'] += m_data[len(attr_item['items']):]
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
goods_info.p_urls = color_urls
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
|
||||
def get_home(self):
|
||||
self.tab.get('https://www.ssfshop.com/main')
|
||||
li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li')
|
||||
categorys = []
|
||||
for li_ele in li_eles:
|
||||
a_ele = li_ele.ele('@tag()=a')
|
||||
categorys.append(f"{a_ele.attr('href')}#{a_ele.text}")
|
||||
files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys)
|
||||
@@ -1,131 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "taketoys"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 5,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
self.tab.get('https://taketoys.sg/')
|
||||
# input('wait:')
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
one_menu_eles = self.tab.eles('xpath=/html/body/header/div[2]/div/nav/ul/li')
|
||||
for one_menu_ele in one_menu_eles[1:-1]:
|
||||
one_link_ele = one_menu_ele.ele('xpath=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=ul/li', timeout=1)
|
||||
for two_index, two_menu_ele in enumerate(two_menu_eles):
|
||||
two_link_ele = two_menu_ele.ele('xpath=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
# try:
|
||||
# three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
|
||||
# except:
|
||||
# continue
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(f"{category_url}?page=1")
|
||||
goods_urls = []
|
||||
|
||||
#####获取页数####
|
||||
|
||||
#################
|
||||
|
||||
goods_urls = []
|
||||
while True:
|
||||
self.tab.scroll.to_bottom()
|
||||
try:
|
||||
self.tab.ele('text=Show more', timeout=5).click()
|
||||
self.tab.wait(1)
|
||||
except:
|
||||
break
|
||||
|
||||
goods_eles = self.tab.eles('.col-xs-6 col-sm-4 col-lg-3 item-wrapper')
|
||||
for goods_ele in goods_eles:
|
||||
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
|
||||
goods_urls.append(goods_url)
|
||||
messages.sendInfo(f"{len(goods_urls)} 个")
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
image_eles = tab.eles('xpath=/html/body/main/div[2]/div[2]/div[2]/div[2]/div/div/div[3]/div[2]/div/div')
|
||||
|
||||
for image_ele in image_eles:
|
||||
image_url = image_ele.ele('xpath=img').attr('src')
|
||||
image_urls.append(image_url)
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
h1_ele = tab.ele('@tag()=h1')
|
||||
brand = ''
|
||||
try:
|
||||
brand = h1_ele.ele('@tag()=strong').text
|
||||
except:
|
||||
pass
|
||||
title = h1_ele.ele('@tag()=span').text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('#product-info-tab').html)
|
||||
except:
|
||||
pass
|
||||
|
||||
price = tab.ele('.price').attr('title').replace('SGD', '').replace(',', '')
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
@@ -1,140 +0,0 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "t-fashion"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 300,
|
||||
worker_num = 24,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.t-fashion.jp/')
|
||||
one_menu_eles = self.tab.ele('.m-aside-menu-body').eles('.js-collest-li')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
# one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/ul/li')
|
||||
for two_menu_ele in two_menu_eles[1:]:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
page = 1
|
||||
self.tab.get(f"{category_url}?page={page}")
|
||||
goods_urls = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
goods_eles = self.tab.eles('xpath=/html/body/div[1]/div[3]/div/section[1]/div/div[1]/div[1]/div[3]/ol/li')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('xpath=a')
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
if len(goods_eles) == 0:
|
||||
break
|
||||
except:
|
||||
break
|
||||
page += 1
|
||||
self.tab.get(f"{category_url}?page={page}")
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
image_eles = tab.eles('xpath=//*[@id="item-gallery-parent"]/li')
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele.ele('xpath=div/img')
|
||||
image_urls.append(image_link_ele.attr('src'))
|
||||
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
try:
|
||||
brand = tab.ele('xpath=//*[@id="current-tgoods-section"]/div/div[2]/div/div[1]/a[2]', timeout=10).text
|
||||
except:
|
||||
pass
|
||||
|
||||
title = tab.ele('@tag()=h1').text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('xpath=//*[@id="slide-down-contents-parent"]'))
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
price = float(tab.ele('.is-fz28-36f is-fw-b ').text.replace(',', '').replace('¥', '').replace('（税込）', ''))
|
||||
except:
|
||||
price = float(tab.ele('.is-fz28-36f is-fw-b is-color-notice').text.replace(',', '').replace('¥', '').replace('（税込）', ''))
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = ['Color', 'Size']
|
||||
goods_info.images = main_images
|
||||
color_eles = tab.eles('xpath=//*[@id="sku-list-box"]/div')
|
||||
for color_ele in color_eles:
|
||||
color_image = color_ele.ele('xpath=div/a').attr('href')
|
||||
color_name = color_ele.ele('xpath=div/a/span').text
|
||||
size_eles = color_ele.eles('xpath=ol/li')
|
||||
for size_ele in size_eles:
|
||||
size_name = size_ele.ele('xpath=div[1]/text()')
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
attr_items=[color_name, size_name],
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
images=[color_image]
|
||||
))
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
Reference in New Issue
Block a user