初始化

This commit is contained in:
2026-04-10 10:57:44 +08:00
parent ec6e7573de
commit a19a468298
70 changed files with 5676 additions and 0 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

176
lib/module/agatameble.py Normal file
View File

@@ -0,0 +1,176 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "agatameble"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.agatameble.pl/')
one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('@tag()=a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
three_menu_eles = two_menu_ele.eles('xpath=ul/li')
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('@tag()=a')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(f"{category_url}")
goods_urls = []
while True:
try:
self.tab.scroll.to_bottom()
self.tab.ele('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/div/div/a').click()
except:
break
goods_eles = self.tab.eles('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/section/div/div/div')
for goods_ele in goods_eles:
goods_link_ele = goods_ele.ele('xpath=a')
goods_urls.append(goods_link_ele.attr('href'))
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
tab.ele('xpath=//*[@id="contentContainer"]/form/div[1]/section/div/div[2]/button[1]').click()
image_eles = tab.eles('xpath=//*[@id="root"]/div/section/div[2]/div/aside/div[2]/div/div/div/div[1]/div/div/div/div')
for image_ele in image_eles:
image_link_ele = image_ele.ele('xpath=div/img[2]')
image_urls.append(image_link_ele.attr('src').replace('width=', '').replace('height=', ''))
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
brand = ''
try:
brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
except:
pass
if brand == '':
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
else:
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text
desc = ""
desc_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[1]/div/div')[:3]
for desc_ele in desc_eles:
desc += formats.clean_html(desc_ele.html)
price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}")
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
color_urls = [url]
color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li')
for color_ele in color_eles[1:]:
color_link_ele = color_ele.ele('xpath=a')
color_urls.append(color_link_ele.attr('href'))
if len(color_urls) > 1:
m_data = ["Color"]
if len(color_urls) > 1:
for color_url in color_urls:
color_images = main_images
if color_url != url:
tab.get(color_url)
if brand == '':
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
else:
title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text
color_images = get_image_urls()
price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}")
old_price = price
try:
item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text
except:
continue
attr_items.append({
'url': color_url,
'items': [item_name],
'price': price,
'old_price': old_price,
'images': color_images
})
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
url=attr_item['url'],
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images'],
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
goods_info.p_urls = color_urls
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

View File

@@ -0,0 +1,325 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "ancientnutrition"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://ancientnutrition.com/')
one_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[4]/div[1]/div[1]/div')
for index, one_menu_ele in enumerate(one_menu_eles[1:]):
one_link_ele = one_menu_ele.ele('xpath=div/a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
print(index+2)
two_menu_eles = self.tab.eles(f'xpath=//*[@id="__next"]/div[4]/div[{index+2}]/div/div/div[2]/div/a')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
# for three_menu_ele in three_menu_eles:
# three_link_ele = three_menu_ele.ele('@tag()=a')
# three_url = three_link_ele.attr('href')
# three_name = three_link_ele.text.replace('/', '-')
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(f"{category_url}")
goods_urls = []
while True:
try:
self.tab.scroll.to_bottom()
self.tab.ele('text=Load More').click()
except:
break
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[9]/div/div[2]/div[2]/div')
for goods_ele in goods_eles:
try:
goods_link_ele = goods_ele.ele('@tag()=a')
except Exception as e:
continue
goods_urls.append(goods_link_ele.attr('href'))
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[1]/div/div[2]/div/div[1]/div')
for image_ele in image_eles:
image_link_ele = image_ele.ele('xpath=div/span/img')
image_urls.append(image_link_ele.attr('src').replace('_150x150', '_1450x1450'))
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
brand = ''
# try:
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
# except:
# pass
# if brand == '':
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
# else:
title = tab.ele('@tag()=h1').text
desc = formats.clean_html(tab.ele('#productDescription').html)
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
try:
tab.ele('text=Select Flavor').click()
m_data.append('Flavor')
except:
pass
try:
tab.ele('text=Select Size').click()
m_data.append('Size')
except:
pass
try:
tab.ele('text=Select Quantity').click()
m_data.append('Quantity')
except:
pass
if 'Flavor' in m_data:
tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click()
color_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div[2]/div')
color_xpaths = []
for color_ele in color_eles:
color_xpaths.append(color_ele.xpath)
for color_xpath in color_xpaths:
while True:
tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click()
try:
color_ele = tab.ele(f"xpath={color_xpath}")
color_name = color_ele.text
break
except:
tab.get(url)
color_ele.click()
tab.wait(3)
color_images = get_image_urls()
if 'Size' in m_data:
try:
size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}")
except:
size_name = 'Default'
if 'Quantity' in m_data:
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
for quantity_index, quantity_ele in enumerate(quantity_eles):
try:
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
quantity_name = quantity_ele.text
except:
continue
quantity_ele.click()
tab.wait(2)
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [color_name, size_name, quantity_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [color_name, size_name],
'price': price,
'old_price': old_price,
'images': color_images
})
size_xpaths = []
for size_ele in size_eles:
try:
size_xpaths.append(size_ele.xpath)
except:
continue
for size_xpath in size_xpaths:
try:
size_ele = tab.ele(f"xpath={size_xpath}")
size_name = size_ele.text
except:
continue
size_ele.click()
tab.wait(3)
color_images = get_image_urls()
if 'Quantity' in m_data:
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
for quantity_index, quantity_ele in enumerate(quantity_eles):
try:
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
quantity_name = quantity_ele.text
except:
continue
quantity_ele.click()
tab.wait(2)
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [color_name, size_name, quantity_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [color_name, size_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [color_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
color_images = []
if 'Size' in m_data:
size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}")
size_xpaths = []
for size_ele in size_eles:
try:
size_xpaths.append(size_ele.xpath)
except:
continue
for size_xpath in size_xpaths:
try:
size_ele = tab.ele(f"xpath={size_xpath}")
size_name = size_ele.text
except:
continue
size_ele.click()
tab.wait(2)
color_images = get_image_urls()
if 'Quantity' in m_data:
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
for quantity_index, quantity_ele in enumerate(quantity_eles):
try:
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
quantity_name = quantity_ele.text
except:
continue
quantity_ele.click()
tab.wait(2)
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [size_name, quantity_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
if 'Quantity' in m_data:
quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}")
for quantity_index, quantity_ele in enumerate(quantity_eles):
try:
quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}")
quantity_name = quantity_ele.text
except:
continue
quantity_ele.click()
tab.wait(2)
price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', ''))
old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', ''))
attr_items.append({
'url': tab.url,
'items': [quantity_name],
'price': price,
'old_price': old_price,
'images': color_images
})
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
url=attr_item['url'],
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images'],
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

201
lib/module/bricomarche.py Normal file
View File

@@ -0,0 +1,201 @@
from core import spiders, types
from utils import formats, messages
from DrissionPage.common import Keys
class SpiderModule(spiders.Spiders):
project_name = "bricomarche"
def __init__(self):
super().__init__(
self.project_name,
bitch = 5,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
self.tab.get('https://www.bricomarche.pl/')
# input('wait:')
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.bricomarche.pl/')
one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('@tag()=a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
three_menu_eles = two_menu_ele.eles('xpath=ul/li')
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('@tag()=a')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
page = 1
limit = 100
self.tab.get(f"{category_url}?page={page}&limit={limit}")
if self.tab.ele('@tag()=h2').text == '执行安全验证':
self.tab.wait(10)
self.tab.actions.type((Keys.TAB, Keys.SPACE))
goods_urls = []
while True:
try:
self.tab.scroll.to_bottom()
goods_eles = self.tab.eles('xpath=//*[@id="spark"]/div[2]/div/section/div/div/main/div[2]/section/div')
for goods_ele in goods_eles:
goods_link_ele = goods_ele.ele('@tag()=a')
goods_urls.append(goods_link_ele.attr('href'))
if len(goods_eles) == 0:
break
except:
break
page += 1
self.tab.get(f"{category_url}?page={page}&limit={limit}")
if self.tab.ele('@tag()=h2').text == '执行安全验证':
self.tab.wait(10)
self.tab.actions.type((Keys.TAB, Keys.SPACE))
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.eles('xpath=//*[@id="spark"]/div[2]/div/main/div/div[2]/section[1]/div/div[1]/div/div/div/div/div')
for image_ele in image_eles:
image_link_ele = image_ele.ele('xpath=button/div/img')
image_urls.append(image_link_ele.attr('src').replace('gallery_100_91', 'gallery'))
return image_urls
tab = self.browser.new_tab(url)
while True:
try:
if tab.ele('@tag()=h2').text == '执行安全验证':
tab.wait(10)
tab.actions.type((Keys.TAB, Keys.SPACE))
tab.wait(5)
else:
break
except:
break
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
brand = ''
# try:
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
# except:
# pass
title = tab.ele('@tag()=h1', timeout=10).text
desc = ""
try:
desc = formats.clean_html(tab.ele('.attributes').html)
except:
pass
# try:
price_ele = tab.ele('.main-price')
price = float(f"{price_ele.ele('.whole').text}{price_ele.ele('.cents').text.replace('', '')}")
# except:
# price = 0.0
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
# color_urls = [url]
# color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li')
# for color_ele in color_eles[1:]:
# color_link_ele = color_ele.ele('xpath=a')
# color_urls.append(color_link_ele.attr('href'))
# if len(color_urls) > 1:
# m_data = ["Color"]
# if len(color_urls) > 1:
# for color_url in color_urls:
# color_images = main_images
# if color_url != url:
# tab.get(color_url)
# if brand == '':
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
# else:
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text
# color_images = get_image_urls()
# price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}")
# old_price = price
# try:
# item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text
# except:
# continue
# attr_items.append({
# 'url': color_url,
# 'items': [item_name],
# 'price': price,
# 'old_price': old_price,
# 'images': color_images
# })
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
url=attr_item['url'],
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images'],
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
# goods_info.p_urls = color_urls
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

136
lib/module/condom69.py Normal file
View File

@@ -0,0 +1,136 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "condom69"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 5,
switch_clash = False,
reflush_browser = False
)
self.tab.get('https://www.condom69.net/')
# input('wait:')
def get_category_urls(self) -> list[str]:
category_urls = []
one_menu_eles = self.tab.eles('xpath=//*[@id="nav-one"]/li')
for one_menu_ele in one_menu_eles[:-1]:
one_link_ele = one_menu_ele.ele('xpath=a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=div/ul/li', timeout=1)
for two_index, two_menu_ele in enumerate(two_menu_eles):
two_link_ele = two_menu_ele.ele('xpath=div/a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-').replace(',', '').replace('#', ' ')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
# try:
# three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
# except:
# continue
# for three_menu_ele in three_menu_eles:
# three_link_ele = three_menu_ele.ele('@tag()=a')
# three_url = three_link_ele.attr('href')
# three_name = three_link_ele.text.replace('/', '-')
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
page = 1
self.tab.get(f"{category_url}&pg={page}")
#####获取页数####
#################
goods_urls = []
url_index = {}
while True:
self.tab.scroll.to_bottom()
goods_eles = self.tab.eles('.divProdItemImg2Inner')
page_urls = []
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('xpath=a').attr('href')}"
if goods_url not in url_index:
page_urls.append(goods_url)
url_index[goods_url] = None
if len(page_urls) > 0:
goods_urls += page_urls
else:
break
page += 1
self.tab.get(f"{category_url}&pg={page}")
messages.sendInfo(f"{len(goods_urls)}")
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.eles('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[1]/div[2]/div')
for image_ele in image_eles:
image_url = image_ele.ele('xpath=div/div/a/img').attr('src')
image_urls.append(image_url)
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
h1_ele = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[1]')
brand = ''
try:
brand = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/table/tbody/tr/td[3]/a').text
except:
pass
title = h1_ele.text
desc = ""
try:
desc = formats.clean_html(tab.ele('.ctl00_cphContent_ucUsrIndProduct_pnlIndProdDesc').html)
except:
pass
price = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[3]/div[1]/div[2]/span').text.replace(',', '')
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

210
lib/module/dk_one.py Normal file
View File

@@ -0,0 +1,210 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "dk_one"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.dk_one.com/')
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('@tag()=a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
if one_name == '라이프':
continue
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
if two_name == '메인':
continue
category_urls.append(f"{two_url}#{one_name}/{two_name}")
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
if two_name == '메인':
continue
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
# for three_menu_ele in three_menu_eles:
# three_link_ele = three_menu_ele.ele('@tag()=a')
# three_url = three_link_ele.attr('href')
# three_name = three_link_ele.text.replace('/', '-')
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(category_url)
goods_urls = []
goods_eles = self.tab.eles('xpath=//*[@id="prodList"]/div')
for goods_ele in goods_eles:
goods_link_ele = goods_ele.ele('.prod-link')
goods_urls.append(goods_link_ele.attr('href'))
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls(tab):
image_urls = []
image_eles = tab.ele('#prod-img').eles('@tag()=img')
for image_ele in image_eles:
image_urls.append(image_ele.attr('src'))
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
brand = ''
# try:
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
# except:
# pass
# if brand == '':
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
# else:
title = tab.ele('.page-title').text
try:
desc = formats.clean_html(tab.ele('@@class=tab-content@@id=tabContent0202').html)
except:
desc = ''
price = float(tab.ele('.price-group').ele('.val').text.replace(',', ''))
try:
old_price = float(tab.ele('.price-group origin').text.replace(',', ''))
except:
old_price = price
main_images = get_image_urls(tab)
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
images=main_images
)
m_data = []
color_data = {}
color_eles = []
# 颜色
tab.ele('.opt-color')
m_data.append('컬러')
color_eles = tab.ele('.color-list-wrap').eles('.prod-color-rdo')
m_data.append('사이즈')
for color_ele in color_eles:
color_input_onclick = color_ele.ele('@tag()=input').attr('onclick')
have_url = False
if 'location.href=' in color_input_onclick:
color_url = 'https://dk-on.com' + formats.re_search(color_input_onclick, r"location\.href\s*=\s*'([^']*)'")
color_name = color_url.split('/')[-1]
have_url = True
elif 'fnColorClicked' in color_input_onclick:
color_url = goods_info.url
color_name = color_ele.ele('@tag()=input').attr('value')
color_ele.click()
else:
color_url = goods_info.url
color_name = goods_info.title.split(' / ')[-1]
color_data[color_name] = {'url': color_url, 'have_url': have_url, 'ele': color_ele}
goods_info.p_urls.append(color_url)
real_color_data = {}
for color_name, color_item in color_data.items():
color_tab = tab
if color_item['have_url']:
color_tab = self.browser.new_tab()
size_data = []
if color_item['have_url']:
color_tab.get(color_item['url'])
else:
color_item['ele'].click()
color_tab.wait(1)
color_item['price'] = str(color_tab.ele('.price-group').ele('.val').text.replace(',', ''))
try:
color_item['old_price'] = str(color_tab.ele('.price-group origin').text.replace(',', ''))
except:
color_item['old_price'] = color_item['price']
# 颜色对应商品图
color_item['images'] = get_image_urls(tab = color_tab)
# 颜色对应尺码
radio_eles = color_tab.ele('.prod-size-list').eles('.prod-size')
for radio_ele in radio_eles:
value = radio_ele.ele('@tag()=label').text
if value == None:
continue
size_data.append(value)
color_item['size'] = size_data
real_color_data[color_tab.ele('.page-title').text.split(' / ')[-1]] = color_item
color_data = real_color_data
goods_info.attr = 'M'
goods_info.title = goods_info.title.split(' / ')[0]
goods_info.attr_items = m_data
for color_name, color_item in color_data.items():
for color_image in color_item['images']:
if color_image in goods_info.images:
continue
goods_info.images.append(color_image)
# if color_data:
item_data = []
for color_name, color_item in color_data.items():
for size in color_item['size']:
item_data.append([color_name, size])
for attribute_items in item_data:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
attr_items=attribute_items,
price=color_data[attribute_items[0]]['price'],
old_price=color_data[attribute_items[0]]['price'],
images=color_data[attribute_items[0]]['images'],
))
except Exception as e:
try:
tab.close()
color_tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
color_tab.close()
except:
pass
return goods_info

86
lib/module/example.py Normal file
View File

@@ -0,0 +1,86 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "example"
def __init__(self):
super().__init__(
self.project_name,
bitch = 10,
worker_num = 1,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
pass
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(f"{category_url}")
goods_urls = []
pass
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
pass
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
desc = "简介"
brand = "品牌"
title = "标题"
price = 0.00
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images']
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
try:
tab.close()
except:
pass
return goods_info

View File

@@ -0,0 +1,141 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "friskybusiness"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 5,
switch_clash = False,
reflush_browser = False
)
self.tab.get('https://friskybusiness.sg/')
# input('wait:')
def get_category_urls(self) -> list[str]:
category_urls = []
one_menu_eles = self.tab.eles('xpath=/html/body/header/height-observer/x-header/nav[2]/ul/li')
for one_index, one_menu_ele in enumerate(one_menu_eles[:-1]):
one_link_ele = one_menu_ele.ele('@tag()=summary')
one_url = one_link_ele.attr('data-follow-link')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
if one_index == 0 or one_index == 6:
two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/div/a')
else:
two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/ul/li', timeout=1)
for two_index, two_menu_ele in enumerate(two_menu_eles):
if one_index == 0 or one_index == 6:
two_link_ele = two_menu_ele
else:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-').replace(',', '').replace('#', ' ')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
try:
three_menu_eles = two_menu_ele.eles(f'xpath=div/ul/li')
except:
continue
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('@tag()=a')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-').replace(',', '').replace('#', ' ')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
page = 1
self.tab.get(f"{category_url}?page={page}")
goods_urls = []
#####获取页数####
#################
goods_urls = []
while True:
self.tab.scroll.to_bottom()
try:
goods_eles = self.tab.eles('#:snize-product-', timeout=5)
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
goods_urls.append(goods_url)
if len(goods_eles) == 0:
break
except:
break
page += 1
self.tab.get(f"{category_url}?page={page}")
messages.sendInfo(f"{len(goods_urls)}")
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.eles('xpath=/html/body/main/section[1]/div/div/product-rerender/div/product-gallery/safe-sticky/product-gallery-navigation/button')
for image_ele in image_eles[-1:]:
image_url = image_ele.ele('xpath=img').attr('src')
image_urls.append(image_url)
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
h1_ele = tab.ele('@tag()=h1')
brand = ''
title = h1_ele.text
desc = ""
try:
desc = formats.clean_html(tab.ele('.liquid').html)
except:
pass
price_list = tab.ele('.price-list price-list--product')
price = price_list.ele('@tag()=sale-price').ele('.money').text.replace('$', '').replace(',', '')
try:
old_price = price_list.ele('@tag()=compare-at-price').ele('.money').text.replace('$', '').replace(',', '')
except:
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

View File

@@ -0,0 +1,217 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "peek-cloppenburg"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 5,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.peek-cloppenburg.at/')
one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
one_link_ele.click()
self.tab.wait(3)
two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/a', timeout=1)
for two_index, two_menu_ele in enumerate(two_menu_eles):
two_link_ele = two_menu_ele
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
try:
three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
except:
continue
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('@tag()=a')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(f"{category_url}?page=1")
goods_urls = []
max_page = 1
#####获取页数####
try:
max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text)
except:
pass
#################
goods_urls = []
if max_page > 1:
for page in range(1, max_page+1):
if page > 1:
self.tab.get(f"{category_url}?page={page}&userId=123456")
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
goods_urls.append(goods_url)
messages.sendInfo(f"{page}/{max_page}")
else:
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
goods_urls.append(goods_url)
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a')
for image_ele in image_eles:
image_url = image_ele.attr('href')
image_urls.append(image_url)
return image_urls
def get_size_eles():
try:
eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li')
except:
pass
try:
eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li')
except:
pass
try:
eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li')
except:
pass
try:
eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li')
except:
pass
return eles
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
h1_eles = tab.ele('@tag()=h1', timeout=10).eles('@tag()=span')
brand = ''
try:
brand = h1_eles[0].text
title = h1_eles[1].text
except:
title = tab.ele('@tag()=h1').text
desc = ""
try:
desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}")
except:
pass
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '.')
try:
old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '.')
except:
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
color_urls = []
try:
color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li')
m_data.append('Farbe')
for color_ele in color_eles:
color_url = color_ele.ele('@tag()=a').attr('href')
color_urls.append(color_url)
except:
pass
m_data.append('Größe')
if len(color_urls) > 0:
for color_url in color_urls:
color_images = main_images
if color_url != url:
tab.get(color_url)
color_images = get_image_urls()
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '')
try:
old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '')
except:
old_price = price
color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]')
size_eles = get_size_eles()
for size_ele in size_eles:
size_name = size_ele.ele('.text-cp-md truncate').text
attr_items.append({
'url': color_url,
'items': [color_name, size_name],
'price': price,
'old_price': old_price,
'images': color_images
})
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
url=attr_item['url'],
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images'],
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
goods_info.p_urls = color_urls
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

View File

@@ -0,0 +1,213 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "peek-cloppenburg-nl"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 5,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.peek-cloppenburg.nl/')
one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
one_link_ele.click()
self.tab.wait(3)
two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/a', timeout=1)
for two_index, two_menu_ele in enumerate(two_menu_eles):
two_link_ele = two_menu_ele
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
try:
three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
except:
continue
for three_menu_ele in three_menu_eles:
three_link_ele = three_menu_ele.ele('@tag()=a')
three_url = three_link_ele.attr('href')
three_name = three_link_ele.text.replace('/', '-')
category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(f"{category_url}?page=1")
goods_urls = []
max_page = 1
#####获取页数####
try:
max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text)
except:
pass
#################
goods_urls = []
if max_page > 1:
for page in range(1, max_page+1):
if page > 1:
self.tab.get(f"{category_url}?page={page}&userId=123456")
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
goods_urls.append(goods_url)
messages.sendInfo(f"{page}/{max_page}")
else:
goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div')
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
goods_urls.append(goods_url)
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a')
for image_ele in image_eles:
image_url = image_ele.attr('href')
image_urls.append(image_url)
return image_urls
def get_size_eles():
try:
eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li')
except:
pass
try:
eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li')
except:
pass
try:
eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li')
except:
pass
try:
eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li')
except:
pass
return eles
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
h1_eles = tab.ele('@tag()=h1').eles('@tag()=span')
brand = h1_eles[0].text
title = h1_eles[1].text
desc = ""
try:
desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}")
except:
pass
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.')
try:
old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.')
except:
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
color_urls = []
try:
color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li')
m_data.append('Kleur')
for color_ele in color_eles:
color_url = color_ele.ele('@tag()=a').attr('href')
color_urls.append(color_url)
except:
pass
m_data.append('Maat')
if len(color_urls) > 0:
for color_url in color_urls:
color_images = main_images
if color_url != url:
tab.get(color_url)
color_images = get_image_urls()
price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.')
try:
old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.')
except:
old_price = price
color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]')
size_eles = get_size_eles()
for size_ele in size_eles:
size_name = size_ele.ele('.text-cp-md truncate').text
attr_items.append({
'url': color_url,
'items': [color_name, size_name],
'price': price,
'old_price': old_price,
'images': color_images
})
if len(attr_items) == 0:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
url=attr_item['url'],
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images'],
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
goods_info.p_urls = color_urls
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

371
lib/module/ssfshop.py Normal file
View File

@@ -0,0 +1,371 @@
from core import spiders, types
from utils import formats, messages, files
import requests
class SpiderModule(spiders.Spiders):
project_name = "ssfshop"
def __init__(self):
super().__init__(
self.project_name,
bitch = 300,
worker_num = 24,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.ssfshop.com/')
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('@tag()=a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
if one_name == '라이프':
continue
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
if two_name == '메인':
continue
category_urls.append(f"{two_url}#{one_name}/{two_name}")
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
for two_menu_ele in two_menu_eles:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
if two_name == '메인':
continue
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
# for three_menu_ele in three_menu_eles:
# three_link_ele = three_menu_ele.ele('@tag()=a')
# three_url = three_link_ele.attr('href')
# three_name = three_link_ele.text.replace('/', '-')
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
page = 1
self.tab.get(f"{category_url}&currentPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
goods_urls = []
url_index = {}
while True:
self.tab.scroll.to_bottom()
page_urls = []
goods_eles = self.tab.eles('.god-item')
for goods_ele in goods_eles:
goods_link_ele = goods_ele.ele('xpath=a')
if goods_link_ele.attr('href') in url_index:
continue
page_urls.append(goods_link_ele.attr('href'))
url_index[goods_link_ele.attr('href')] = None
if len(page_urls) == 0:
break
if len(goods_urls) > 5000:
break
goods_urls += page_urls
page += 1
self.tab.get(f"{category_url}&currentPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
self.tab.wait(1)
return goods_urls
def get_goods_info(self, url: str) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
for _ in range(10):
image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div')
if len(image_eles) > 0:
break
tab.wait(3)
for image_ele in image_eles:
image_link_ele = image_ele
image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data'))
return image_urls
def get_options(godNo, optValCd1, optValCd2 = ''):
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Origin': 'https://www.ssfshop.com',
'Referer': url,
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Sec-Fetch-Site': 'same-origin',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0',
'X-Requested-With': 'XMLHttpRequest',
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
# 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950',
}
data = {
'optValCd1': optValCd1,
'optValCd2': optValCd2,
'optValCd3': '',
'godNo': godNo,
}
response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'})
json_data = response.json()
options = {}
if 'opt' not in json_data:
return options
for option in json_data['opt']:
for i in range(1, 6):
key = f"optValNm{i}"
if option[key]:
options[option[key]] = {}
break
return options
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
brand = url.split('/')[3]
godNo = url.split('/')[4]
title = tab.ele('#goodDtlTitle').text
desc = ""
try:
desc = tab.ele('#godsTabView').html
except:
pass
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
try:
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
except:
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
color_urls = []
try:
tab.ele('.gods-option').ele('text=색상').click()
color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label')
for color_ele in color_eles:
color_url = color_ele.attr('onclick')
if color_url:
color_url = color_url.split("'")[1]
else:
color_url = url
if 'https://' not in color_url:
color_url = 'https://www.ssfshop.com'+color_url
color_urls.append(color_url)
except:
pass
if len(color_urls) > 1:
m_data = ["색상"]
try:
option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv')
except:
option_eles = []
for option_ele in option_eles:
m_data.append(option_ele.ele('@tag()=input').attr('opt-nm'))
m_data = m_data[:3]
option_data = {}
if len(option_eles) > 0:
try:
option_eles[0].click()
except:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
try:
tab.close()
except:
pass
return goods_info
tab.wait(0.5)
option1_eles = option_eles[0].eles('@tag()=li')
for option1_ele in option1_eles:
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
for option2_name, items in option_data[option1_ele.text].items():
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
if len(color_urls) > 1:
for color_url in color_urls:
color_images = main_images
if color_url != url:
tab.get(color_url)
title = tab.ele('#goodDtlTitle').text
color_images = get_image_urls()
godNo = color_url.split('/')[4]
try:
option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied')
except:
option_eles = []
option_data = {}
if len(option_eles) > 0:
option1_eles = option_eles[0].eles('xpath=div/ul/li')
for option1_ele in option1_eles:
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
for option2_name, items in option_data[option1_ele.text].items():
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
try:
color_name = title.split('-')[1]
except:
color_name = tab.ele('.buy-txts').text
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
try:
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
except:
old_price = price
if len(option_data) > 0:
for option1_name, items1 in option_data.items():
if len(items1) > 0:
for option2_name, items2 in items1.items():
attr_items.append({
'url': color_url,
'items': [color_name, option1_name, option2_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
attr_items.append({
'url': color_url,
'items': [color_name, option1_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
attr_items.append({
'url': color_url,
'items': [color_name],
'price': price,
'old_price': old_price,
'images': color_images
})
else:
for option1_name, items1 in option_data.items():
if len(items1) > 0:
for option2_name, items2 in items1.items():
if len(items2) > 0:
for option3_name, items3 in items2.items():
attr_items.append({
'url': url,
'items': [option1_name, option2_name, option3_name],
'price': price,
'old_price': old_price,
'images': []
})
else:
attr_items.append({
'url': url,
'items': [option1_name, option2_name],
'price': price,
'old_price': old_price,
'images': []
})
else:
attr_items.append({
'url': url,
'items': [option1_name],
'price': price,
'old_price': old_price,
'images': []
})
if len(attr_items) == 0:
try:
tab.ele('.gods-option').ele('text=사이즈').click()
size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li')
goods_info.attr = 'M'
goods_info.attr_items = ['사이즈']
goods_info.images = main_images
for size_ele in size_eles:
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
attr_items=[size_ele.text],
price=price,
old_price=old_price,
images=[],
))
except:
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
else:
goods_info.attr = 'M'
goods_info.attr_items = m_data
for attr_item in attr_items:
if len(attr_item['items']) != len(m_data):
attr_item['items'] += m_data[len(attr_item['items']):]
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
url=attr_item['url'],
attr_items=attr_item['items'],
price=attr_item['price'],
old_price=attr_item['old_price'],
images=attr_item['images'],
))
for image_url in attr_item['images']:
if image_url not in main_images:
main_images.append(image_url)
goods_info.images = main_images
goods_info.p_urls = color_urls
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info
def get_home(self):
self.tab.get('https://www.ssfshop.com/main')
li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li')
categorys = []
for li_ele in li_eles:
a_ele = li_ele.ele('@tag()=a')
categorys.append(f"{a_ele.attr('href')}#{a_ele.text}")
files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys)

131
lib/module/taketoys.py Normal file
View File

@@ -0,0 +1,131 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "taketoys"
def __init__(self):
super().__init__(
self.project_name,
bitch = 30,
worker_num = 5,
switch_clash = False,
reflush_browser = False
)
self.tab.get('https://taketoys.sg/')
# input('wait:')
def get_category_urls(self) -> list[str]:
category_urls = []
one_menu_eles = self.tab.eles('xpath=/html/body/header/div[2]/div/nav/ul/li')
for one_menu_ele in one_menu_eles[1:-1]:
one_link_ele = one_menu_ele.ele('xpath=a')
one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=ul/li', timeout=1)
for two_index, two_menu_ele in enumerate(two_menu_eles):
two_link_ele = two_menu_ele.ele('xpath=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-').replace(',', '').replace('#', ' ')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
# try:
# three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li')
# except:
# continue
# for three_menu_ele in three_menu_eles:
# three_link_ele = three_menu_ele.ele('@tag()=a')
# three_url = three_link_ele.attr('href')
# three_name = three_link_ele.text.replace('/', '-')
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
self.tab.get(f"{category_url}?page=1")
goods_urls = []
#####获取页数####
#################
goods_urls = []
while True:
self.tab.scroll.to_bottom()
try:
self.tab.ele('text=Show more', timeout=5).click()
self.tab.wait(1)
except:
break
goods_eles = self.tab.eles('.col-xs-6 col-sm-4 col-lg-3 item-wrapper')
for goods_ele in goods_eles:
goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}"
goods_urls.append(goods_url)
messages.sendInfo(f"{len(goods_urls)}")
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.eles('xpath=/html/body/main/div[2]/div[2]/div[2]/div[2]/div/div/div[3]/div[2]/div/div')
for image_ele in image_eles:
image_url = image_ele.ele('xpath=img').attr('src')
image_urls.append(image_url)
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
h1_ele = tab.ele('@tag()=h1')
brand = ''
try:
brand = h1_ele.ele('@tag()=strong').text
except:
pass
title = h1_ele.ele('@tag()=span').text
desc = ""
try:
desc = formats.clean_html(tab.ele('#product-info-tab').html)
except:
pass
price = tab.ele('.price').attr('title').replace('SGD', '').replace(',', '')
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
goods_info.attr = 'S'
goods_info.attr_items = []
goods_info.images = main_images
goods_info.p_lists = []
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info

140
lib/module/tfashion.py Normal file
View File

@@ -0,0 +1,140 @@
from core import spiders, types
from utils import formats, messages
class SpiderModule(spiders.Spiders):
project_name = "t-fashion"
def __init__(self):
super().__init__(
self.project_name,
bitch = 300,
worker_num = 24,
switch_clash = False,
reflush_browser = False
)
def get_category_urls(self) -> list[str]:
category_urls = []
self.tab.get('https://www.t-fashion.jp/')
one_menu_eles = self.tab.ele('.m-aside-menu-body').eles('.js-collest-li')
for one_menu_ele in one_menu_eles:
one_link_ele = one_menu_ele.ele('@tag()=a')
# one_url = one_link_ele.attr('href')
one_name = one_link_ele.text.replace('/', '-')
# category_urls.append(f"{one_url}#{one_name}")
messages.sendInfo(one_name)
two_menu_eles = one_menu_ele.eles('xpath=div/ul/li')
for two_menu_ele in two_menu_eles[1:]:
two_link_ele = two_menu_ele.ele('@tag()=a')
two_url = two_link_ele.attr('href')
two_name = two_link_ele.text.replace('/', '-')
category_urls.append(f"{two_url}#{one_name}/{two_name}")
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
# for three_menu_ele in three_menu_eles:
# three_link_ele = three_menu_ele.ele('@tag()=a')
# three_url = three_link_ele.attr('href')
# three_name = three_link_ele.text.replace('/', '-')
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
return category_urls
def get_goods_urls(self, category_url: str) -> list[str]:
page = 1
self.tab.get(f"{category_url}?page={page}")
goods_urls = []
while True:
try:
goods_eles = self.tab.eles('xpath=/html/body/div[1]/div[3]/div/section[1]/div/div[1]/div[1]/div[3]/ol/li')
for goods_ele in goods_eles:
goods_link_ele = goods_ele.ele('xpath=a')
goods_urls.append(goods_link_ele.attr('href'))
if len(goods_eles) == 0:
break
except:
break
page += 1
self.tab.get(f"{category_url}?page={page}")
return goods_urls
def get_goods_info(self, url) -> types.GoodsInfo:
def get_image_urls():
image_urls = []
image_eles = tab.eles('xpath=//*[@id="item-gallery-parent"]/li')
for image_ele in image_eles:
image_link_ele = image_ele.ele('xpath=div/img')
image_urls.append(image_link_ele.attr('src'))
return image_urls
tab = self.browser.new_tab(url)
tab.set.window.max()
spu = formats.url_to_spu(url)
try:
brand = ''
try:
brand = tab.ele('xpath=//*[@id="current-tgoods-section"]/div/div[2]/div/div[1]/a[2]', timeout=10).text
except:
pass
title = tab.ele('@tag()=h1').text
desc = ""
try:
desc = formats.clean_html(tab.ele('xpath=//*[@id="slide-down-contents-parent"]'))
except:
pass
try:
price = float(tab.ele('.is-fz28-36f is-fw-b ').text.replace(',', '').replace('¥', '').replace('(税込)', ''))
except:
price = float(tab.ele('.is-fz28-36f is-fw-b is-color-notice').text.replace(',', '').replace('¥', '').replace('(税込)', ''))
old_price = price
main_images = get_image_urls()
goods_info = types.GoodsInfo(
title=title,
desc=desc,
brand=brand,
url=url,
spu=spu,
price=price,
old_price=old_price,
)
m_data = []
attr_items = []
goods_info.attr = 'M'
goods_info.attr_items = ['Color', 'Size']
goods_info.images = main_images
color_eles = tab.eles('xpath=//*[@id="sku-list-box"]/div')
for color_ele in color_eles:
color_image = color_ele.ele('xpath=div/a').attr('href')
color_name = color_ele.ele('xpath=div/a/span').text
size_eles = color_ele.eles('xpath=ol/li')
for size_ele in size_eles:
size_name = size_ele.ele('xpath=div[1]/text()')
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
attr_items=[color_name, size_name],
price=price,
old_price=old_price,
images=[color_image]
))
except Exception as e:
try:
tab.close()
except:
pass
raise Exception(e)
try:
tab.close()
except:
pass
return goods_info