初始化
This commit is contained in:
210
lib/module/dk_one.py
Normal file
210
lib/module/dk_one.py
Normal file
@@ -0,0 +1,210 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "dk_one"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 30,
|
||||
worker_num = 1,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.dk_one.com/')
|
||||
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
if one_name == '라이프':
|
||||
continue
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
self.tab.get(category_url)
|
||||
goods_urls = []
|
||||
goods_eles = self.tab.eles('xpath=//*[@id="prodList"]/div')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('.prod-link')
|
||||
goods_urls.append(goods_link_ele.attr('href'))
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url) -> types.GoodsInfo:
|
||||
def get_image_urls(tab):
|
||||
image_urls = []
|
||||
image_eles = tab.ele('#prod-img').eles('@tag()=img')
|
||||
for image_ele in image_eles:
|
||||
image_urls.append(image_ele.attr('src'))
|
||||
return image_urls
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = ''
|
||||
# try:
|
||||
# brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text
|
||||
# except:
|
||||
# pass
|
||||
# if brand == '':
|
||||
# title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text
|
||||
# else:
|
||||
title = tab.ele('.page-title').text
|
||||
|
||||
try:
|
||||
desc = formats.clean_html(tab.ele('@@class=tab-content@@id=tabContent0202').html)
|
||||
except:
|
||||
desc = ''
|
||||
|
||||
price = float(tab.ele('.price-group').ele('.val').text.replace(',', ''))
|
||||
try:
|
||||
old_price = float(tab.ele('.price-group origin').text.replace(',', ''))
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls(tab)
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
images=main_images
|
||||
)
|
||||
|
||||
m_data = []
|
||||
color_data = {}
|
||||
color_eles = []
|
||||
# 颜色
|
||||
tab.ele('.opt-color')
|
||||
m_data.append('컬러')
|
||||
color_eles = tab.ele('.color-list-wrap').eles('.prod-color-rdo')
|
||||
|
||||
m_data.append('사이즈')
|
||||
|
||||
for color_ele in color_eles:
|
||||
color_input_onclick = color_ele.ele('@tag()=input').attr('onclick')
|
||||
have_url = False
|
||||
if 'location.href=' in color_input_onclick:
|
||||
color_url = 'https://dk-on.com' + formats.re_search(color_input_onclick, r"location\.href\s*=\s*'([^']*)'")
|
||||
color_name = color_url.split('/')[-1]
|
||||
have_url = True
|
||||
elif 'fnColorClicked' in color_input_onclick:
|
||||
color_url = goods_info.url
|
||||
color_name = color_ele.ele('@tag()=input').attr('value')
|
||||
color_ele.click()
|
||||
else:
|
||||
color_url = goods_info.url
|
||||
color_name = goods_info.title.split(' / ')[-1]
|
||||
|
||||
color_data[color_name] = {'url': color_url, 'have_url': have_url, 'ele': color_ele}
|
||||
goods_info.p_urls.append(color_url)
|
||||
|
||||
real_color_data = {}
|
||||
|
||||
|
||||
for color_name, color_item in color_data.items():
|
||||
color_tab = tab
|
||||
if color_item['have_url']:
|
||||
color_tab = self.browser.new_tab()
|
||||
|
||||
size_data = []
|
||||
if color_item['have_url']:
|
||||
color_tab.get(color_item['url'])
|
||||
else:
|
||||
color_item['ele'].click()
|
||||
color_tab.wait(1)
|
||||
color_item['price'] = str(color_tab.ele('.price-group').ele('.val').text.replace(',', ''))
|
||||
try:
|
||||
color_item['old_price'] = str(color_tab.ele('.price-group origin').text.replace(',', ''))
|
||||
except:
|
||||
color_item['old_price'] = color_item['price']
|
||||
|
||||
# 颜色对应商品图
|
||||
color_item['images'] = get_image_urls(tab = color_tab)
|
||||
|
||||
# 颜色对应尺码
|
||||
radio_eles = color_tab.ele('.prod-size-list').eles('.prod-size')
|
||||
for radio_ele in radio_eles:
|
||||
value = radio_ele.ele('@tag()=label').text
|
||||
if value == None:
|
||||
continue
|
||||
size_data.append(value)
|
||||
color_item['size'] = size_data
|
||||
|
||||
real_color_data[color_tab.ele('.page-title').text.split(' / ')[-1]] = color_item
|
||||
color_data = real_color_data
|
||||
|
||||
goods_info.attr = 'M'
|
||||
goods_info.title = goods_info.title.split(' / ')[0]
|
||||
goods_info.attr_items = m_data
|
||||
for color_name, color_item in color_data.items():
|
||||
for color_image in color_item['images']:
|
||||
if color_image in goods_info.images:
|
||||
continue
|
||||
goods_info.images.append(color_image)
|
||||
|
||||
# if color_data:
|
||||
item_data = []
|
||||
for color_name, color_item in color_data.items():
|
||||
for size in color_item['size']:
|
||||
item_data.append([color_name, size])
|
||||
for attribute_items in item_data:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
attr_items=attribute_items,
|
||||
price=color_data[attribute_items[0]]['price'],
|
||||
old_price=color_data[attribute_items[0]]['price'],
|
||||
images=color_data[attribute_items[0]]['images'],
|
||||
))
|
||||
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
color_tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
color_tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
Reference in New Issue
Block a user