from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "dk_one" def __init__(self): super().__init__( self.project_name, bitch = 30, worker_num = 1, switch_clash = False, reflush_browser = False ) def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('https://www.dk_one.com/') one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li') for one_menu_ele in one_menu_eles: one_link_ele = one_menu_ele.ele('@tag()=a') one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-') if one_name == '라이프': continue category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') if two_name == '메인': continue category_urls.append(f"{two_url}#{one_name}/{two_name}") two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') if two_name == '메인': continue category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}") # three_menu_eles = two_menu_ele.eles('xpath=ul/li') # for three_menu_ele in three_menu_eles: # three_link_ele = three_menu_ele.ele('@tag()=a') # three_url = three_link_ele.attr('href') # three_name = three_link_ele.text.replace('/', '-') # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: self.tab.get(category_url) goods_urls = [] goods_eles = self.tab.eles('xpath=//*[@id="prodList"]/div') for goods_ele in goods_eles: goods_link_ele = goods_ele.ele('.prod-link') goods_urls.append(goods_link_ele.attr('href')) return goods_urls def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(tab): image_urls = [] image_eles = tab.ele('#prod-img').eles('@tag()=img') for image_ele in image_eles: image_urls.append(image_ele.attr('src')) return image_urls tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) try: brand = '' # try: # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text # except: # pass # if brand == '': # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text # else: title = tab.ele('.page-title').text try: desc = formats.clean_html(tab.ele('@@class=tab-content@@id=tabContent0202').html) except: desc = '' price = float(tab.ele('.price-group').ele('.val').text.replace(',', '')) try: old_price = float(tab.ele('.price-group origin').text.replace(',', '')) except: old_price = price main_images = get_image_urls(tab) goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, images=main_images ) m_data = [] color_data = {} color_eles = [] # 颜色 tab.ele('.opt-color') m_data.append('컬러') color_eles = tab.ele('.color-list-wrap').eles('.prod-color-rdo') m_data.append('사이즈') for color_ele in color_eles: color_input_onclick = color_ele.ele('@tag()=input').attr('onclick') have_url = False if 'location.href=' in color_input_onclick: color_url = 'https://dk-on.com' + formats.re_search(color_input_onclick, r"location\.href\s*=\s*'([^']*)'") color_name = color_url.split('/')[-1] have_url = True elif 'fnColorClicked' in color_input_onclick: color_url = goods_info.url color_name = color_ele.ele('@tag()=input').attr('value') color_ele.click() else: color_url = goods_info.url color_name = goods_info.title.split(' / ')[-1] color_data[color_name] = {'url': color_url, 'have_url': have_url, 'ele': color_ele} goods_info.p_urls.append(color_url) real_color_data = {} for color_name, color_item in color_data.items(): color_tab = tab if color_item['have_url']: color_tab = self.browser.new_tab() size_data = [] if color_item['have_url']: color_tab.get(color_item['url']) else: color_item['ele'].click() color_tab.wait(1) color_item['price'] = str(color_tab.ele('.price-group').ele('.val').text.replace(',', '')) try: color_item['old_price'] = str(color_tab.ele('.price-group origin').text.replace(',', '')) except: color_item['old_price'] = color_item['price'] # 颜色对应商品图 color_item['images'] = get_image_urls(tab = color_tab) # 颜色对应尺码 radio_eles = color_tab.ele('.prod-size-list').eles('.prod-size') for radio_ele in radio_eles: value = radio_ele.ele('@tag()=label').text if value == None: continue size_data.append(value) color_item['size'] = size_data real_color_data[color_tab.ele('.page-title').text.split(' / ')[-1]] = color_item color_data = real_color_data goods_info.attr = 'M' goods_info.title = goods_info.title.split(' / ')[0] goods_info.attr_items = m_data for color_name, color_item in color_data.items(): for color_image in color_item['images']: if color_image in goods_info.images: continue goods_info.images.append(color_image) # if color_data: item_data = [] for color_name, color_item in color_data.items(): for size in color_item['size']: item_data.append([color_name, size]) for attribute_items in item_data: goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( attr_items=attribute_items, price=color_data[attribute_items[0]]['price'], old_price=color_data[attribute_items[0]]['price'], images=color_data[attribute_items[0]]['images'], )) except Exception as e: try: tab.close() color_tab.close() except: pass raise Exception(e) try: tab.close() color_tab.close() except: pass return goods_info