from core import spiders, types from utils import formats, messages from DrissionPage.common import Keys class SpiderModule(spiders.Spiders): project_name = "bricomarche" def __init__(self): super().__init__( self.project_name, bitch = 5, worker_num = 1, switch_clash = False, reflush_browser = False ) self.tab.get('https://www.bricomarche.pl/') # input('wait:') def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('https://www.bricomarche.pl/') one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div') for one_menu_ele in one_menu_eles: one_link_ele = one_menu_ele.ele('@tag()=a') one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-') category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') category_urls.append(f"{two_url}#{one_name}/{two_name}") three_menu_eles = two_menu_ele.eles('xpath=ul/li') for three_menu_ele in three_menu_eles: three_link_ele = three_menu_ele.ele('@tag()=a') three_url = three_link_ele.attr('href') three_name = three_link_ele.text.replace('/', '-') category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: page = 1 limit = 100 self.tab.get(f"{category_url}?page={page}&limit={limit}") if self.tab.ele('@tag()=h2').text == '执行安全验证': self.tab.wait(10) self.tab.actions.type((Keys.TAB, Keys.SPACE)) goods_urls = [] while True: try: self.tab.scroll.to_bottom() goods_eles = self.tab.eles('xpath=//*[@id="spark"]/div[2]/div/section/div/div/main/div[2]/section/div') for goods_ele in goods_eles: goods_link_ele = goods_ele.ele('@tag()=a') goods_urls.append(goods_link_ele.attr('href')) if len(goods_eles) == 0: break except: break page += 1 self.tab.get(f"{category_url}?page={page}&limit={limit}") if self.tab.ele('@tag()=h2').text == '执行安全验证': self.tab.wait(10) self.tab.actions.type((Keys.TAB, Keys.SPACE)) return goods_urls def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] image_eles = tab.eles('xpath=//*[@id="spark"]/div[2]/div/main/div/div[2]/section[1]/div/div[1]/div/div/div/div/div') for image_ele in image_eles: image_link_ele = image_ele.ele('xpath=button/div/img') image_urls.append(image_link_ele.attr('src').replace('gallery_100_91', 'gallery')) return image_urls tab = self.browser.new_tab(url) while True: try: if tab.ele('@tag()=h2').text == '执行安全验证': tab.wait(10) tab.actions.type((Keys.TAB, Keys.SPACE)) tab.wait(5) else: break except: break tab.set.window.max() spu = formats.url_to_spu(url) try: brand = '' # try: # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text # except: # pass title = tab.ele('@tag()=h1', timeout=10).text desc = "" try: desc = formats.clean_html(tab.ele('.attributes').html) except: pass # try: price_ele = tab.ele('.main-price') price = float(f"{price_ele.ele('.whole').text}{price_ele.ele('.cents').text.replace(' zł', '')}") # except: # price = 0.0 old_price = price main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) m_data = [] attr_items = [] # color_urls = [url] # color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li') # for color_ele in color_eles[1:]: # color_link_ele = color_ele.ele('xpath=a') # color_urls.append(color_link_ele.attr('href')) # if len(color_urls) > 1: # m_data = ["Color"] # if len(color_urls) > 1: # for color_url in color_urls: # color_images = main_images # if color_url != url: # tab.get(color_url) # if brand == '': # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text # else: # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text # color_images = get_image_urls() # price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") # old_price = price # try: # item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text # except: # continue # attr_items.append({ # 'url': color_url, # 'items': [item_name], # 'price': price, # 'old_price': old_price, # 'images': color_images # }) if len(attr_items) == 0: goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] else: goods_info.attr = 'M' goods_info.attr_items = m_data for attr_item in attr_items: goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( url=attr_item['url'], attr_items=attr_item['items'], price=attr_item['price'], old_price=attr_item['old_price'], images=attr_item['images'], )) for image_url in attr_item['images']: if image_url not in main_images: main_images.append(image_url) goods_info.images = main_images # goods_info.p_urls = color_urls except Exception as e: try: tab.close() except: pass raise Exception(e) try: tab.close() except: pass return goods_info