from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "peek-cloppenburg" def __init__(self): super().__init__( self.project_name, bitch = 30, worker_num = 5, switch_clash = False, reflush_browser = False ) def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('https://www.peek-cloppenburg.at/') one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a') for one_menu_ele in one_menu_eles: one_link_ele = one_menu_ele one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-') category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) one_link_ele.click() self.tab.wait(3) two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/a', timeout=1) for two_index, two_menu_ele in enumerate(two_menu_eles): two_link_ele = two_menu_ele two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') category_urls.append(f"{two_url}#{one_name}/{two_name}") try: three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') except: continue for three_menu_ele in three_menu_eles: three_link_ele = three_menu_ele.ele('@tag()=a') three_url = three_link_ele.attr('href') three_name = three_link_ele.text.replace('/', '-') category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: self.tab.get(f"{category_url}?page=1") goods_urls = [] max_page = 1 #####获取页数#### try: max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text) except: pass ################# goods_urls = [] if max_page > 1: for page in range(1, max_page+1): if page > 1: self.tab.get(f"{category_url}?page={page}&userId=123456") goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') for goods_ele in goods_eles: goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" goods_urls.append(goods_url) messages.sendInfo(f"{page}/{max_page}") else: goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') for goods_ele in goods_eles: goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" goods_urls.append(goods_url) return goods_urls def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a') for image_ele in image_eles: image_url = image_ele.attr('href') image_urls.append(image_url) return image_urls def get_size_eles(): try: eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li') except: pass try: eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li') except: pass try: eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li') except: pass try: eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li') except: pass return eles tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) try: h1_eles = tab.ele('@tag()=h1', timeout=10).eles('@tag()=span') brand = '' try: brand = h1_eles[0].text title = h1_eles[1].text except: title = tab.ele('@tag()=h1').text desc = "" try: desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}") except: pass price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '.') try: old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '.') except: old_price = price main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) m_data = [] attr_items = [] color_urls = [] try: color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li') m_data.append('Farbe') for color_ele in color_eles: color_url = color_ele.ele('@tag()=a').attr('href') color_urls.append(color_url) except: pass m_data.append('Größe') if len(color_urls) > 0: for color_url in color_urls: color_images = main_images if color_url != url: tab.get(color_url) color_images = get_image_urls() price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '') try: old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '') except: old_price = price color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]') size_eles = get_size_eles() for size_ele in size_eles: size_name = size_ele.ele('.text-cp-md truncate').text attr_items.append({ 'url': color_url, 'items': [color_name, size_name], 'price': price, 'old_price': old_price, 'images': color_images }) if len(attr_items) == 0: goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] else: goods_info.attr = 'M' goods_info.attr_items = m_data for attr_item in attr_items: goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( url=attr_item['url'], attr_items=attr_item['items'], price=attr_item['price'], old_price=attr_item['old_price'], images=attr_item['images'], )) for image_url in attr_item['images']: if image_url not in main_images: main_images.append(image_url) goods_info.images = main_images goods_info.p_urls = color_urls except Exception as e: try: tab.close() except: pass raise Exception(e) try: tab.close() except: pass return goods_info