from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "friskybusiness" def __init__(self): super().__init__( self.project_name, bitch = 30, worker_num = 5, switch_clash = False, reflush_browser = False ) self.tab.get('https://friskybusiness.sg/') # input('wait:') def get_category_urls(self) -> list[str]: category_urls = [] one_menu_eles = self.tab.eles('xpath=/html/body/header/height-observer/x-header/nav[2]/ul/li') for one_index, one_menu_ele in enumerate(one_menu_eles[:-1]): one_link_ele = one_menu_ele.ele('@tag()=summary') one_url = one_link_ele.attr('data-follow-link') one_name = one_link_ele.text.replace('/', '-') category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) if one_index == 0 or one_index == 6: two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/div/a') else: two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/ul/li', timeout=1) for two_index, two_menu_ele in enumerate(two_menu_eles): if one_index == 0 or one_index == 6: two_link_ele = two_menu_ele else: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-').replace(',', ',').replace('#', ' ') category_urls.append(f"{two_url}#{one_name}/{two_name}") try: three_menu_eles = two_menu_ele.eles(f'xpath=div/ul/li') except: continue for three_menu_ele in three_menu_eles: three_link_ele = three_menu_ele.ele('@tag()=a') three_url = three_link_ele.attr('href') three_name = three_link_ele.text.replace('/', '-').replace(',', ',').replace('#', ' ') category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: page = 1 self.tab.get(f"{category_url}?page={page}") goods_urls = [] #####获取页数#### ################# goods_urls = [] while True: self.tab.scroll.to_bottom() try: goods_eles = self.tab.eles('#:snize-product-', timeout=5) for goods_ele in goods_eles: goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" goods_urls.append(goods_url) if len(goods_eles) == 0: break except: break page += 1 self.tab.get(f"{category_url}?page={page}") messages.sendInfo(f"{len(goods_urls)} 个") return goods_urls def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] image_eles = tab.eles('xpath=/html/body/main/section[1]/div/div/product-rerender/div/product-gallery/safe-sticky/product-gallery-navigation/button') for image_ele in image_eles[-1:]: image_url = image_ele.ele('xpath=img').attr('src') image_urls.append(image_url) return image_urls tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) try: h1_ele = tab.ele('@tag()=h1') brand = '' title = h1_ele.text desc = "" try: desc = formats.clean_html(tab.ele('.liquid').html) except: pass price_list = tab.ele('.price-list price-list--product') price = price_list.ele('@tag()=sale-price').ele('.money').text.replace('$', '').replace(',', '') try: old_price = price_list.ele('@tag()=compare-at-price').ele('.money').text.replace('$', '').replace(',', '') except: old_price = price main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] except Exception as e: try: tab.close() except: pass raise Exception(e) try: tab.close() except: pass return goods_info