from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "ancientnutrition" def __init__(self): super().__init__( self.project_name, bitch = 30, worker_num = 1, switch_clash = False, reflush_browser = False ) def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('https://ancientnutrition.com/') one_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[4]/div[1]/div[1]/div') for index, one_menu_ele in enumerate(one_menu_eles[1:]): one_link_ele = one_menu_ele.ele('xpath=div/a') one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-') category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) print(index+2) two_menu_eles = self.tab.eles(f'xpath=//*[@id="__next"]/div[4]/div[{index+2}]/div/div/div[2]/div/a') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') category_urls.append(f"{two_url}#{one_name}/{two_name}") # three_menu_eles = two_menu_ele.eles('xpath=ul/li') # for three_menu_ele in three_menu_eles: # three_link_ele = three_menu_ele.ele('@tag()=a') # three_url = three_link_ele.attr('href') # three_name = three_link_ele.text.replace('/', '-') # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: self.tab.get(f"{category_url}") goods_urls = [] while True: try: self.tab.scroll.to_bottom() self.tab.ele('text=Load More').click() except: break goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[9]/div/div[2]/div[2]/div') for goods_ele in goods_eles: try: goods_link_ele = goods_ele.ele('@tag()=a') except Exception as e: continue goods_urls.append(goods_link_ele.attr('href')) return goods_urls def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] image_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[1]/div/div[2]/div/div[1]/div') for image_ele in image_eles: image_link_ele = image_ele.ele('xpath=div/span/img') image_urls.append(image_link_ele.attr('src').replace('_150x150', '_1450x1450')) return image_urls tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) try: brand = '' # try: # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text # except: # pass # if brand == '': # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text # else: title = tab.ele('@tag()=h1').text desc = formats.clean_html(tab.ele('#productDescription').html) price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) m_data = [] attr_items = [] try: tab.ele('text=Select Flavor').click() m_data.append('Flavor') except: pass try: tab.ele('text=Select Size').click() m_data.append('Size') except: pass try: tab.ele('text=Select Quantity').click() m_data.append('Quantity') except: pass if 'Flavor' in m_data: tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click() color_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div[2]/div') color_xpaths = [] for color_ele in color_eles: color_xpaths.append(color_ele.xpath) for color_xpath in color_xpaths: while True: tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click() try: color_ele = tab.ele(f"xpath={color_xpath}") color_name = color_ele.text break except: tab.get(url) color_ele.click() tab.wait(3) color_images = get_image_urls() if 'Size' in m_data: try: size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}") except: size_name = 'Default' if 'Quantity' in m_data: quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") for quantity_index, quantity_ele in enumerate(quantity_eles): try: quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") quantity_name = quantity_ele.text except: continue quantity_ele.click() tab.wait(2) price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [color_name, size_name, quantity_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [color_name, size_name], 'price': price, 'old_price': old_price, 'images': color_images }) size_xpaths = [] for size_ele in size_eles: try: size_xpaths.append(size_ele.xpath) except: continue for size_xpath in size_xpaths: try: size_ele = tab.ele(f"xpath={size_xpath}") size_name = size_ele.text except: continue size_ele.click() tab.wait(3) color_images = get_image_urls() if 'Quantity' in m_data: quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") for quantity_index, quantity_ele in enumerate(quantity_eles): try: quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") quantity_name = quantity_ele.text except: continue quantity_ele.click() tab.wait(2) price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [color_name, size_name, quantity_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [color_name, size_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [color_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: color_images = [] if 'Size' in m_data: size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}") size_xpaths = [] for size_ele in size_eles: try: size_xpaths.append(size_ele.xpath) except: continue for size_xpath in size_xpaths: try: size_ele = tab.ele(f"xpath={size_xpath}") size_name = size_ele.text except: continue size_ele.click() tab.wait(2) color_images = get_image_urls() if 'Quantity' in m_data: quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") for quantity_index, quantity_ele in enumerate(quantity_eles): try: quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") quantity_name = quantity_ele.text except: continue quantity_ele.click() tab.wait(2) price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [size_name, quantity_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: if 'Quantity' in m_data: quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") for quantity_index, quantity_ele in enumerate(quantity_eles): try: quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") quantity_name = quantity_ele.text except: continue quantity_ele.click() tab.wait(2) price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) attr_items.append({ 'url': tab.url, 'items': [quantity_name], 'price': price, 'old_price': old_price, 'images': color_images }) if len(attr_items) == 0: goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] else: goods_info.attr = 'M' goods_info.attr_items = m_data for attr_item in attr_items: goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( url=attr_item['url'], attr_items=attr_item['items'], price=attr_item['price'], old_price=attr_item['old_price'], images=attr_item['images'], )) for image_url in attr_item['images']: if image_url not in main_images: main_images.append(image_url) goods_info.images = main_images except Exception as e: try: tab.close() except: pass raise Exception(e) try: tab.close() except: pass return goods_info