from core import spiders, types from utils import formats, messages class SpiderModule(spiders.Spiders): project_name = "t-fashion" def __init__(self): super().__init__( self.project_name, bitch = 300, worker_num = 24, switch_clash = False, reflush_browser = False ) def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('https://www.t-fashion.jp/') one_menu_eles = self.tab.ele('.m-aside-menu-body').eles('.js-collest-li') for one_menu_ele in one_menu_eles: one_link_ele = one_menu_ele.ele('@tag()=a') # one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-') # category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) two_menu_eles = one_menu_ele.eles('xpath=div/ul/li') for two_menu_ele in two_menu_eles[1:]: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') category_urls.append(f"{two_url}#{one_name}/{two_name}") # three_menu_eles = two_menu_ele.eles('xpath=ul/li') # for three_menu_ele in three_menu_eles: # three_link_ele = three_menu_ele.ele('@tag()=a') # three_url = three_link_ele.attr('href') # three_name = three_link_ele.text.replace('/', '-') # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: page = 1 self.tab.get(f"{category_url}?page={page}") goods_urls = [] while True: try: goods_eles = self.tab.eles('xpath=/html/body/div[1]/div[3]/div/section[1]/div/div[1]/div[1]/div[3]/ol/li') for goods_ele in goods_eles: goods_link_ele = goods_ele.ele('xpath=a') goods_urls.append(goods_link_ele.attr('href')) if len(goods_eles) == 0: break except: break page += 1 self.tab.get(f"{category_url}?page={page}") return goods_urls def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] image_eles = tab.eles('xpath=//*[@id="item-gallery-parent"]/li') for image_ele in image_eles: image_link_ele = image_ele.ele('xpath=div/img') image_urls.append(image_link_ele.attr('src')) return image_urls tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) try: brand = '' try: brand = tab.ele('xpath=//*[@id="current-tgoods-section"]/div/div[2]/div/div[1]/a[2]', timeout=10).text except: pass title = tab.ele('@tag()=h1').text desc = "" try: desc = formats.clean_html(tab.ele('xpath=//*[@id="slide-down-contents-parent"]')) except: pass try: price = float(tab.ele('.is-fz28-36f is-fw-b ').text.replace(',', '').replace('¥', '').replace('(税込)', '')) except: price = float(tab.ele('.is-fz28-36f is-fw-b is-color-notice').text.replace(',', '').replace('¥', '').replace('(税込)', '')) old_price = price main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) m_data = [] attr_items = [] goods_info.attr = 'M' goods_info.attr_items = ['Color', 'Size'] goods_info.images = main_images color_eles = tab.eles('xpath=//*[@id="sku-list-box"]/div') for color_ele in color_eles: color_image = color_ele.ele('xpath=div/a').attr('href') color_name = color_ele.ele('xpath=div/a/span').text size_eles = color_ele.eles('xpath=ol/li') for size_ele in size_eles: size_name = size_ele.ele('xpath=div[1]/text()') goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( attr_items=[color_name, size_name], price=price, old_price=old_price, images=[color_image] )) except Exception as e: try: tab.close() except: pass raise Exception(e) try: tab.close() except: pass return goods_info