from core import spiders, types from utils import formats, messages, files import requests class SpiderModule(spiders.Spiders): project_name = "ssfshop" def __init__(self): super().__init__( self.project_name, bitch = 300, worker_num = 24, switch_clash = False, reflush_browser = False ) def get_category_urls(self) -> list[str]: category_urls = [] self.tab.get('https://www.ssfshop.com/') one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li') for one_menu_ele in one_menu_eles: one_link_ele = one_menu_ele.ele('@tag()=a') one_url = one_link_ele.attr('href') one_name = one_link_ele.text.replace('/', '-') if one_name == '라이프': continue category_urls.append(f"{one_url}#{one_name}") messages.sendInfo(one_name) two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') if two_name == '메인': continue category_urls.append(f"{two_url}#{one_name}/{two_name}") two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li') for two_menu_ele in two_menu_eles: two_link_ele = two_menu_ele.ele('@tag()=a') two_url = two_link_ele.attr('href') two_name = two_link_ele.text.replace('/', '-') if two_name == '메인': continue category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}") # three_menu_eles = two_menu_ele.eles('xpath=ul/li') # for three_menu_ele in three_menu_eles: # three_link_ele = three_menu_ele.ele('@tag()=a') # three_url = three_link_ele.attr('href') # three_name = three_link_ele.text.replace('/', '-') # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls def get_goods_urls(self, category_url: str) -> list[str]: page = 1 self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N") goods_urls = [] url_index = {} while True: self.tab.scroll.to_bottom() page_urls = [] goods_eles = self.tab.eles('.god-item') for goods_ele in goods_eles: goods_link_ele = goods_ele.ele('xpath=a') if goods_link_ele.attr('href') in url_index: continue page_urls.append(goods_link_ele.attr('href')) url_index[goods_link_ele.attr('href')] = None if len(page_urls) == 0: break if len(goods_urls) > 5000: break goods_urls += page_urls page += 1 self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N") self.tab.wait(1) return goods_urls def get_goods_info(self, url: str) -> types.GoodsInfo: def get_image_urls(): image_urls = [] for _ in range(10): image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div') if len(image_eles) > 0: break tab.wait(3) for image_ele in image_eles: image_link_ele = image_ele image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data')) return image_urls def get_options(godNo, optValCd1, optValCd2 = ''): headers = { 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', 'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'Origin': 'https://www.ssfshop.com', 'Referer': url, 'Sec-Fetch-Dest': 'empty', 'Sec-Fetch-Mode': 'cors', 'Sec-Fetch-Site': 'same-origin', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0', 'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0', 'X-Requested-With': 'XMLHttpRequest', 'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"', 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': '"Windows"', # 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950', } data = { 'optValCd1': optValCd1, 'optValCd2': optValCd2, 'optValCd3': '', 'godNo': godNo, } response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'}) json_data = response.json() options = {} if 'opt' not in json_data: return options for option in json_data['opt']: for i in range(1, 6): key = f"optValNm{i}" if option[key]: options[option[key]] = {} break return options tab = self.browser.new_tab(url) tab.set.window.max() spu = formats.url_to_spu(url) try: brand = url.split('/')[3] godNo = url.split('/')[4] title = tab.ele('#goodDtlTitle').text desc = "" try: desc = tab.ele('#godsTabView').html except: pass price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', '')) try: old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', '')) except: old_price = price main_images = get_image_urls() goods_info = types.GoodsInfo( title=title, desc=desc, brand=brand, url=url, spu=spu, price=price, old_price=old_price, ) m_data = [] attr_items = [] color_urls = [] try: tab.ele('.gods-option').ele('text=색상').click() color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label') for color_ele in color_eles: color_url = color_ele.attr('onclick') if color_url: color_url = color_url.split("'")[1] else: color_url = url if 'https://' not in color_url: color_url = 'https://www.ssfshop.com'+color_url color_urls.append(color_url) except: pass if len(color_urls) > 1: m_data = ["색상"] try: option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv') except: option_eles = [] for option_ele in option_eles: m_data.append(option_ele.ele('@tag()=input').attr('opt-nm')) m_data = m_data[:3] option_data = {} if len(option_eles) > 0: try: option_eles[0].click() except: goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] try: tab.close() except: pass return goods_info tab.wait(0.5) option1_eles = option_eles[0].eles('@tag()=li') for option1_ele in option1_eles: option_data[option1_ele.text] = get_options(godNo, option1_ele.text) for option2_name, items in option_data[option1_ele.text].items(): option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name) if len(color_urls) > 1: for color_url in color_urls: color_images = main_images if color_url != url: tab.get(color_url) title = tab.ele('#goodDtlTitle').text color_images = get_image_urls() godNo = color_url.split('/')[4] try: option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied') except: option_eles = [] option_data = {} if len(option_eles) > 0: option1_eles = option_eles[0].eles('xpath=div/ul/li') for option1_ele in option1_eles: option_data[option1_ele.text] = get_options(godNo, option1_ele.text) for option2_name, items in option_data[option1_ele.text].items(): option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name) try: color_name = title.split('-')[1] except: color_name = tab.ele('.buy-txts').text price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', '')) try: old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', '')) except: old_price = price if len(option_data) > 0: for option1_name, items1 in option_data.items(): if len(items1) > 0: for option2_name, items2 in items1.items(): attr_items.append({ 'url': color_url, 'items': [color_name, option1_name, option2_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: attr_items.append({ 'url': color_url, 'items': [color_name, option1_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: attr_items.append({ 'url': color_url, 'items': [color_name], 'price': price, 'old_price': old_price, 'images': color_images }) else: for option1_name, items1 in option_data.items(): if len(items1) > 0: for option2_name, items2 in items1.items(): if len(items2) > 0: for option3_name, items3 in items2.items(): attr_items.append({ 'url': url, 'items': [option1_name, option2_name, option3_name], 'price': price, 'old_price': old_price, 'images': [] }) else: attr_items.append({ 'url': url, 'items': [option1_name, option2_name], 'price': price, 'old_price': old_price, 'images': [] }) else: attr_items.append({ 'url': url, 'items': [option1_name], 'price': price, 'old_price': old_price, 'images': [] }) if len(attr_items) == 0: try: tab.ele('.gods-option').ele('text=사이즈').click() size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li') goods_info.attr = 'M' goods_info.attr_items = ['사이즈'] goods_info.images = main_images for size_ele in size_eles: goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( attr_items=[size_ele.text], price=price, old_price=old_price, images=[], )) except: goods_info.attr = 'S' goods_info.attr_items = [] goods_info.images = main_images goods_info.p_lists = [] else: goods_info.attr = 'M' goods_info.attr_items = m_data for attr_item in attr_items: if len(attr_item['items']) != len(m_data): attr_item['items'] += m_data[len(attr_item['items']):] goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( url=attr_item['url'], attr_items=attr_item['items'], price=attr_item['price'], old_price=attr_item['old_price'], images=attr_item['images'], )) for image_url in attr_item['images']: if image_url not in main_images: main_images.append(image_url) goods_info.images = main_images goods_info.p_urls = color_urls except Exception as e: try: tab.close() except: pass raise Exception(e) try: tab.close() except: pass return goods_info def get_home(self): self.tab.get('https://www.ssfshop.com/main') li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li') categorys = [] for li_ele in li_eles: a_ele = li_ele.ele('@tag()=a') categorys.append(f"{a_ele.attr('href')}#{a_ele.text}") files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys)