372 lines
19 KiB
Python
372 lines
19 KiB
Python
from core import spiders, types
|
|
from utils import formats, messages, files
|
|
import requests
|
|
|
|
class SpiderModule(spiders.Spiders):
|
|
project_name = "ssfshop"
|
|
|
|
def __init__(self):
|
|
super().__init__(
|
|
self.project_name,
|
|
bitch = 300,
|
|
worker_num = 24,
|
|
switch_clash = False,
|
|
reflush_browser = False
|
|
)
|
|
|
|
def get_category_urls(self) -> list[str]:
|
|
category_urls = []
|
|
|
|
self.tab.get('https://www.ssfshop.com/')
|
|
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
|
|
for one_menu_ele in one_menu_eles:
|
|
one_link_ele = one_menu_ele.ele('@tag()=a')
|
|
one_url = one_link_ele.attr('href')
|
|
one_name = one_link_ele.text.replace('/', '-')
|
|
if one_name == '라이프':
|
|
continue
|
|
category_urls.append(f"{one_url}#{one_name}")
|
|
messages.sendInfo(one_name)
|
|
|
|
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
|
|
for two_menu_ele in two_menu_eles:
|
|
two_link_ele = two_menu_ele.ele('@tag()=a')
|
|
two_url = two_link_ele.attr('href')
|
|
two_name = two_link_ele.text.replace('/', '-')
|
|
if two_name == '메인':
|
|
continue
|
|
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
|
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
|
|
for two_menu_ele in two_menu_eles:
|
|
two_link_ele = two_menu_ele.ele('@tag()=a')
|
|
two_url = two_link_ele.attr('href')
|
|
two_name = two_link_ele.text.replace('/', '-')
|
|
if two_name == '메인':
|
|
continue
|
|
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
|
|
|
|
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
|
# for three_menu_ele in three_menu_eles:
|
|
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
|
# three_url = three_link_ele.attr('href')
|
|
# three_name = three_link_ele.text.replace('/', '-')
|
|
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
|
|
|
return category_urls
|
|
|
|
def get_goods_urls(self, category_url: str) -> list[str]:
|
|
page = 1
|
|
self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
|
|
goods_urls = []
|
|
|
|
url_index = {}
|
|
while True:
|
|
self.tab.scroll.to_bottom()
|
|
page_urls = []
|
|
goods_eles = self.tab.eles('.god-item')
|
|
for goods_ele in goods_eles:
|
|
goods_link_ele = goods_ele.ele('xpath=a')
|
|
if goods_link_ele.attr('href') in url_index:
|
|
continue
|
|
page_urls.append(goods_link_ele.attr('href'))
|
|
url_index[goods_link_ele.attr('href')] = None
|
|
|
|
if len(page_urls) == 0:
|
|
break
|
|
if len(goods_urls) > 5000:
|
|
break
|
|
goods_urls += page_urls
|
|
page += 1
|
|
self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
|
|
self.tab.wait(1)
|
|
|
|
return goods_urls
|
|
|
|
def get_goods_info(self, url: str) -> types.GoodsInfo:
|
|
def get_image_urls():
|
|
image_urls = []
|
|
|
|
for _ in range(10):
|
|
image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div')
|
|
if len(image_eles) > 0:
|
|
break
|
|
tab.wait(3)
|
|
for image_ele in image_eles:
|
|
image_link_ele = image_ele
|
|
image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data'))
|
|
|
|
return image_urls
|
|
|
|
def get_options(godNo, optValCd1, optValCd2 = ''):
|
|
headers = {
|
|
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
|
'Connection': 'keep-alive',
|
|
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
|
'Origin': 'https://www.ssfshop.com',
|
|
'Referer': url,
|
|
'Sec-Fetch-Dest': 'empty',
|
|
'Sec-Fetch-Mode': 'cors',
|
|
'Sec-Fetch-Site': 'same-origin',
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
|
|
'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0',
|
|
'X-Requested-With': 'XMLHttpRequest',
|
|
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
|
|
'sec-ch-ua-mobile': '?0',
|
|
'sec-ch-ua-platform': '"Windows"',
|
|
# 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950',
|
|
}
|
|
|
|
data = {
|
|
'optValCd1': optValCd1,
|
|
'optValCd2': optValCd2,
|
|
'optValCd3': '',
|
|
'godNo': godNo,
|
|
}
|
|
|
|
response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'})
|
|
json_data = response.json()
|
|
|
|
options = {}
|
|
if 'opt' not in json_data:
|
|
return options
|
|
|
|
for option in json_data['opt']:
|
|
for i in range(1, 6):
|
|
key = f"optValNm{i}"
|
|
if option[key]:
|
|
options[option[key]] = {}
|
|
break
|
|
return options
|
|
|
|
tab = self.browser.new_tab(url)
|
|
tab.set.window.max()
|
|
spu = formats.url_to_spu(url)
|
|
|
|
try:
|
|
brand = url.split('/')[3]
|
|
godNo = url.split('/')[4]
|
|
|
|
title = tab.ele('#goodDtlTitle').text
|
|
|
|
desc = ""
|
|
try:
|
|
desc = tab.ele('#godsTabView').html
|
|
except:
|
|
pass
|
|
|
|
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
|
|
try:
|
|
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
|
|
except:
|
|
old_price = price
|
|
|
|
main_images = get_image_urls()
|
|
|
|
goods_info = types.GoodsInfo(
|
|
title=title,
|
|
desc=desc,
|
|
brand=brand,
|
|
url=url,
|
|
spu=spu,
|
|
price=price,
|
|
old_price=old_price,
|
|
)
|
|
|
|
m_data = []
|
|
attr_items = []
|
|
color_urls = []
|
|
try:
|
|
tab.ele('.gods-option').ele('text=색상').click()
|
|
color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label')
|
|
for color_ele in color_eles:
|
|
color_url = color_ele.attr('onclick')
|
|
if color_url:
|
|
color_url = color_url.split("'")[1]
|
|
else:
|
|
color_url = url
|
|
if 'https://' not in color_url:
|
|
color_url = 'https://www.ssfshop.com'+color_url
|
|
color_urls.append(color_url)
|
|
except:
|
|
pass
|
|
|
|
if len(color_urls) > 1:
|
|
m_data = ["색상"]
|
|
|
|
try:
|
|
option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv')
|
|
except:
|
|
option_eles = []
|
|
for option_ele in option_eles:
|
|
m_data.append(option_ele.ele('@tag()=input').attr('opt-nm'))
|
|
m_data = m_data[:3]
|
|
|
|
option_data = {}
|
|
if len(option_eles) > 0:
|
|
try:
|
|
option_eles[0].click()
|
|
except:
|
|
goods_info.attr = 'S'
|
|
goods_info.attr_items = []
|
|
goods_info.images = main_images
|
|
goods_info.p_lists = []
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
return goods_info
|
|
tab.wait(0.5)
|
|
option1_eles = option_eles[0].eles('@tag()=li')
|
|
for option1_ele in option1_eles:
|
|
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
|
|
for option2_name, items in option_data[option1_ele.text].items():
|
|
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
|
|
|
|
if len(color_urls) > 1:
|
|
for color_url in color_urls:
|
|
color_images = main_images
|
|
if color_url != url:
|
|
tab.get(color_url)
|
|
title = tab.ele('#goodDtlTitle').text
|
|
color_images = get_image_urls()
|
|
|
|
godNo = color_url.split('/')[4]
|
|
try:
|
|
option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied')
|
|
except:
|
|
option_eles = []
|
|
option_data = {}
|
|
if len(option_eles) > 0:
|
|
option1_eles = option_eles[0].eles('xpath=div/ul/li')
|
|
for option1_ele in option1_eles:
|
|
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
|
|
for option2_name, items in option_data[option1_ele.text].items():
|
|
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
|
|
|
|
try:
|
|
color_name = title.split('-')[1]
|
|
except:
|
|
color_name = tab.ele('.buy-txts').text
|
|
|
|
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
|
|
try:
|
|
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
|
|
except:
|
|
old_price = price
|
|
|
|
if len(option_data) > 0:
|
|
for option1_name, items1 in option_data.items():
|
|
if len(items1) > 0:
|
|
for option2_name, items2 in items1.items():
|
|
attr_items.append({
|
|
'url': color_url,
|
|
'items': [color_name, option1_name, option2_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': color_images
|
|
})
|
|
else:
|
|
attr_items.append({
|
|
'url': color_url,
|
|
'items': [color_name, option1_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': color_images
|
|
})
|
|
else:
|
|
attr_items.append({
|
|
'url': color_url,
|
|
'items': [color_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': color_images
|
|
})
|
|
else:
|
|
for option1_name, items1 in option_data.items():
|
|
if len(items1) > 0:
|
|
for option2_name, items2 in items1.items():
|
|
if len(items2) > 0:
|
|
for option3_name, items3 in items2.items():
|
|
attr_items.append({
|
|
'url': url,
|
|
'items': [option1_name, option2_name, option3_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': []
|
|
})
|
|
else:
|
|
attr_items.append({
|
|
'url': url,
|
|
'items': [option1_name, option2_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': []
|
|
})
|
|
else:
|
|
attr_items.append({
|
|
'url': url,
|
|
'items': [option1_name],
|
|
'price': price,
|
|
'old_price': old_price,
|
|
'images': []
|
|
})
|
|
|
|
if len(attr_items) == 0:
|
|
try:
|
|
tab.ele('.gods-option').ele('text=사이즈').click()
|
|
size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li')
|
|
goods_info.attr = 'M'
|
|
goods_info.attr_items = ['사이즈']
|
|
goods_info.images = main_images
|
|
for size_ele in size_eles:
|
|
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
|
attr_items=[size_ele.text],
|
|
price=price,
|
|
old_price=old_price,
|
|
images=[],
|
|
))
|
|
except:
|
|
goods_info.attr = 'S'
|
|
goods_info.attr_items = []
|
|
goods_info.images = main_images
|
|
goods_info.p_lists = []
|
|
else:
|
|
goods_info.attr = 'M'
|
|
goods_info.attr_items = m_data
|
|
for attr_item in attr_items:
|
|
if len(attr_item['items']) != len(m_data):
|
|
attr_item['items'] += m_data[len(attr_item['items']):]
|
|
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
|
url=attr_item['url'],
|
|
attr_items=attr_item['items'],
|
|
price=attr_item['price'],
|
|
old_price=attr_item['old_price'],
|
|
images=attr_item['images'],
|
|
))
|
|
for image_url in attr_item['images']:
|
|
if image_url not in main_images:
|
|
main_images.append(image_url)
|
|
goods_info.images = main_images
|
|
goods_info.p_urls = color_urls
|
|
except Exception as e:
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
raise Exception(e)
|
|
try:
|
|
tab.close()
|
|
except:
|
|
pass
|
|
return goods_info
|
|
|
|
def get_home(self):
|
|
self.tab.get('https://www.ssfshop.com/main')
|
|
li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li')
|
|
categorys = []
|
|
for li_ele in li_eles:
|
|
a_ele = li_ele.ele('@tag()=a')
|
|
categorys.append(f"{a_ele.attr('href')}#{a_ele.text}")
|
|
files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys)
|