初始化
This commit is contained in:
371
lib/module/ssfshop.py
Normal file
371
lib/module/ssfshop.py
Normal file
@@ -0,0 +1,371 @@
|
||||
from core import spiders, types
|
||||
from utils import formats, messages, files
|
||||
import requests
|
||||
|
||||
class SpiderModule(spiders.Spiders):
|
||||
project_name = "ssfshop"
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
self.project_name,
|
||||
bitch = 300,
|
||||
worker_num = 24,
|
||||
switch_clash = False,
|
||||
reflush_browser = False
|
||||
)
|
||||
|
||||
def get_category_urls(self) -> list[str]:
|
||||
category_urls = []
|
||||
|
||||
self.tab.get('https://www.ssfshop.com/')
|
||||
one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li')
|
||||
for one_menu_ele in one_menu_eles:
|
||||
one_link_ele = one_menu_ele.ele('@tag()=a')
|
||||
one_url = one_link_ele.attr('href')
|
||||
one_name = one_link_ele.text.replace('/', '-')
|
||||
if one_name == '라이프':
|
||||
continue
|
||||
category_urls.append(f"{one_url}#{one_name}")
|
||||
messages.sendInfo(one_name)
|
||||
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/{two_name}")
|
||||
two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li')
|
||||
for two_menu_ele in two_menu_eles:
|
||||
two_link_ele = two_menu_ele.ele('@tag()=a')
|
||||
two_url = two_link_ele.attr('href')
|
||||
two_name = two_link_ele.text.replace('/', '-')
|
||||
if two_name == '메인':
|
||||
continue
|
||||
category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}")
|
||||
|
||||
# three_menu_eles = two_menu_ele.eles('xpath=ul/li')
|
||||
# for three_menu_ele in three_menu_eles:
|
||||
# three_link_ele = three_menu_ele.ele('@tag()=a')
|
||||
# three_url = three_link_ele.attr('href')
|
||||
# three_name = three_link_ele.text.replace('/', '-')
|
||||
# category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}")
|
||||
|
||||
return category_urls
|
||||
|
||||
def get_goods_urls(self, category_url: str) -> list[str]:
|
||||
page = 1
|
||||
self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
|
||||
goods_urls = []
|
||||
|
||||
url_index = {}
|
||||
while True:
|
||||
self.tab.scroll.to_bottom()
|
||||
page_urls = []
|
||||
goods_eles = self.tab.eles('.god-item')
|
||||
for goods_ele in goods_eles:
|
||||
goods_link_ele = goods_ele.ele('xpath=a')
|
||||
if goods_link_ele.attr('href') in url_index:
|
||||
continue
|
||||
page_urls.append(goods_link_ele.attr('href'))
|
||||
url_index[goods_link_ele.attr('href')] = None
|
||||
|
||||
if len(page_urls) == 0:
|
||||
break
|
||||
if len(goods_urls) > 5000:
|
||||
break
|
||||
goods_urls += page_urls
|
||||
page += 1
|
||||
self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N")
|
||||
self.tab.wait(1)
|
||||
|
||||
return goods_urls
|
||||
|
||||
def get_goods_info(self, url: str) -> types.GoodsInfo:
|
||||
def get_image_urls():
|
||||
image_urls = []
|
||||
|
||||
for _ in range(10):
|
||||
image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div')
|
||||
if len(image_eles) > 0:
|
||||
break
|
||||
tab.wait(3)
|
||||
for image_ele in image_eles:
|
||||
image_link_ele = image_ele
|
||||
image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data'))
|
||||
|
||||
return image_urls
|
||||
|
||||
def get_options(godNo, optValCd1, optValCd2 = ''):
|
||||
headers = {
|
||||
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
|
||||
'Connection': 'keep-alive',
|
||||
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
||||
'Origin': 'https://www.ssfshop.com',
|
||||
'Referer': url,
|
||||
'Sec-Fetch-Dest': 'empty',
|
||||
'Sec-Fetch-Mode': 'cors',
|
||||
'Sec-Fetch-Site': 'same-origin',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
|
||||
'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0',
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
|
||||
'sec-ch-ua-mobile': '?0',
|
||||
'sec-ch-ua-platform': '"Windows"',
|
||||
# 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950',
|
||||
}
|
||||
|
||||
data = {
|
||||
'optValCd1': optValCd1,
|
||||
'optValCd2': optValCd2,
|
||||
'optValCd3': '',
|
||||
'godNo': godNo,
|
||||
}
|
||||
|
||||
response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'})
|
||||
json_data = response.json()
|
||||
|
||||
options = {}
|
||||
if 'opt' not in json_data:
|
||||
return options
|
||||
|
||||
for option in json_data['opt']:
|
||||
for i in range(1, 6):
|
||||
key = f"optValNm{i}"
|
||||
if option[key]:
|
||||
options[option[key]] = {}
|
||||
break
|
||||
return options
|
||||
|
||||
tab = self.browser.new_tab(url)
|
||||
tab.set.window.max()
|
||||
spu = formats.url_to_spu(url)
|
||||
|
||||
try:
|
||||
brand = url.split('/')[3]
|
||||
godNo = url.split('/')[4]
|
||||
|
||||
title = tab.ele('#goodDtlTitle').text
|
||||
|
||||
desc = ""
|
||||
try:
|
||||
desc = tab.ele('#godsTabView').html
|
||||
except:
|
||||
pass
|
||||
|
||||
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
|
||||
try:
|
||||
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
main_images = get_image_urls()
|
||||
|
||||
goods_info = types.GoodsInfo(
|
||||
title=title,
|
||||
desc=desc,
|
||||
brand=brand,
|
||||
url=url,
|
||||
spu=spu,
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
)
|
||||
|
||||
m_data = []
|
||||
attr_items = []
|
||||
color_urls = []
|
||||
try:
|
||||
tab.ele('.gods-option').ele('text=색상').click()
|
||||
color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label')
|
||||
for color_ele in color_eles:
|
||||
color_url = color_ele.attr('onclick')
|
||||
if color_url:
|
||||
color_url = color_url.split("'")[1]
|
||||
else:
|
||||
color_url = url
|
||||
if 'https://' not in color_url:
|
||||
color_url = 'https://www.ssfshop.com'+color_url
|
||||
color_urls.append(color_url)
|
||||
except:
|
||||
pass
|
||||
|
||||
if len(color_urls) > 1:
|
||||
m_data = ["색상"]
|
||||
|
||||
try:
|
||||
option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv')
|
||||
except:
|
||||
option_eles = []
|
||||
for option_ele in option_eles:
|
||||
m_data.append(option_ele.ele('@tag()=input').attr('opt-nm'))
|
||||
m_data = m_data[:3]
|
||||
|
||||
option_data = {}
|
||||
if len(option_eles) > 0:
|
||||
try:
|
||||
option_eles[0].click()
|
||||
except:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
tab.wait(0.5)
|
||||
option1_eles = option_eles[0].eles('@tag()=li')
|
||||
for option1_ele in option1_eles:
|
||||
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
|
||||
for option2_name, items in option_data[option1_ele.text].items():
|
||||
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
|
||||
|
||||
if len(color_urls) > 1:
|
||||
for color_url in color_urls:
|
||||
color_images = main_images
|
||||
if color_url != url:
|
||||
tab.get(color_url)
|
||||
title = tab.ele('#goodDtlTitle').text
|
||||
color_images = get_image_urls()
|
||||
|
||||
godNo = color_url.split('/')[4]
|
||||
try:
|
||||
option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied')
|
||||
except:
|
||||
option_eles = []
|
||||
option_data = {}
|
||||
if len(option_eles) > 0:
|
||||
option1_eles = option_eles[0].eles('xpath=div/ul/li')
|
||||
for option1_ele in option1_eles:
|
||||
option_data[option1_ele.text] = get_options(godNo, option1_ele.text)
|
||||
for option2_name, items in option_data[option1_ele.text].items():
|
||||
option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name)
|
||||
|
||||
try:
|
||||
color_name = title.split('-')[1]
|
||||
except:
|
||||
color_name = tab.ele('.buy-txts').text
|
||||
|
||||
price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', ''))
|
||||
try:
|
||||
old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', ''))
|
||||
except:
|
||||
old_price = price
|
||||
|
||||
if len(option_data) > 0:
|
||||
for option1_name, items1 in option_data.items():
|
||||
if len(items1) > 0:
|
||||
for option2_name, items2 in items1.items():
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name, option1_name, option2_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name, option1_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': color_url,
|
||||
'items': [color_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': color_images
|
||||
})
|
||||
else:
|
||||
for option1_name, items1 in option_data.items():
|
||||
if len(items1) > 0:
|
||||
for option2_name, items2 in items1.items():
|
||||
if len(items2) > 0:
|
||||
for option3_name, items3 in items2.items():
|
||||
attr_items.append({
|
||||
'url': url,
|
||||
'items': [option1_name, option2_name, option3_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': []
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': url,
|
||||
'items': [option1_name, option2_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': []
|
||||
})
|
||||
else:
|
||||
attr_items.append({
|
||||
'url': url,
|
||||
'items': [option1_name],
|
||||
'price': price,
|
||||
'old_price': old_price,
|
||||
'images': []
|
||||
})
|
||||
|
||||
if len(attr_items) == 0:
|
||||
try:
|
||||
tab.ele('.gods-option').ele('text=사이즈').click()
|
||||
size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li')
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = ['사이즈']
|
||||
goods_info.images = main_images
|
||||
for size_ele in size_eles:
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
attr_items=[size_ele.text],
|
||||
price=price,
|
||||
old_price=old_price,
|
||||
images=[],
|
||||
))
|
||||
except:
|
||||
goods_info.attr = 'S'
|
||||
goods_info.attr_items = []
|
||||
goods_info.images = main_images
|
||||
goods_info.p_lists = []
|
||||
else:
|
||||
goods_info.attr = 'M'
|
||||
goods_info.attr_items = m_data
|
||||
for attr_item in attr_items:
|
||||
if len(attr_item['items']) != len(m_data):
|
||||
attr_item['items'] += m_data[len(attr_item['items']):]
|
||||
goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP(
|
||||
url=attr_item['url'],
|
||||
attr_items=attr_item['items'],
|
||||
price=attr_item['price'],
|
||||
old_price=attr_item['old_price'],
|
||||
images=attr_item['images'],
|
||||
))
|
||||
for image_url in attr_item['images']:
|
||||
if image_url not in main_images:
|
||||
main_images.append(image_url)
|
||||
goods_info.images = main_images
|
||||
goods_info.p_urls = color_urls
|
||||
except Exception as e:
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
raise Exception(e)
|
||||
try:
|
||||
tab.close()
|
||||
except:
|
||||
pass
|
||||
return goods_info
|
||||
|
||||
def get_home(self):
|
||||
self.tab.get('https://www.ssfshop.com/main')
|
||||
li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li')
|
||||
categorys = []
|
||||
for li_ele in li_eles:
|
||||
a_ele = li_ele.ele('@tag()=a')
|
||||
categorys.append(f"{a_ele.attr('href')}#{a_ele.text}")
|
||||
files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys)
|
||||
Reference in New Issue
Block a user