Cerys/spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

优化项目结构,添加注释

Browse Source
This commit is contained in:
Cerys
2026-04-14 15:00:43 +08:00
parent 9673d14a7b
commit 5d8b746817
7 changed files with 46 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
core/gether.py Normal file
View File

@@ -0,0 +1,126 @@
import os
import traceback
import concurrent.futures
from utils import files, messages, clash, formats
import lib
from core import Config, types, goods
def get_category_urls():
category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt"
# if os.path.exists(category_urls_path):
# return
category_urls = lib.Server.get_category_urls()
files.save_list(category_urls_path, category_urls)
def get_goods_urls(categorys_len: int = 3, home_mode: bool = False):
goods_urls_path = f"{lib.Server.__urls_folder__}/goods.txt"
category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt"
if home_mode:
goods_urls_path = goods_urls_path.replace('.txt', '(home).txt')
category_urls_path = category_urls_path.replace('.txt', '(home).txt')
category_index = {}
category_urls = files.load_lines(category_urls_path)
if os.path.exists(goods_urls_path):
old_goods_urls = files.load_lines(goods_urls_path)
for old_goods_url in old_goods_urls:
category = old_goods_url.split('#')[-1]
category_index[category] = None
index = 0
max_len = len(category_urls)
for category_url in category_urls:
category = category_url.split('#')[-1]
category_url = '#'.join(category_url.split('#')[:-1])
index += 1
if 'https' not in category_url:
continue
if Config.skip_category and home_mode == False and len(category.split('/')) < categorys_len:
continue
if category in category_index:
continue
try:
goods_urls = lib.Server.get_goods_urls(category_url)
goods_category_urls = []
for goods_url in goods_urls:
goods_category_urls.append(f"{goods_url}#{category}")
files.add_list(goods_urls_path, goods_category_urls)
except Exception as e:
files.add_text(f"{lib.Server.__errors_folder__}/category_errors.log", f"{category_url}:\n[{e}] {traceback.format_exc()}")
messages.sendError(str(e))
messages.sendInfo(f"{index}/{max_len} {len(goods_urls)}条")
if home_mode == False:
formats.de_repeat_urls(goods_urls_path)
def get_goods_info():
p_urls = {}
def workers(goods_urls, index):
with concurrent.futures.ThreadPoolExecutor(max_workers=lib.Server.worker_num) as executor:
futures = []
for goods_url in goods_urls:
if goods_url in p_urls:
continue
future = executor.submit(lib.Server.get_goods_info, goods_url)
futures.append(future)
results: list[types.GoodsInfo] = []
for future_index, future in enumerate(futures):
index += 1
try:
result = future.result() # 阻塞直到该任务完成
except Exception as e:
try:
files.add_text(f"{lib.Server.__errors_folder__}/goods_errors.log", f"{goods_urls[future_index]}:\n[{e}] {traceback.format_exc()}")
except:
pass
messages.sendWarn(f"{goods_urls[future_index]} 出错")
continue
results.append(result)
messages.sendInfo(f"{index}/{max_len}")
for goods_info in results:
try:
GoodsModel.add_goods(goods_info)
for p_url in goods_info.p_urls:
p_urls[p_url] = None
except Exception as e:
messages.sendWarn(f"{e}")
return index
goods_urls = files.load_lines(f"{lib.Server.__urls_folder__}/goods.txt")
messages.sendInfo('读取url完成')
messages.sendInfo('读取上次存档')
GoodsModel = goods.GoodsModel(lib.Server.__database_path__)
excel_index = GoodsModel.select_goods_to_dict()
messages.sendInfo('读取完成')
de_save_url = []
for goods_url in goods_urls:
if goods_url in excel_index:
continue
if 'http' not in goods_url:
continue
de_save_url.append(goods_url)
goods_urls = de_save_url
del excel_index
max_len = len(goods_urls)
index = 1
for i in range(0, len(goods_urls), lib.Server.bitch):
index = workers(goods_urls=goods_urls[i:i+lib.Server.bitch], index=index)
if lib.Server.switch_clash:
try:
clash.change_proxy()
except:
pass
if lib.Server.reflush_browsers:
if lib.Server.worker_num > 1:
lib.Server.reflush_browser()
messages.sendInfo("已保存")

383
core/goods.py Normal file
View File

@@ -0,0 +1,383 @@
#############################################################################
# Author: Cerys
# Update: 2026-02-06
#############################################################################
import os
import orjson
from core import excels, types
from utils import db, messages, files, formats
class GoodsModel:
def __init__(self, db_path: str):
"""
商品模型
:param db_path: 数据库文件路径(不存在则自动创建并初始化表)
"""
self.__source_folder__ = "sources"
create_table = False
if os.path.exists(db_path) == False:
create_table = True
self.Db = db.Db(db_path)
if create_table:
self.__create_table__()
def __create_table__(self):
table_sql = '''
PRAGMA foreign_keys = false;
DROP TABLE IF EXISTS "goods";
CREATE TABLE "goods" (
"spu" TEXT NOT NULL,
"title" TEXT NOT NULL,
"brand" TEXT,
"attr" TEXT NOT NULL,
"desc" TEXT,
"item1" TEXT,
"item2" TEXT,
"item3" TEXT,
"price" NUMBER NOT NULL,
"old_price" NUMBER NOT NULL,
"images" blob NOT NULL,
"url" TEXT NOT NULL,
"other_data" blob DEFAULT "{}",
PRIMARY KEY ("spu")
);
DROP TABLE IF EXISTS "goods_son";
CREATE TABLE "goods_son" (
"spu" TEXT NOT NULL,
"item1" TEXT,
"item2" TEXT,
"item3" TEXT,
"price" NUMBER NOT NULL,
"old_price" NUMBER NOT NULL,
"images" blob NOT NULL,
"url" TEXT,
"other_data" blob DEFAULT "{}"
);
CREATE INDEX "search_spu"
ON "goods_son" (
"spu" ASC
);
PRAGMA foreign_keys = true;
'''
self.Db.execute_script(table_sql)
def add_goods(self, goods_info: types.GoodsInfo):
"""
添加商品
:param goods_info: GoodsInfo 类型数据
"""
main_info, son_infos = goods_info.to_db_data()
if main_info['attr'] == 'M' and len(son_infos) == 0:
raise Exception(f"变体丢失 {main_info}")
if len(main_info['images']) == 0:
raise Exception('无图片')
self.Db.table('goods').insert(main_info, True)
self.Db.table('goods_son').insert_all(son_infos)
def is_repeat(self, url):
"""
判断是否需要合并
:param url: 商品链接
"""
count = self.Db.table('goods_son').where('url', url).count()
if count > 1:
return True
return False
def select_goods_to_lists(self) -> list[dict]:
"""获取商品列表（包含变体）"""
messages.sendInfo('开始查询')
db_data = self.Db.table('goods').join('goods_son', 'goods.spu = goods_son.spu').select(
'''
goods.spu,
goods.title,
goods.brand,
goods.attr,
goods.desc,
goods.item1 as main_item1,
goods.item2 as main_item2,
goods.item3 as main_item3,
goods.price,
goods.old_price,
goods.images,
goods.url,
goods.other_data,
goods_son.item1 as son_item1,
goods_son.item2 as son_item2,
goods_son.item3 as son_item3,
goods_son.price as son_price,
goods_son.old_price as son_old_price,
goods_son.images as son_images,
goods_son.url as son_url,
goods_son.other_data as son_other_data
'''
)
messages.sendInfo('查询完成')
return db_data
def select_goods_to_dict(self, key: str = 'url') -> dict:
"""
获取商品列表并转换为字典格式
:param key: 字典的键
"""
db_data = self.select_goods_to_lists()
dict_data = {}
for item in db_data:
if item[key] in dict_data:
dict_data[item[key]]['son_lists'].append({
'attr_items': [item['son_item1'], item['son_item2'], item['son_item3']],
'price': item['son_price'],
'old_price': item['son_old_price'],
'images': orjson.loads(item['son_images']),
'url': item['son_url'],
'son_other_data': orjson.loads(item['son_other_data'])
})
else:
dict_data[item[key]] = {
'spu': item['spu'],
'title': item['title'],
'brand': item['brand'],
'desc': item['desc'],
'attr': item['attr'],
'attr_items': [item['main_item1'], item['main_item2'], item['main_item3']],
'price': item['price'],
'old_price': item['old_price'],
'images': orjson.loads(item['images']),
'url': item['url'],
'other_data': orjson.loads(item['other_data'])
}
if item['attr'] == 'M':
if item['son_price'] == None:
continue
dict_data[item[key]]['son_lists'] = [
{
'attr_items': [item['son_item1'], item['son_item2'], item['son_item3']],
'price': item['son_price'],
'old_price': item['son_old_price'],
'images': orjson.loads(item['son_images']),
'url': item['son_url'],
'son_other_data': orjson.loads(item['son_other_data'])
}
]
return dict_data
def select_goods_to_goodsinfo(self) -> list[types.GoodsInfo]:
"""获取商品列表并转换为GoodsInfo对象列表"""
db_data = self.select_goods_to_lists()
index_data = {}
for item in db_data:
if item['spu'] in index_data:
index_data[item['spu']].p_lists.append(
types.GoodsInfo.GoodsInfoP(
attr_items = [item['son_item1'], item['son_item2'], item['son_item3']],
price = item['son_price'],
old_price = item['son_old_price'],
images = orjson.loads(item['son_images']),
url = item['son_url'],
other_data = orjson.loads(item['son_other_data'])
)
)
else:
try:
index_data[item['spu']] = types.GoodsInfo(
spu = item['spu'],
title = item['title'],
brand = item['brand'],
attr = item['attr'],
desc = item['desc'],
attr_items = [item['main_item1'], item['main_item2'], item['main_item3']],
price = item['price'],
old_price = item['old_price'],
images = orjson.loads(item['images']),
url = item['url'],
other_data = orjson.loads(item['other_data'])
)
if item['attr'] == 'M':
index_data[item['spu']].p_lists.append(
types.GoodsInfo.GoodsInfoP(
attr_items = [item['son_item1'], item['son_item2'], item['son_item3']],
price = item['son_price'],
old_price = item['son_old_price'],
images = orjson.loads(item['son_images']),
url = item['son_url'],
other_data = orjson.loads(item['son_other_data'])
)
)
except Exception as e:
messages.sendError(e)
return list(index_data.values())
def to_amazon_xlsx(self, goods_url_path: str, save_path: str):
"""
整数据库导出为亚马逊导入表格
:param save_path: 保存路径
:param goods_url_path: 未去重商品链接路径
"""
goods_data = self.select_goods_to_dict()
goods_urls = files.load_lines(goods_url_path)
WookBook = excels.WorkBook(
f'{self.__source_folder__}/amazon商品模板.xlsx',
save_path
)
for goods_url in goods_urls:
key = '#'.join(goods_url.split('#')[:-1])
if key not in goods_data:
continue
spu = formats.url_to_spu(key)
category = goods_url.split('#')[-1]
WookBook.add_row_of_list([
goods_data[key]['title'],
spu,
goods_data[key]['other_data']['popover'],
goods_data[key]['other_data']['popover_num'],
0,
goods_data[key]['price'],
goods_data[key]['old_price'],
orjson.dumps(goods_data[key]['images']),
goods_data[key]['url'],
category
])
# for goods_info in goods_info_data:
# WookBook.add_row_of_list([
# goods_info.title,
# goods_info.price,
# goods_info.images[0],
# goods_info.other_data['popover'],
# goods_info.other_data['popover_num'],
# goods_info.url,
# orjson.dumps(goods_info.images)
# ])
WookBook.save()
WookBook.close()
def to_shopyy_xlsx(self, save_path: str, save_bitch: int = 150000, shift_urls_path: str = ''):
"""
整数据库导出为Shopyy导入表格
:param save_path: 保存路径
:param repeat: 是否合并商品
:param save_bitch: 超过多少行时自动分割
"""
def get_wookbook(bitch_index: int):
return excels.WorkBook(
f'{self.__source_folder__}/shopyy商品模板.xlsx',
save_path.replace('.xlsx', f'_{bitch_index}.xlsx')
)
def get_shift_url():
shift_urls = {}
if shift_urls_path:
urls = files.load_lines(shift_urls_path)
for url in urls:
if url == '':
continue
shift_urls[url] = None
return shift_urls
goods_info_data = self.select_goods_to_goodsinfo()
bitch = 1
shift_urls = get_shift_url()
WookBook = get_wookbook(bitch)
count = len(goods_info_data)
for index, goods_info in enumerate(goods_info_data):
messages.sendInfo(f"{index}/{count}")
if shift_urls_path and goods_info.url not in shift_urls:
continue
rows_data = goods_info.to_shopyy_row_data()
for row_data in rows_data:
WookBook.add_row_of_list(row_data)
if WookBook.start_row > save_bitch:
WookBook.save()
WookBook.close()
bitch += 1
WookBook = get_wookbook(bitch)
WookBook.save()
WookBook.close()
def to_woo_csv(self, goods_url_path: str, save_path: str):
"""
整数据库导出为woo商品CSV表格
:param goods_url_path: 未去重链接txt路径
:param save_path: 保存路径
"""
goods_urls = files.load_lines(goods_url_path)
category_data = {}
for goods_url in goods_urls:
category_name = goods_url.split('#')[-1].replace('/', '>')
goods_url = '#'.join(goods_url.split('#')[:-1])
category_data[goods_url] = category_name
goods_info_data = self.select_goods_to_goodsinfo()
bitch = 0
save_data = []
for goods_info in goods_info_data:
rows_data = goods_info.to_woo_row_data(category_data[goods_info.url])
save_data += rows_data
if len(save_data) > 10000:
excels.save_lists_to_csv(save_data, save_path.replace('.csv', f"{bitch}.csv"), r'sources\woo商品模板.csv')
save_data = []
bitch += 1
excels.save_lists_to_csv(save_data, save_path.replace('.csv', f"{bitch}.csv"), r'sources\woo商品模板.csv')
def import_shopyy_xlsx(self, excel_path: str):
"""
导入shopyy表格为数据库数据
:param excel_path: 表格路径
"""
shopyy_data = excels.read_excel_to_dict(excel_path)
for url, goods_info in shopyy_data.items():
self.add_goods(goods_info)
def splice_db(self, db_folder: str):
"""
拼接指定文件夹下所有数据库文件
:param db_folder: 数据库文件夹路径
"""
db_lists = os.listdir(db_folder)
db_index = self.select_goods_to_dict()
for db_name in db_lists:
splice_db = GoodsModel(f"{db_folder}/{db_name}")
goods_lists = splice_db.select_goods_to_goodsinfo()
for goods_info in goods_lists:
if goods_info.url in db_index:
continue
try:
self.add_goods(goods_info)
except Exception as e:
messages.sendError(f"{e} {goods_info.spu}")
db_index[goods_info.url] = None
messages.sendInfo(db_name)

370
core/shopyy.py Normal file
View File

@@ -0,0 +1,370 @@
import requests
from core import excels
from utils import browsers, files, formats
class Shopyy:
def __init__(self, save_folder):
self.save_folder = save_folder
self.__album_template_path__ = 'sources/专辑模板.xlsx'
self.__navigation_template_path__ = 'sources/导航模板.xlsx'
# 导航
def generate_navigation(self, category_path):
index = {}
category_urls = files.load_lines(category_path)
WorkBook = excels.WorkBook(
template=self.__navigation_template_path__,
save_path=f"{self.save_folder}/导航表格.xlsx"
)
for category_url in category_urls:
category = category_url.split('#')[-1]
if category in index:
continue
category_lists = category.split('/')
category_lists += ['', '']
row_data = [
'顶部导航菜单',
category_lists[0],
category_lists[1],
category_lists[2],
'collection',
category
]
WorkBook.add_row_of_list(row_data)
index[category] = None
WorkBook.save()
return self
# 专辑
def generate_album(self, goods_url_path):
def open_workbook(bitch_index) -> excels.WorkBook:
WorkBook = excels.WorkBook(
template=self.__album_template_path__,
save_path=f"{self.save_folder}/专辑表格({bitch_index}).xlsx"
)
return WorkBook
album_data = {}
count = 0
bitch_index = 1
WorkBook = open_workbook(bitch_index)
goods_urls = files.load_lines(goods_url_path)
for goods_url in goods_urls:
if goods_url == '':
continue
goods_url_split = goods_url.split('#')
goods_categorys = goods_url_split[-1].split('/')
goods_url = '#'.join(goods_url_split[:-1])
goods_url = goods_url.split('?')[0]
for category_index, goods_category in enumerate(goods_categorys):
goods_category = '/'.join(goods_categorys[:category_index + 1])
if goods_category in album_data:
mode = ''
sort = ''
else:
mode = 'add'
sort = 'collections_position'
album_data[goods_category] = []
row_data = [
'',
goods_category,
mode,
sort,
'',
formats.url_to_spu(goods_url)
]
album_data[goods_category].append(row_data)
count += 1
save_bitch = count/3+1
for goods_category, row_datas in album_data.items():
for row_data in row_datas:
WorkBook.add_row_of_list(row_data)
if WorkBook.start_row > save_bitch and bitch_index <= 2:
WorkBook.save()
WorkBook.close()
bitch_index += 1
WorkBook = open_workbook(bitch_index)
WorkBook.save()
WorkBook.close()
return self
class ShopyyVerify:
def __init__(self, url):
self.url = url
browser = browsers.getBrowsers()
self.tab = browser.get_tab(0)
self.login()
def login(self):
self.tab.get(f"{self.url}/admin/login")
print('等待登录')
while True:
token = self.tab.local_storage('CARDADMIN_TOKEN')
if token:
print('登录成功')
self.token = token.replace('"', '')
break
self.tab.wait(1)
def get_zero_collections(self, delete: bool = False):
def get_collections(page):
headers = {
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'priority': 'u=1, i',
# 'referer': 'https://cisalfasport.zenshop.cn/admin/collections?page=1&pagesize=50',
'sec-ch-ua': '"Not(A:Brand";v="8", "Chromium";v="144", "Microsoft Edge";v="144"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'token': self.token,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 Edg/144.0.0.0',
# 'cookie': '__stripe_mid=89b6ec3f-bbac-483d-b8ab-9938859a0b340b72d8; _aiihbe_vid=18FC7E3E-1CA7-FDC2-C52D-37A4BB0E1DF0; _aiihbe_cvid=75CFD325-E408-2C25-C126-07A349ED46C5; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1770185710%2C%22expire_time%22%3A1772777710%7D%5D; landing_page=aHR0cHM6Ly9jaXNhbGZhc3BvcnQuemVuc2hvcC5jbi9wcm9kdWN0cy90b3VyLWRlLWZyYW5jZS1qcj9wcmV2aWV3PTE%3D; first_http_referer=https%3A%2F%2Fcisalfasport.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1770185710; PHPSESSID=6329e9a44018771ce40064bc50fbe189; _aiihbe_vs=36B35E5C-874E-4D6C-8945-F5CF6F323F99; theme_id=411278; _AIIHBE_ua=Mozilla%252F5.0%2520(Windows%2520NT%252010.0%253B%2520Win64%253B%2520x64)%2520AppleWebKit%252F537.36%2520(KHTML%252C%2520like%2520Gecko)%2520Chrome%252F144.0.0.0%2520Safari%252F537.36%2520Edg%252F144.0.0.0; _AIIHBE_pu=https%253A%252F%252Fcisalfasport.zenshop.cn%252Fadmin%252Ftheme; _AIIHBE_tz=Etc%2FGMT-8; _AIIHBE_ss=1920X1080; _AIIHBE_lang=zh-CN; _AIIHBE_vs=1874X930; _AIIHBE_dt=2026-02-06%2016%3A30%3A44; acw_tc=a3b54ee217704477374193466e23fb58e387e54f5b84504527a4beab6b; cdn_sec_tc=a3b54ee217704477374193466e23fb58e387e54f5b84504527a4beab6b; __stripe_sid=49bcd8c1-4ae4-46ba-8e95-a29205606cb2980c18; visit_token=4be69765c90de228978e9738878367fbkj083fsy; _aiihbe_ka=A298A3E1-379E-4DA0-2B28-A5961B741FA9; is_theme_edit=1',
}
params = {
'page': str(page),
'pagesize': '600',
}
response = requests.get(f'{self.url}/api/collections', params=params, headers=headers)
response_data = response.json()
if response_data['code'] != 0:
raise Exception(response_data['msg'])
return response_data['data']
def delete_collections(ids: list):
headers = {
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'cache-control': 'no-cache',
'content-type': 'application/json',
'origin': 'https://peekcloppenburgat.zenshop.cn',
'pragma': 'no-cache',
'priority': 'u=1, i',
# 'referer': 'https://peekcloppenburgat.zenshop.cn/admin/collections?page=1&pagesize=600',
'sec-ch-ua': '"Not:A-Brand";v="99", "Microsoft Edge";v="145", "Chromium";v="145"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'token': self.token,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0',
# 'cookie': '__stripe_mid=49d8f368-8d82-4b14-9c42-63e0d33542ef6372ca; _bzcece_vid=E823CDA9-7EDE-3617-71BB-80F879AD878E; _bzcece_cvid=36F627EC-095F-9323-5047-43325149C808; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1773370701%2C%22expire_time%22%3A1775962701%7D%5D; landing_page=aHR0cHM6Ly9wZWVrY2xvcHBlbmJ1cmdhdC56ZW5zaG9wLmNuL3Byb2R1Y3RzL3Zlcm8tbW9kYS13aWRlLWxlZy1qZWFucy1hdXMtYmF1bXdvbGwtbWl4LW1vZGVsbC10ZXNzYS1pbi1oZWxsZ3JhdT9wcmV2aWV3PTE%3D; first_http_referer=https%3A%2F%2Fpeekcloppenburgat.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1773370701; PHPSESSID=fba65fe62a66071f174801bb432b942c; theme_id=418468; __stripe_sid=323c1341-4108-4c33-b331-ea6b782f9a2349679f; visit_token=f1d5170d234867f2ca490821f454239bmwnlrqr7; _bzcece_vs=5F572333-803E-D038-11EC-6CFCBA6F32D3; _bzcece_ka=B2503B21-4B44-A5A0-F261-71EC86D52A6E; is_theme_edit=1; acw_tc=9b66b49a17734819180038754ede11478ee763b95bb1c9acff317f521f; cdn_sec_tc=9b66b49a17734819180038754ede11478ee763b95bb1c9acff317f521f',
}
json_data = {
'ids': ids,
}
response = requests.post(
f'{self.url}/api/collections/batchdelete',
headers=headers,
json=json_data,
)
print(response.json())
print('开始检查专辑数量')
collection_items = []
data = get_collections(1)
page_total = data['paginate']['pageTotal']
collection_items += data['collections']
print(f"1/{page_total}")
if page_total > 1:
for page in range(2, page_total+1):
data = get_collections(page)
collection_items += data['collections']
print(f"{page}/{page_total}")
lost_collection = []
zero_ids = []
for collection_item in collection_items:
if collection_item['collection_products_count'] == 0:
lost_collection.append(collection_item['title'])
print(f"{collection_item['title']} 数量为0")
zero_ids.append(collection_item['id'])
if delete:
delete_collections(zero_ids)
if len(lost_collection) > 0:
with open('商品数量为0专辑.txt', 'w', encoding='utf-8') as f:
f.write('\n'.join(lost_collection))
return self
def get_error_images(self, delete: bool = False):
def get_errors(page):
headers = {
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'cache-control': 'no-cache',
'dnt': '1',
'pragma': 'no-cache',
'priority': 'u=1, i',
# 'referer': 'https://vanuffelenmode.zenshop.cn/admin/resources',
'sec-ch-ua': '"Not:A-Brand";v="99", "Microsoft Edge";v="145", "Chromium";v="145"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'token': self.token,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0',
# 'cookie': 'acw_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; cdn_sec_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; __stripe_mid=cc09d025-360b-466e-bef4-ba92d045fdc94448e2; __stripe_sid=511d0972-d39d-4aa1-a031-5258db651ad1d4124e; visit_token=9b8172c220d5be292045835470d38951px0lh404',
}
params = {
'file_name': '',
'page_type': 'fail',
'page': str(page),
'pagesize': '600',
}
response = requests.get(f'{self.url}/api/resources', params=params, headers=headers)
response_data = response.json()
if response_data['code'] != 0:
raise Exception(response_data['msg'])
return response_data['data']
def post_retry(ids: list):
headers = {
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'content-type': 'application/json',
# 'origin': 'https://ssfshop.zenshop.cn',
'priority': 'u=1, i',
# 'referer': 'https://ssfshop.zenshop.cn/admin/resources',
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'token': self.token,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
# 'cookie': '__stripe_mid=a8059a05-4d7e-476d-b376-9e54f222efbe3c23c7; _bzgigi_vid=85F94DA2-7BD9-525E-E4A5-8D686E51B2C6; _bzgigi_cvid=EEFE28A2-FDC4-DC54-E76A-2BC1B2929243; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1775195861%2C%22expire_time%22%3A1777787861%7D%5D; landing_page=aHR0cHM6Ly9zc2ZzaG9wLnplbnNob3AuY24vcHJvZHVjdHMvJUU4JTg3JUFBJUU3JTk0JUIxJUU1JThDJUJBLWwtMTklRTUlOEYlQjclRTMlODIlQjUlRTMlODIlQTQlRTMlODIlQkElRTMlODElQkUlRTMlODElQTclRTUlQjElOTUlRTklOTYlOEItJUUzJTgyJUFCJUUzJTgyJUJGJUUzJTgzJUFEJUUzJTgyJUIwJUU2JThFJUIyJUU4JUJDJTg5LSVFMyU4MiVCQiVFMyU4MyU4MyVFMyU4MyU4OCVFMyU4MiVBMiVFMyU4MyU4MyVFMyU4MyU5NyVFNSVBRiVCRSVFNSVCRiU5Qy0lRTMlODIlQkIlRTMlODMlQUMlRTMlODMlQTIlRTMlODMlOEIlRTMlODMlQkMtJUU5JTgwJTlBJUU1JThCJUE0LSVFMyU4MyU4OCVFMyU4MyVBQSVFMyU4MiVBMiVFMyU4MiVCQiVFMyU4MyU4MCVFMyU4MyU5NiVFMyU4MyVBQiVFMyU4MiVBRiVFMyU4MyVBRCVFMyU4MiVCOSVFMyU4MyU4NiVFMyU4MyVCQyVFMyU4MyVBOSVFMyU4MyVCQyVFMyU4MyU4OS0lRTMlODIlQjglRTMlODMlQTMlRTMlODIlQjElRTMlODMlODMlRTMlODMlODhfOGNhZjQ4Y2U%2FcHJldmlldz0x; first_http_referer=https%3A%2F%2Fssfshop.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1775195861; visit_token=41be25876fa245bb99018d68f7338d89y49l7utp; _bzgigi_vs=E5BE5924-EEF9-E277-6AFC-5F70BB548D0F; PHPSESSID=b08ff5f29b9469e1e92740efef87c09e; _BZGIGI_ua=Mozilla%252F5.0%2520(Windows%2520NT%252010.0%253B%2520Win64%253B%2520x64)%2520AppleWebKit%252F537.36%2520(KHTML%252C%2520like%2520Gecko)%2520Chrome%252F146.0.0.0%2520Safari%252F537.36%2520Edg%252F146.0.0.0; _BZGIGI_pu=https%253A%252F%252Fssfshop.zenshop.cn%252Fadmin%252Fproducts; _BZGIGI_tz=Etc%2FGMT-8; _BZGIGI_ss=2560X1440; _BZGIGI_lang=zh-CN; _BZGIGI_vs=2514X1316; _BZGIGI_dt=2026-04-07%2014%3A48%3A20; theme_id=429104; is_theme_edit=1; acw_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; cdn_sec_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; __stripe_sid=a0f6f277-4a95-4d43-bd6f-c38f108e17f52fc752',
}
response = requests.post(f'{self.url}/api/resources/retry', headers=headers, json={'ids':ids})
print(response.json())
def delete_images(ids: list):
headers = {
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'content-type': 'application/json',
# 'origin': 'https://ssfshop.zenshop.cn',
'priority': 'u=1, i',
# 'referer': 'https://ssfshop.zenshop.cn/admin/resources',
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'token': self.token,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
# 'cookie': '__stripe_mid=a8059a05-4d7e-476d-b376-9e54f222efbe3c23c7; _bzgigi_vid=85F94DA2-7BD9-525E-E4A5-8D686E51B2C6; _bzgigi_cvid=EEFE28A2-FDC4-DC54-E76A-2BC1B2929243; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1775195861%2C%22expire_time%22%3A1777787861%7D%5D; landing_page=aHR0cHM6Ly9zc2ZzaG9wLnplbnNob3AuY24vcHJvZHVjdHMvJUU4JTg3JUFBJUU3JTk0JUIxJUU1JThDJUJBLWwtMTklRTUlOEYlQjclRTMlODIlQjUlRTMlODIlQTQlRTMlODIlQkElRTMlODElQkUlRTMlODElQTclRTUlQjElOTUlRTklOTYlOEItJUUzJTgyJUFCJUUzJTgyJUJGJUUzJTgzJUFEJUUzJTgyJUIwJUU2JThFJUIyJUU4JUJDJTg5LSVFMyU4MiVCQiVFMyU4MyU4MyVFMyU4MyU4OCVFMyU4MiVBMiVFMyU4MyU4MyVFMyU4MyU5NyVFNSVBRiVCRSVFNSVCRiU5Qy0lRTMlODIlQkIlRTMlODMlQUMlRTMlODMlQTIlRTMlODMlOEIlRTMlODMlQkMtJUU5JTgwJTlBJUU1JThCJUE0LSVFMyU4MyU4OCVFMyU4MyVBQSVFMyU4MiVBMiVFMyU4MiVCQiVFMyU4MyU4MCVFMyU4MyU5NiVFMyU4MyVBQiVFMyU4MiVBRiVFMyU4MyVBRCVFMyU4MiVCOSVFMyU4MyU4NiVFMyU4MyVCQyVFMyU4MyVBOSVFMyU4MyVCQyVFMyU4MyU4OS0lRTMlODIlQjglRTMlODMlQTMlRTMlODIlQjElRTMlODMlODMlRTMlODMlODhfOGNhZjQ4Y2U%2FcHJldmlldz0x; first_http_referer=https%3A%2F%2Fssfshop.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1775195861; visit_token=41be25876fa245bb99018d68f7338d89y49l7utp; _bzgigi_vs=E5BE5924-EEF9-E277-6AFC-5F70BB548D0F; PHPSESSID=b08ff5f29b9469e1e92740efef87c09e; _BZGIGI_ua=Mozilla%252F5.0%2520(Windows%2520NT%252010.0%253B%2520Win64%253B%2520x64)%2520AppleWebKit%252F537.36%2520(KHTML%252C%2520like%2520Gecko)%2520Chrome%252F146.0.0.0%2520Safari%252F537.36%2520Edg%252F146.0.0.0; _BZGIGI_pu=https%253A%252F%252Fssfshop.zenshop.cn%252Fadmin%252Fproducts; _BZGIGI_tz=Etc%2FGMT-8; _BZGIGI_ss=2560X1440; _BZGIGI_lang=zh-CN; _BZGIGI_vs=2514X1316; _BZGIGI_dt=2026-04-07%2014%3A48%3A20; theme_id=429104; is_theme_edit=1; acw_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; cdn_sec_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; __stripe_sid=60ef73cf-e5ed-483a-a342-9e581a0387d64df267',
}
json_data = {
'ids': ids,
'delete_product': 0
}
response = requests.delete('https://ssfshop.zenshop.cn/api/resources/batchdelete', headers=headers, json=json_data)
print(response.json())
print('开始检查失败图片')
image_items = []
data = get_errors(1)
page_total = data['paginate']['pageTotal']
image_items += data['list']
print(f"1/{page_total}")
if page_total > 1:
for page in range(2, page_total+1):
ids = []
data = get_errors(1)
for item in data['list']:
ids.append(item['id'])
if delete:
delete_images(ids)
else:
post_retry(ids)
page_total = data['paginate']['pageTotal']
print(f"{page}/{page_total}")
# lost_image_urls = []
# for image_item in image_items:
# lost_image_urls.append(image_item['remote_url'])
# if len(lost_image_urls) > 0:
# files.save_list('采集失败图片链接.txt', lost_image_urls)
# print(f"共有: {len(lost_image_urls)} 条失败链接")
def get_all_images(self):
def get_images(page):
headers = {
'accept': 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
'cache-control': 'no-cache',
'dnt': '1',
'pragma': 'no-cache',
'priority': 'u=1, i',
# 'referer': 'https://vanuffelenmode.zenshop.cn/admin/resources',
'sec-ch-ua': '"Not:A-Brand";v="99", "Microsoft Edge";v="145", "Chromium";v="145"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'token': self.token,
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0',
# 'cookie': 'acw_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; cdn_sec_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; __stripe_mid=cc09d025-360b-466e-bef4-ba92d045fdc94448e2; __stripe_sid=511d0972-d39d-4aa1-a031-5258db651ad1d4124e; visit_token=9b8172c220d5be292045835470d38951px0lh404',
}
params = {
'file_name': '',
'page_type': 'all',
'page': str(page),
'pagesize': '600',
}
response = requests.get(f'{self.url}/api/resources', params=params, headers=headers)
response_data = response.json()
if response_data['code'] != 0:
raise Exception(response_data['msg'])
return response_data['data']
print('开始获取图片url')
image_items = []
data = get_images(1)
page_total = data['paginate']['pageTotal']
image_items += data['list']
print(f"1/{page_total}")
if page_total > 1:
for page in range(2, page_total+1):
data = get_images(page)
image_items += data['list']
print(f"{page}/{page_total}")
image_urls = []
for image_item in image_items:
image_urls.append(image_item['remote_url'])
if len(image_urls) > 0:
files.save_list('图片链接.txt', image_urls)
print(f"共有: {len(image_urls)} 张图片")