diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..058a64e --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/data/* +/database/* +/release/ +/test/ +/.vscode/ +/main.build/ +/main.dist/ \ No newline at end of file diff --git a/build b/build new file mode 100644 index 0000000..c9d3d61 --- /dev/null +++ b/build @@ -0,0 +1 @@ +nuitka --mingw64 --standalone main.py \ No newline at end of file diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000..65cdb26 --- /dev/null +++ b/core/__init__.py @@ -0,0 +1,3 @@ +from core import types + +Config = types.Config() diff --git a/core/__pycache__/__init__.cpython-312.pyc b/core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..9e31eae Binary files /dev/null and b/core/__pycache__/__init__.cpython-312.pyc differ diff --git a/core/__pycache__/browsers.cpython-310.pyc b/core/__pycache__/browsers.cpython-310.pyc new file mode 100644 index 0000000..e3e0a4a Binary files /dev/null and b/core/__pycache__/browsers.cpython-310.pyc differ diff --git a/core/__pycache__/browsers.cpython-312.pyc b/core/__pycache__/browsers.cpython-312.pyc new file mode 100644 index 0000000..e4f43e8 Binary files /dev/null and b/core/__pycache__/browsers.cpython-312.pyc differ diff --git a/core/__pycache__/excels.cpython-312.pyc b/core/__pycache__/excels.cpython-312.pyc new file mode 100644 index 0000000..d478792 Binary files /dev/null and b/core/__pycache__/excels.cpython-312.pyc differ diff --git a/core/__pycache__/messages.cpython-312.pyc b/core/__pycache__/messages.cpython-312.pyc new file mode 100644 index 0000000..45c11fd Binary files /dev/null and b/core/__pycache__/messages.cpython-312.pyc differ diff --git a/core/__pycache__/spiders.cpython-312.pyc b/core/__pycache__/spiders.cpython-312.pyc new file mode 100644 index 0000000..d4f1d32 Binary files /dev/null and b/core/__pycache__/spiders.cpython-312.pyc differ diff --git a/core/__pycache__/types.cpython-312.pyc b/core/__pycache__/types.cpython-312.pyc new file mode 100644 index 0000000..706b980 Binary files /dev/null and b/core/__pycache__/types.cpython-312.pyc differ diff --git a/core/browsers.py b/core/browsers.py new file mode 100644 index 0000000..0186775 --- /dev/null +++ b/core/browsers.py @@ -0,0 +1,25 @@ +import socket +import random +from DrissionPage import Chromium, ChromiumOptions + +def getBrowsers(chrome_zoom = 1.0, save_path: str = "") -> Chromium: + while True: + port = random.randint(9000, 65535) + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + if s.connect_ex(('127.0.0.1', port)) != 10061: + continue + break + + if save_path: + options = ChromiumOptions().set_local_port(port).set_user_data_path(save_path) + else: + options = ChromiumOptions().auto_port() + options = options.set_argument('--force-device-scale-factor', str(chrome_zoom)) + # options = options.set_user_agent('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)') + while True: + try: + browser = Chromium(options) + break + except: + pass + return browser diff --git a/core/excels.py b/core/excels.py new file mode 100644 index 0000000..b2a19e7 --- /dev/null +++ b/core/excels.py @@ -0,0 +1,84 @@ +import csv +import traceback +from openpyxl import load_workbook +from core import types +from utils import messages + +def read_excel_to_dict(excel_path: str, sheet_name: str = 'Sheet1') -> dict[str, types.GoodsInfo]: + """ + 从 Excel 文件中读取商品数据，返回 {url: GoodsInfo} 的字典。 + + :param excel_path: Excel 文件路径 + :param sheet_name: 工作表名称，默认 'Sheet1' + :return: dict{GoodsInfo.url: GoodsInfo} + """ + wb = load_workbook(excel_path, data_only=True) + ws = wb[sheet_name] + + goods_dict = {} + + # 从第3行开始读取（假设第1、2行为标题/说明） + for row in ws.iter_rows(min_row=3, values_only=True): + # 过滤空行（如果整行都为空则跳过） + if all(cell is None or str(cell).strip() == '' for cell in row): + continue + + # 转为字符串列表，None 转为空字符串 + row_data = [str(cell) if cell is not None else '' for cell in row] + + try: + goods = types.GoodsInfo.from_row_data(row_data) + if goods.url and (goods.attribute == "M" or goods.attribute == "S"): # 只有 url 非空才加入字典 + goods_dict[goods.url] = goods + elif goods.url and goods.attribute == "P": + if goods.url in goods_dict: + goods_dict[goods.url].p_lists.append(goods) + except Exception as e: + messages.sendError(f"解析行失败（跳过）: {row_data[:5]}... 错误: {e}\n{traceback.format_exc()}") + continue + wb.close() + + return goods_dict + +def save_lists_to_csv(data: list, save_path: str, template_path: str = ''): + header = [] + if template_path: + with open(template_path, 'r', encoding='utf-8-sig') as f: + template = csv.reader(f) + for row in template: + header = list(row) + data.insert(0, header) + with open(save_path, 'w', encoding='utf-8', newline="") as f: + writer = csv.writer(f) + writer.writerows(data) + +class WorkBook: + def __init__(self, template: str, save_path: str, sheet_name: str = 'Sheet1'): + wb = load_workbook(template) + ws = wb[sheet_name] + last_row = ws.max_row + if ws.cell(row=last_row, column=1).value is None and "此行导入时不可删除" in str(ws.cell(row=2, column=1).value): + start_row = last_row + 1 + else: + while ws.cell(row=last_row, column=1).value is not None or any(ws.cell(row=last_row, column=c).value for c in range(2, 30)): + last_row += 1 + start_row = last_row + + self.wb = wb + self.ws = ws + self.start_row = start_row + self.save_path = save_path + + def add_row_of_list(self, data: list): + for col_idx, value in enumerate(data, start=1): + try: + self.ws.cell(row=self.start_row, column=col_idx, value=value) + except: + pass + self.start_row += 1 + + def save(self): + self.wb.save(self.save_path) + + def close(self): + self.wb.close() diff --git a/core/spiders.py b/core/spiders.py new file mode 100644 index 0000000..cdf4df1 --- /dev/null +++ b/core/spiders.py @@ -0,0 +1,31 @@ +import os +from core import browsers + +class Spiders: + def __init__(self, project_name: str, bitch: int = 5, worker_num: int = 1, switch_clash: bool = False, reflush_browser: bool = False): + self.bitch = bitch + self.worker_num = worker_num + self.switch_clash = switch_clash + self.reflush_browsers = reflush_browser + + self.__data_path__ = f"data/{project_name}" + self.__database_path__ = f"database/{project_name}.db" + self.__urls_folder__ = f"{self.__data_path__}/urls" + self.__excels_folder__ = f"{self.__data_path__}/excels" + self.__errors_folder__ = f"{self.__data_path__}/errors" + + if os.path.exists(self.__data_path__) == False: + os.makedirs(self.__data_path__) + os.makedirs(self.__urls_folder__) + os.makedirs(self.__excels_folder__) + os.makedirs(self.__errors_folder__) + + self.browser = browsers.getBrowsers() + self.browser.set.timeouts(3) + self.tab = self.browser.get_tab(0) + self.tab.set.window.max() + + def reflush_browser(self): + self.browser.quit() + self.browser = browsers.getBrowsers() + self.browser.set.timeouts(3) diff --git a/core/types.py b/core/types.py new file mode 100644 index 0000000..7ac1e4a --- /dev/null +++ b/core/types.py @@ -0,0 +1,263 @@ +import orjson +from bs4 import BeautifulSoup +from pydantic import BaseModel + +class Config(BaseModel): + home_mode: bool = True + skip_category: bool = True + +class GoodsInfo(BaseModel): + """ + 商品类(包含处理) + + :param spu: [必填] + :param title: 商品标题 [必填] + :param brand: 品牌 + :param desc: 简介 + :param attr: 属性 S|M + :param attr_items: 变体[名称]列表 [Color, Size] + :param price: 价格 + :param old_price: 原价 + :param images: 图片列表 + :param url: 主商品链接 + :param p_urls: 变体链接列表 + :param p_lists: 变体类列表 + :param other_data: 其他参数 + + """ + class GoodsInfoP(BaseModel): + """ + 变体类 + + :param attr_items: 变体[值]列表 [red, 30] + + """ + attr_items: list = [] # 变体名称1,2,3 + price: float = 0.00 + old_price: float = 0.00 + images: list[str] = [] + url: str = '' # 商品链接 + other_data: dict = {} # 其他参数 + + spu: str + title: str + brand: str = '' # 品牌 + desc: str = '' # 简介 + attr: str = 'S' # 属性 S M + attr_items: list = [] # 变体名称1,2,3 + price: float = 0.00 + old_price: float = 0.00 + images: list[str] = [] + url: str = '' # 商品链接 + p_urls: list[str] = [] # 变体链接列表(筛选用,可略) + p_lists: list[GoodsInfoP] = [] # 变体列表 + other_data: dict = {} # 其他参数 + + def to_db_data(self): + """转为goods_info.add_goods()所需格式""" + + main_info = { + 'spu': self.spu, + 'title': self.title, + 'brand': self.brand, + 'attr': self.attr, + 'desc': self.desc, + 'price': self.price, + 'old_price': self.old_price, + 'images': orjson.dumps(self.images), + 'url': self.url, + 'other_data': orjson.dumps(self.other_data) + } + for index, item_name in enumerate(self.attr_items): + if item_name: + main_info[f"item{index+1}"] = item_name + + son_infos = [] + for p_info in self.p_lists: + item = { + 'spu': self.spu, + 'price': p_info.price, + 'old_price': p_info.old_price, + 'images': orjson.dumps(p_info.images), + 'url': p_info.url, + 'other_data': orjson.dumps(p_info.other_data) + } + p_info.attr_items += [None, None] + for index, item_name in enumerate(p_info.attr_items[:3]): + if item_name: + item[f"item{index+1}"] = item_name + son_infos.append(item) + + return main_info, son_infos + + def to_shopyy_row_data(self): + """转为shopyy行数据""" + prices = [self.price, self.old_price] + for p_info in self.p_lists: + prices.append(p_info.price) + prices.append(p_info.old_price) + old_price = max(prices) + + rows_data = [] + rows_data.append([ + "", # 商品ID（系统生成，留空） + "", + f"{self.brand} {self.title}" if self.brand != "" else self.title, + self.attr, + "", + "", + self.desc, + "", + "", + "", + "", + "", + "", + "Y", + "Y", + self.spu, + 0, + "N", + 2, + self.brand, # 专辑名称 + "", + "", + *(self.attr_items[:3] + [''] * (3 - len(self.attr_items)))[:3], # 款式1/2/3 + self.price, + old_price, + "", + "", + "", + 50, + '', # 变体备注1（未在模型中定义，可扩展） + '', # 变体备注2 + ','.join(self.images) if self.images else '', + self.url, + ','.join(self.p_urls) if self.p_urls else '' + ]) + + p_index = [] + for p_info in self.p_lists: + if p_info.attr_items in p_index: + continue + p_index.append(p_info.attr_items) + rows_data.append([ + "", # 商品ID（系统生成，留空） + "", + f"{self.brand} {self.title}" if self.brand != "" else self.title, + "P", + "", + "", + self.desc, + "", + "", + "", + "", + "", + "", + "Y", + "Y", + self.spu, + 0, + "N", + 2, + self.brand, # 专辑名称 + "", + "", + *(p_info.attr_items[:3] + [''] * (3 - len(p_info.attr_items)))[:3], # 款式1/2/3 + p_info.price, + old_price, + "", + "", + "", + 50, + '', # 变体备注1（未在模型中定义，可扩展） + '', # 变体备注2 + ','.join(p_info.images) if p_info.images else '', + p_info.url, + "" + ]) + return rows_data + + def to_woo_row_data(self, category_name): + """转为woo行数据""" + def get_lists(attr = 'variable',spu = '', price = '', old_price = '', item_name1: str = '', item_value1: str = '', item_name2: str = '', item_value2: str = '', item_name3: str = '', item_value3: str = '', images: list = []): + return [ + attr, + self.spu if attr == 'variable' or attr == 'simple' else "", + self.title if attr == 'variable' or attr == 'simple' else "", + 1, + desc if attr == 'variable' or attr == 'simple' else "", + 1, + 100, + price, + old_price, + category_name if attr == 'variable' or attr == 'simple' else "", + ','.join(images) if images else '', + self.spu if attr == 'variation' else "", + item_name1, + item_value1, + 1, + item_name2, + item_value2, + 1, + item_name3, + item_value3, + 1, + ] + + desc = BeautifulSoup(self.desc, 'html.parser').get_text() + rows_data = [] + if self.attr == 'S': + rows_data.append(get_lists( + attr='simple', + spu=self.spu, + price=self.price, + old_price=self.old_price, + images=self.images + )) + else: + + item_values1 = set() + item_values2 = set() + item_values3 = set() + for p_info in self.p_lists: + item_value1 = p_info.attr_items[0] if len(p_info.attr_items) >= 1 else "" + item_value2 = p_info.attr_items[1] if len(p_info.attr_items) >= 2 else "" + item_value3 = p_info.attr_items[2] if len(p_info.attr_items) >= 3 else "" + + item_value1 = item_value1.replace(',', '，').replace('|', ' ') + item_values1.add(item_value1) + if item_value2: + item_value2 = item_value2.replace(',', '，').replace('|', ' ') + item_values2.add(item_value2) + if item_value3: + item_value3 = item_value3.replace(',', '，').replace('|', ' ') + item_values3.add(item_value3) + + rows_data.append(get_lists( + attr='variation', + spu='', + price=p_info.price, + old_price=p_info.old_price, + images=p_info.images, + item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "", + item_value1=item_value1, + item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "", + item_value2=item_value2, + item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "", + item_value3=item_value3 + )) + rows_data.insert(0, get_lists( + spu=self.spu, + price='', + old_price='', + images=self.images, + item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "", + item_value1='|'.join(item_values1), + item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "", + item_value2='|'.join(item_values2), + item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "", + item_value3='|'.join(item_values3) + )) + return rows_data diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..8297ea0 --- /dev/null +++ b/lib/__init__.py @@ -0,0 +1,4 @@ +from core import browsers +from lib.module import example + +Server: example.SpiderModule diff --git a/lib/__pycache__/__init__.cpython-312.pyc b/lib/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000..278bf31 Binary files /dev/null and b/lib/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/__pycache__/get_goods.cpython-312.pyc b/lib/__pycache__/get_goods.cpython-312.pyc new file mode 100644 index 0000000..11404c7 Binary files /dev/null and b/lib/__pycache__/get_goods.cpython-312.pyc differ diff --git a/lib/__pycache__/gether.cpython-312.pyc b/lib/__pycache__/gether.cpython-312.pyc new file mode 100644 index 0000000..f8283ed Binary files /dev/null and b/lib/__pycache__/gether.cpython-312.pyc differ diff --git a/lib/__pycache__/shopyy.cpython-312.pyc b/lib/__pycache__/shopyy.cpython-312.pyc new file mode 100644 index 0000000..74181bd Binary files /dev/null and b/lib/__pycache__/shopyy.cpython-312.pyc differ diff --git a/lib/__pycache__/shopyy_verify.cpython-312.pyc b/lib/__pycache__/shopyy_verify.cpython-312.pyc new file mode 100644 index 0000000..87a6301 Binary files /dev/null and b/lib/__pycache__/shopyy_verify.cpython-312.pyc differ diff --git a/lib/gether.py b/lib/gether.py new file mode 100644 index 0000000..4968883 --- /dev/null +++ b/lib/gether.py @@ -0,0 +1,131 @@ +import os +import traceback +import concurrent.futures +from utils import files, messages, clash, formats +import lib +from core import Config, types +from lib.model import goods + +def get_category_urls(): + category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt" + if os.path.exists(category_urls_path): + return + + category_urls = lib.Server.get_category_urls() + for urls in category_urls: + if 'http' not in urls: + continue + files.save_line(category_urls_path, urls) + +def get_goods_urls(categorys_len: int = 3): + goods_urls_path = f"{lib.Server.__urls_folder__}/goods.txt" + category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt" + if Config.home_mode: + goods_urls_path = goods_urls_path.replace('.txt', '(home).txt') + category_urls_path = category_urls_path.replace('.txt', '(home).txt') + + category_index = {} + category_urls = files.load_lines(category_urls_path) + + if os.path.exists(goods_urls_path): + old_goods_urls = files.load_lines(goods_urls_path) + for old_goods_url in old_goods_urls: + category = old_goods_url.split('#')[-1] + category_index[category] = None + + index = 0 + max_len = len(category_urls) + for category_url in category_urls: + category = category_url.split('#')[-1] + category_url = '#'.join(category_url.split('#')[:-1]) + + index += 1 + if 'https' not in category_url: + continue + if Config.skip_category and Config.home_mode == False and len(category.split('/')) < categorys_len: + continue + if category in category_index: + continue + + try: + goods_urls = lib.Server.get_goods_urls(category_url) + goods_category_urls = [] + for goods_url in goods_urls: + goods_category_urls.append(f"{goods_url}#{category}") + + files.add_list(goods_urls_path, goods_category_urls) + except Exception as e: + files.add_text(f"{lib.Server.__errors_folder__}/category_errors.log", f"{category_url}:\n[{e}] {traceback.format_exc()}") + messages.sendError(str(e)) + messages.sendInfo(f"{index}/{max_len} {len(goods_urls)}条") + if Config.home_mode == False: + formats.de_repeat_urls(goods_urls_path) + +def get_goods_info(): + p_urls = {} + def workers(goods_urls, index): + with concurrent.futures.ThreadPoolExecutor(max_workers=lib.Server.worker_num) as executor: + futures = [] + for goods_url in goods_urls: + if goods_url in p_urls: + continue + future = executor.submit(lib.Server.get_goods_info, goods_url) + futures.append(future) + + results: list[types.GoodsInfo] = [] + for future_index, future in enumerate(futures): + index += 1 + try: + result = future.result() # 阻塞直到该任务完成 + except Exception as e: + try: + files.add_text(f"{lib.Server.__errors_folder__}/goods_errors.log", f"{goods_urls[future_index]}:\n[{e}] {traceback.format_exc()}") + except: + pass + messages.sendWarn(f"{goods_urls[future_index]} 出错") + continue + + results.append(result) + messages.sendInfo(f"{index}/{max_len}") + + for goods_info in results: + try: + GoodsModel.add_goods(*goods_info.to_db_data()) + for p_url in goods_info.p_urls: + p_urls[p_url] = None + except Exception as e: + messages.sendWarn(f"{e}") + + return index + + goods_urls = files.load_lines(f"{lib.Server.__urls_folder__}/goods.txt") + messages.sendInfo('读取url完成') + + messages.sendInfo('读取上次存档') + GoodsModel = goods.GoodsModel(lib.Server.__database_path__) + excel_index = GoodsModel.select_goods_to_dict() + messages.sendInfo('读取完成') + + de_save_url = [] + for goods_url in goods_urls: + if goods_url in excel_index: + continue + if 'http' not in goods_url: + continue + de_save_url.append(goods_url) + goods_urls = de_save_url + del excel_index + + max_len = len(goods_urls) + index = 1 + for i in range(0, len(goods_urls), lib.Server.bitch): + index = workers(goods_urls=goods_urls[i:i+lib.Server.bitch], index=index) + if lib.Server.switch_clash: + try: + clash.change_proxy() + except: + pass + if lib.Server.reflush_browsers: + if lib.Server.worker_num > 1: + lib.Server.reflush_browser() + messages.sendInfo("已保存") diff --git a/lib/model/__pycache__/goods.cpython-312.pyc b/lib/model/__pycache__/goods.cpython-312.pyc new file mode 100644 index 0000000..db30e53 Binary files /dev/null and b/lib/model/__pycache__/goods.cpython-312.pyc differ diff --git a/lib/model/goods.py b/lib/model/goods.py new file mode 100644 index 0000000..f303304 --- /dev/null +++ b/lib/model/goods.py @@ -0,0 +1,417 @@ +############################################################################# + +# Author: Cerys +# Update: 2026-02-06 + +############################################################################# + +import os +import orjson +from core import excels, types +from utils import db, messages, files, formats + +class GoodsModel: + def __init__(self, db_path: str): + """ + 商品模型 + + :param db_path: 数据库文件路径(不存在则自动创建并初始化表) + + """ + self.__source_folder__ = "sources" + + create_table = False + if os.path.exists(db_path) == False: + create_table = True + + self.Db = db.Db(db_path) + if create_table: + self.__create_table__() + + def __create_table__(self): + table_sql = ''' + PRAGMA foreign_keys = false; + + DROP TABLE IF EXISTS "goods"; + CREATE TABLE "goods" ( + "spu" TEXT NOT NULL, + "title" TEXT NOT NULL, + "brand" TEXT, + "attr" TEXT NOT NULL, + "desc" TEXT, + "item1" TEXT, + "item2" TEXT, + "item3" TEXT, + "price" NUMBER NOT NULL, + "old_price" NUMBER NOT NULL, + "images" blob NOT NULL, + "url" TEXT NOT NULL, + "other_data" blob DEFAULT "{}", + PRIMARY KEY ("spu") + ); + + DROP TABLE IF EXISTS "goods_son"; + CREATE TABLE "goods_son" ( + "spu" TEXT NOT NULL, + "item1" TEXT, + "item2" TEXT, + "item3" TEXT, + "price" NUMBER NOT NULL, + "old_price" NUMBER NOT NULL, + "images" blob NOT NULL, + "url" TEXT, + "other_data" blob DEFAULT "{}" + ); + + CREATE INDEX "search_spu" + ON "goods_son" ( + "spu" ASC + ); + + PRAGMA foreign_keys = true; + ''' + self.Db.execute_script(table_sql) + + def add_goods(self, main_info: dict, son_infos: list = []): + """ + 添加商品 + + :param main_info: 主表信息 + :param son_infos: 变体信息列表 + + """ + if main_info['attr'] == 'M' and len(son_infos) == 0: + raise Exception(f"变体丢失 {main_info}") + if len(main_info['images']) == 0: + raise Exception('无图片') + + self.Db.table('goods').insert(main_info, True) + self.Db.table('goods_son').insert_all(son_infos) + + def is_repeat(self, url): + """ + 判断是否需要合并 + + :param url: 商品链接 + + """ + count = self.Db.table('goods_son').where('url', url).count() + if count > 1: + return True + return False + + def select_goods_to_lists(self) -> list[dict]: + """获取商品列表（包含变体）""" + messages.sendInfo('开始查询') + db_data = self.Db.table('goods').join('goods_son', 'goods.spu = goods_son.spu').select( + ''' + goods.spu, + goods.title, + goods.brand, + goods.attr, + goods.desc, + goods.item1 as main_item1, + goods.item2 as main_item2, + goods.item3 as main_item3, + goods.price, + goods.old_price, + goods.images, + goods.url, + goods.other_data, + goods_son.item1 as son_item1, + goods_son.item2 as son_item2, + goods_son.item3 as son_item3, + goods_son.price as son_price, + goods_son.old_price as son_old_price, + goods_son.images as son_images, + goods_son.url as son_url, + goods_son.other_data as son_other_data + ''' + ) + messages.sendInfo('查询完成') + return db_data + + def select_goods_to_dict(self, key: str = 'url') -> dict: + """ + 获取商品列表并转换为字典格式 + + :param key: 字典的键 + + """ + db_data = self.select_goods_to_lists() + dict_data = {} + for item in db_data: + if item[key] in dict_data: + dict_data[item[key]]['son_lists'].append({ + 'attr_items': [item['son_item1'], item['son_item2'], item['son_item3']], + 'price': item['son_price'], + 'old_price': item['son_old_price'], + 'images': orjson.loads(item['son_images']), + 'url': item['son_url'], + 'son_other_data': orjson.loads(item['son_other_data']) + }) + else: + dict_data[item[key]] = { + 'spu': item['spu'], + 'title': item['title'], + 'brand': item['brand'], + 'desc': item['desc'], + 'attr': item['attr'], + 'attr_items': [item['main_item1'], item['main_item2'], item['main_item3']], + 'price': item['price'], + 'old_price': item['old_price'], + 'images': orjson.loads(item['images']), + 'url': item['url'], + 'other_data': orjson.loads(item['other_data']) + } + if item['attr'] == 'M': + if item['son_price'] == None: + continue + dict_data[item[key]]['son_lists'] = [ + { + 'attr_items': [item['son_item1'], item['son_item2'], item['son_item3']], + 'price': item['son_price'], + 'old_price': item['son_old_price'], + 'images': orjson.loads(item['son_images']), + 'url': item['son_url'], + 'son_other_data': orjson.loads(item['son_other_data']) + } + ] + return dict_data + + def select_goods_to_goodsinfo(self) -> list[types.GoodsInfo]: + """获取商品列表并转换为GoodsInfo对象列表""" + db_data = self.select_goods_to_lists() + index_data = {} + for item in db_data: + if item['spu'] in index_data: + index_data[item['spu']].p_lists.append( + types.GoodsInfo.GoodsInfoP( + attr_items = [item['son_item1'], item['son_item2'], item['son_item3']], + price = item['son_price'], + old_price = item['son_old_price'], + images = orjson.loads(item['son_images']), + url = item['son_url'], + other_data = orjson.loads(item['son_other_data']) + ) + ) + else: + try: + index_data[item['spu']] = types.GoodsInfo( + spu = item['spu'], + title = item['title'], + brand = item['brand'], + attr = item['attr'], + desc = item['desc'], + attr_items = [item['main_item1'], item['main_item2'], item['main_item3']], + price = item['price'], + old_price = item['old_price'], + images = orjson.loads(item['images']), + url = item['url'], + other_data = orjson.loads(item['other_data']) + ) + if item['attr'] == 'M': + index_data[item['spu']].p_lists.append( + types.GoodsInfo.GoodsInfoP( + attr_items = [item['son_item1'], item['son_item2'], item['son_item3']], + price = item['son_price'], + old_price = item['son_old_price'], + images = orjson.loads(item['son_images']), + url = item['son_url'], + other_data = orjson.loads(item['son_other_data']) + ) + ) + except Exception as e: + messages.sendError(e) + return list(index_data.values()) + + def to_amazon_xlsx(self, goods_url_path: str, save_path: str): + """ + 整数据库导出为亚马逊导入表格 + + :param save_path: 保存路径 + :param goods_url_path: 未去重商品链接路径 + + """ + goods_data = self.select_goods_to_dict() + goods_urls = files.load_lines(goods_url_path) + WookBook = excels.WorkBook( + f'{self.__source_folder__}/amazon商品模板.xlsx', + save_path + ) + + for goods_url in goods_urls: + key = '#'.join(goods_url.split('#')[:-1]) + if key not in goods_data: + continue + + spu = formats.url_to_spu(key) + category = goods_url.split('#')[-1] + + WookBook.add_row_of_list([ + goods_data[key]['title'], + spu, + goods_data[key]['other_data']['popover'], + goods_data[key]['other_data']['popover_num'], + 0, + goods_data[key]['price'], + goods_data[key]['old_price'], + orjson.dumps(goods_data[key]['images']), + goods_data[key]['url'], + category + ]) + + # for goods_info in goods_info_data: + # WookBook.add_row_of_list([ + # goods_info.title, + # goods_info.price, + # goods_info.images[0], + # goods_info.other_data['popover'], + # goods_info.other_data['popover_num'], + # goods_info.url, + # orjson.dumps(goods_info.images) + # ]) + + WookBook.save() + WookBook.close() + + def to_shopyy_xlsx(self, save_path: str, save_bitch: int = 150000, shift_urls_path: str = ''): + """ + 整数据库导出为Shopyy导入表格 + + :param save_path: 保存路径 + :param repeat: 是否合并商品 + :param save_bitch: 超过多少行时自动分割 + + """ + def get_wookbook(bitch_index: int): + return excels.WorkBook( + f'{self.__source_folder__}/shopyy商品模板.xlsx', + save_path.replace('.xlsx', f'_{bitch_index}.xlsx') + ) + + def get_shift_url(): + shift_urls = {} + if shift_urls_path: + urls = files.load_lines(shift_urls_path) + for url in urls: + if url == '': + continue + shift_urls[url] = None + return shift_urls + + goods_info_data = self.select_goods_to_goodsinfo() + bitch = 1 + shift_urls = get_shift_url() + WookBook = get_wookbook(bitch) + + count = len(goods_info_data) + for index, goods_info in enumerate(goods_info_data): + messages.sendInfo(f"{index}/{count}") + if shift_urls_path and goods_info.url not in shift_urls: + continue + + rows_data = goods_info.to_shopyy_row_data() + for row_data in rows_data: + WookBook.add_row_of_list(row_data) + if WookBook.start_row > save_bitch: + WookBook.save() + WookBook.close() + bitch += 1 + WookBook = get_wookbook(bitch) + + WookBook.save() + WookBook.close() + + def to_woo_csv(self, goods_url_path: str, save_path: str): + """ + 整数据库导出为woo商品CSV表格 + + :param goods_url_path: 未去重链接txt路径 + :param save_path: 保存路径 + + """ + goods_urls = files.load_lines(goods_url_path) + category_data = {} + for goods_url in goods_urls: + category_name = goods_url.split('#')[-1].replace('/', '>') + goods_url = '#'.join(goods_url.split('#')[:-1]) + category_data[goods_url] = category_name + goods_info_data = self.select_goods_to_goodsinfo() + + bitch = 0 + save_data = [] + for goods_info in goods_info_data: + rows_data = goods_info.to_woo_row_data(category_data[goods_info.url]) + save_data += rows_data + if len(save_data) > 10000: + excels.save_lists_to_csv(save_data, save_path.replace('.csv', f"{bitch}.csv"), r'sources\woo商品模板.csv') + save_data = [] + bitch += 1 + excels.save_lists_to_csv(save_data, save_path.replace('.csv', f"{bitch}.csv"), r'sources\woo商品模板.csv') + + def import_shopyy_xlsx(self, excel_path: str): + """ + 导入shopyy表格为数据库数据 + + :param excel_path: 表格路径 + """ + shopyy_data = excels.read_excel_to_dict(excel_path) + for url, main_goods_info in shopyy_data.items(): + main_info = { + 'spu': main_goods_info.spu, + 'title': main_goods_info.title, + 'brand': main_goods_info.category_name, + 'attr': main_goods_info.attribute, + 'desc': main_goods_info.desc, + 'price': main_goods_info.price, + 'old_price': main_goods_info.old_price, + 'images': orjson.dumps(main_goods_info.images), + 'url': main_goods_info.url, + 'other_data': orjson.dumps({}) + } + for index, item_name in enumerate(main_goods_info.attribute_items): + if item_name: + main_info[f"item{index+1}"] = item_name + + son_infos = [] + for p_info in main_goods_info.p_lists: + item = { + 'spu': p_info.spu, + 'price': p_info.price, + 'old_price': p_info.old_price, + 'images': orjson.dumps(p_info.images), + 'url': p_info.url, + 'other_data': orjson.dumps({}), + 'item1': None, + 'item2': None, + 'item3': None + } + for index, item_name in enumerate(p_info.attribute_items): + if item_name: + item[f"item{index+1}"] = item_name + son_infos.append(item) + + self.add_goods(main_info, son_infos) + + def splice_db(self, db_folder: str): + """ + 拼接指定文件夹下所有数据库文件 + + :param db_folder: 数据库文件夹路径 + + """ + db_lists = os.listdir(db_folder) + db_index = self.select_goods_to_dict() + for db_name in db_lists: + splice_db = GoodsModel(f"{db_folder}/{db_name}") + goods_lists = splice_db.select_goods_to_goodsinfo() + for goods_info in goods_lists: + if goods_info.url in db_index: + continue + try: + self.add_goods(*goods_info.to_db_data()) + except Exception as e: + messages.sendError(f"{e} {goods_info.spu}") + db_index[goods_info.url] = None + messages.sendInfo(db_name) diff --git a/lib/module/__pycache__/agatameble.cpython-312.pyc b/lib/module/__pycache__/agatameble.cpython-312.pyc new file mode 100644 index 0000000..2c88467 Binary files /dev/null and b/lib/module/__pycache__/agatameble.cpython-312.pyc differ diff --git a/lib/module/__pycache__/ancientnutrition.cpython-312.pyc b/lib/module/__pycache__/ancientnutrition.cpython-312.pyc new file mode 100644 index 0000000..1ea323f Binary files /dev/null and b/lib/module/__pycache__/ancientnutrition.cpython-312.pyc differ diff --git a/lib/module/__pycache__/bricomarche.cpython-312.pyc b/lib/module/__pycache__/bricomarche.cpython-312.pyc new file mode 100644 index 0000000..b171261 Binary files /dev/null and b/lib/module/__pycache__/bricomarche.cpython-312.pyc differ diff --git a/lib/module/__pycache__/condom69.cpython-312.pyc b/lib/module/__pycache__/condom69.cpython-312.pyc new file mode 100644 index 0000000..4013302 Binary files /dev/null and b/lib/module/__pycache__/condom69.cpython-312.pyc differ diff --git a/lib/module/__pycache__/dk_one.cpython-312.pyc b/lib/module/__pycache__/dk_one.cpython-312.pyc new file mode 100644 index 0000000..28b52cc Binary files /dev/null and b/lib/module/__pycache__/dk_one.cpython-312.pyc differ diff --git a/lib/module/__pycache__/example.cpython-312.pyc b/lib/module/__pycache__/example.cpython-312.pyc new file mode 100644 index 0000000..a3e80c2 Binary files /dev/null and b/lib/module/__pycache__/example.cpython-312.pyc differ diff --git a/lib/module/__pycache__/friskybusiness.cpython-312.pyc b/lib/module/__pycache__/friskybusiness.cpython-312.pyc new file mode 100644 index 0000000..45aecbb Binary files /dev/null and b/lib/module/__pycache__/friskybusiness.cpython-312.pyc differ diff --git a/lib/module/__pycache__/peek_cloppenburg.cpython-312.pyc b/lib/module/__pycache__/peek_cloppenburg.cpython-312.pyc new file mode 100644 index 0000000..e700ddb Binary files /dev/null and b/lib/module/__pycache__/peek_cloppenburg.cpython-312.pyc differ diff --git a/lib/module/__pycache__/peek_cloppenburg_nl.cpython-312.pyc b/lib/module/__pycache__/peek_cloppenburg_nl.cpython-312.pyc new file mode 100644 index 0000000..6de47d9 Binary files /dev/null and b/lib/module/__pycache__/peek_cloppenburg_nl.cpython-312.pyc differ diff --git a/lib/module/__pycache__/ssfshop.cpython-312.pyc b/lib/module/__pycache__/ssfshop.cpython-312.pyc new file mode 100644 index 0000000..fc05670 Binary files /dev/null and b/lib/module/__pycache__/ssfshop.cpython-312.pyc differ diff --git a/lib/module/__pycache__/taketoys.cpython-312.pyc b/lib/module/__pycache__/taketoys.cpython-312.pyc new file mode 100644 index 0000000..c72f3b9 Binary files /dev/null and b/lib/module/__pycache__/taketoys.cpython-312.pyc differ diff --git a/lib/module/__pycache__/tfashion.cpython-312.pyc b/lib/module/__pycache__/tfashion.cpython-312.pyc new file mode 100644 index 0000000..efab6ad Binary files /dev/null and b/lib/module/__pycache__/tfashion.cpython-312.pyc differ diff --git a/lib/module/agatameble.py b/lib/module/agatameble.py new file mode 100644 index 0000000..47bf346 --- /dev/null +++ b/lib/module/agatameble.py @@ -0,0 +1,176 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "agatameble" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 1, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.agatameble.pl/') + one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('@tag()=a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + three_menu_eles = two_menu_ele.eles('xpath=ul/li') + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('@tag()=a') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(f"{category_url}") + goods_urls = [] + + while True: + try: + self.tab.scroll.to_bottom() + self.tab.ele('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/div/div/a').click() + except: + break + + goods_eles = self.tab.eles('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/section/div/div/div') + for goods_ele in goods_eles: + goods_link_ele = goods_ele.ele('xpath=a') + goods_urls.append(goods_link_ele.attr('href')) + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + + tab.ele('xpath=//*[@id="contentContainer"]/form/div[1]/section/div/div[2]/button[1]').click() + image_eles = tab.eles('xpath=//*[@id="root"]/div/section/div[2]/div/aside/div[2]/div/div/div/div[1]/div/div/div/div') + for image_ele in image_eles: + image_link_ele = image_ele.ele('xpath=div/img[2]') + image_urls.append(image_link_ele.attr('src').replace('width=', '').replace('height=', '')) + + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + brand = '' + try: + brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text + except: + pass + if brand == '': + title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text + else: + title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text + + desc = "" + desc_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[1]/div/div')[:3] + for desc_ele in desc_eles: + desc += formats.clean_html(desc_ele.html) + + price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + color_urls = [url] + color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li') + for color_ele in color_eles[1:]: + color_link_ele = color_ele.ele('xpath=a') + color_urls.append(color_link_ele.attr('href')) + if len(color_urls) > 1: + m_data = ["Color"] + + if len(color_urls) > 1: + for color_url in color_urls: + color_images = main_images + if color_url != url: + tab.get(color_url) + if brand == '': + title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text + else: + title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text + color_images = get_image_urls() + + price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") + old_price = price + + try: + item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text + except: + continue + attr_items.append({ + 'url': color_url, + 'items': [item_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + + if len(attr_items) == 0: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + url=attr_item['url'], + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'], + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + goods_info.p_urls = color_urls + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/ancientnutrition.py b/lib/module/ancientnutrition.py new file mode 100644 index 0000000..41970d2 --- /dev/null +++ b/lib/module/ancientnutrition.py @@ -0,0 +1,325 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "ancientnutrition" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 1, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://ancientnutrition.com/') + one_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[4]/div[1]/div[1]/div') + for index, one_menu_ele in enumerate(one_menu_eles[1:]): + one_link_ele = one_menu_ele.ele('xpath=div/a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + print(index+2) + two_menu_eles = self.tab.eles(f'xpath=//*[@id="__next"]/div[4]/div[{index+2}]/div/div/div[2]/div/a') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + # three_menu_eles = two_menu_ele.eles('xpath=ul/li') + # for three_menu_ele in three_menu_eles: + # three_link_ele = three_menu_ele.ele('@tag()=a') + # three_url = three_link_ele.attr('href') + # three_name = three_link_ele.text.replace('/', '-') + # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(f"{category_url}") + goods_urls = [] + + while True: + try: + self.tab.scroll.to_bottom() + self.tab.ele('text=Load More').click() + except: + break + + goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[9]/div/div[2]/div[2]/div') + for goods_ele in goods_eles: + try: + goods_link_ele = goods_ele.ele('@tag()=a') + except Exception as e: + continue + goods_urls.append(goods_link_ele.attr('href')) + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + + image_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[1]/div/div[2]/div/div[1]/div') + for image_ele in image_eles: + image_link_ele = image_ele.ele('xpath=div/span/img') + image_urls.append(image_link_ele.attr('src').replace('_150x150', '_1450x1450')) + + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + brand = '' + # try: + # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text + # except: + # pass + # if brand == '': + # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text + # else: + title = tab.ele('@tag()=h1').text + + desc = formats.clean_html(tab.ele('#productDescription').html) + + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + try: + tab.ele('text=Select Flavor').click() + m_data.append('Flavor') + except: + pass + try: + tab.ele('text=Select Size').click() + m_data.append('Size') + except: + pass + try: + tab.ele('text=Select Quantity').click() + m_data.append('Quantity') + except: + pass + + if 'Flavor' in m_data: + tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click() + color_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div[2]/div') + color_xpaths = [] + for color_ele in color_eles: + color_xpaths.append(color_ele.xpath) + + for color_xpath in color_xpaths: + while True: + tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click() + try: + color_ele = tab.ele(f"xpath={color_xpath}") + color_name = color_ele.text + break + except: + tab.get(url) + color_ele.click() + tab.wait(3) + color_images = get_image_urls() + if 'Size' in m_data: + try: + size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}") + except: + size_name = 'Default' + if 'Quantity' in m_data: + quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") + for quantity_index, quantity_ele in enumerate(quantity_eles): + try: + quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") + quantity_name = quantity_ele.text + except: + continue + quantity_ele.click() + tab.wait(2) + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [color_name, size_name, quantity_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [color_name, size_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + size_xpaths = [] + for size_ele in size_eles: + try: + size_xpaths.append(size_ele.xpath) + except: + continue + + for size_xpath in size_xpaths: + try: + size_ele = tab.ele(f"xpath={size_xpath}") + size_name = size_ele.text + except: + continue + size_ele.click() + tab.wait(3) + color_images = get_image_urls() + if 'Quantity' in m_data: + quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") + for quantity_index, quantity_ele in enumerate(quantity_eles): + try: + quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") + quantity_name = quantity_ele.text + except: + continue + quantity_ele.click() + tab.wait(2) + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [color_name, size_name, quantity_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [color_name, size_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [color_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + color_images = [] + if 'Size' in m_data: + size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}") + size_xpaths = [] + for size_ele in size_eles: + try: + size_xpaths.append(size_ele.xpath) + except: + continue + + for size_xpath in size_xpaths: + try: + size_ele = tab.ele(f"xpath={size_xpath}") + size_name = size_ele.text + except: + continue + size_ele.click() + tab.wait(2) + color_images = get_image_urls() + if 'Quantity' in m_data: + quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") + for quantity_index, quantity_ele in enumerate(quantity_eles): + try: + quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") + quantity_name = quantity_ele.text + except: + continue + quantity_ele.click() + tab.wait(2) + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [size_name, quantity_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + if 'Quantity' in m_data: + quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") + for quantity_index, quantity_ele in enumerate(quantity_eles): + try: + quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") + quantity_name = quantity_ele.text + except: + continue + quantity_ele.click() + tab.wait(2) + price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) + old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) + attr_items.append({ + 'url': tab.url, + 'items': [quantity_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + + if len(attr_items) == 0: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + url=attr_item['url'], + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'], + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/bricomarche.py b/lib/module/bricomarche.py new file mode 100644 index 0000000..97c616d --- /dev/null +++ b/lib/module/bricomarche.py @@ -0,0 +1,201 @@ +from core import spiders, types +from utils import formats, messages +from DrissionPage.common import Keys + +class SpiderModule(spiders.Spiders): + project_name = "bricomarche" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 5, + worker_num = 1, + switch_clash = False, + reflush_browser = False + ) + self.tab.get('https://www.bricomarche.pl/') + # input('wait:') + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.bricomarche.pl/') + one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('@tag()=a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + three_menu_eles = two_menu_ele.eles('xpath=ul/li') + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('@tag()=a') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + page = 1 + limit = 100 + self.tab.get(f"{category_url}?page={page}&limit={limit}") + if self.tab.ele('@tag()=h2').text == '执行安全验证': + self.tab.wait(10) + self.tab.actions.type((Keys.TAB, Keys.SPACE)) + goods_urls = [] + + while True: + try: + self.tab.scroll.to_bottom() + goods_eles = self.tab.eles('xpath=//*[@id="spark"]/div[2]/div/section/div/div/main/div[2]/section/div') + for goods_ele in goods_eles: + goods_link_ele = goods_ele.ele('@tag()=a') + goods_urls.append(goods_link_ele.attr('href')) + if len(goods_eles) == 0: + break + except: + break + page += 1 + self.tab.get(f"{category_url}?page={page}&limit={limit}") + if self.tab.ele('@tag()=h2').text == '执行安全验证': + self.tab.wait(10) + self.tab.actions.type((Keys.TAB, Keys.SPACE)) + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + + image_eles = tab.eles('xpath=//*[@id="spark"]/div[2]/div/main/div/div[2]/section[1]/div/div[1]/div/div/div/div/div') + for image_ele in image_eles: + image_link_ele = image_ele.ele('xpath=button/div/img') + image_urls.append(image_link_ele.attr('src').replace('gallery_100_91', 'gallery')) + + return image_urls + + tab = self.browser.new_tab(url) + while True: + try: + if tab.ele('@tag()=h2').text == '执行安全验证': + tab.wait(10) + tab.actions.type((Keys.TAB, Keys.SPACE)) + tab.wait(5) + else: + break + except: + break + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + brand = '' + # try: + # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text + # except: + # pass + + title = tab.ele('@tag()=h1', timeout=10).text + + desc = "" + try: + desc = formats.clean_html(tab.ele('.attributes').html) + except: + pass + + # try: + price_ele = tab.ele('.main-price') + price = float(f"{price_ele.ele('.whole').text}{price_ele.ele('.cents').text.replace(' zł', '')}") + # except: + # price = 0.0 + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + # color_urls = [url] + # color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li') + # for color_ele in color_eles[1:]: + # color_link_ele = color_ele.ele('xpath=a') + # color_urls.append(color_link_ele.attr('href')) + # if len(color_urls) > 1: + # m_data = ["Color"] + + # if len(color_urls) > 1: + # for color_url in color_urls: + # color_images = main_images + # if color_url != url: + # tab.get(color_url) + # if brand == '': + # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text + # else: + # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text + # color_images = get_image_urls() + + # price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") + # old_price = price + + # try: + # item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text + # except: + # continue + # attr_items.append({ + # 'url': color_url, + # 'items': [item_name], + # 'price': price, + # 'old_price': old_price, + # 'images': color_images + # }) + + if len(attr_items) == 0: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + url=attr_item['url'], + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'], + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + # goods_info.p_urls = color_urls + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/condom69.py b/lib/module/condom69.py new file mode 100644 index 0000000..51a6893 --- /dev/null +++ b/lib/module/condom69.py @@ -0,0 +1,136 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "condom69" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 5, + switch_clash = False, + reflush_browser = False + ) + self.tab.get('https://www.condom69.net/') + # input('wait:') + + def get_category_urls(self) -> list[str]: + category_urls = [] + + one_menu_eles = self.tab.eles('xpath=//*[@id="nav-one"]/li') + for one_menu_ele in one_menu_eles[:-1]: + one_link_ele = one_menu_ele.ele('xpath=a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=div/ul/li', timeout=1) + for two_index, two_menu_ele in enumerate(two_menu_eles): + two_link_ele = two_menu_ele.ele('xpath=div/a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + # try: + # three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') + # except: + # continue + # for three_menu_ele in three_menu_eles: + # three_link_ele = three_menu_ele.ele('@tag()=a') + # three_url = three_link_ele.attr('href') + # three_name = three_link_ele.text.replace('/', '-') + # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + page = 1 + self.tab.get(f"{category_url}&pg={page}") + + #####获取页数#### + + ################# + + goods_urls = [] + url_index = {} + while True: + self.tab.scroll.to_bottom() + goods_eles = self.tab.eles('.divProdItemImg2Inner') + page_urls = [] + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('xpath=a').attr('href')}" + if goods_url not in url_index: + page_urls.append(goods_url) + url_index[goods_url] = None + if len(page_urls) > 0: + goods_urls += page_urls + else: + break + page += 1 + self.tab.get(f"{category_url}&pg={page}") + + messages.sendInfo(f"{len(goods_urls)} 个") + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + image_eles = tab.eles('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[1]/div[2]/div') + + for image_ele in image_eles: + image_url = image_ele.ele('xpath=div/div/a/img').attr('src') + image_urls.append(image_url) + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + h1_ele = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[1]') + brand = '' + try: + brand = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/table/tbody/tr/td[3]/a').text + except: + pass + title = h1_ele.text + + desc = "" + try: + desc = formats.clean_html(tab.ele('.ctl00_cphContent_ucUsrIndProduct_pnlIndProdDesc').html) + except: + pass + + price = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[3]/div[1]/div[2]/span').text.replace(',', '') + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/dk_one.py b/lib/module/dk_one.py new file mode 100644 index 0000000..e037a4f --- /dev/null +++ b/lib/module/dk_one.py @@ -0,0 +1,210 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "dk_one" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 1, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.dk_one.com/') + one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('@tag()=a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + if one_name == '라이프': + continue + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + if two_name == '메인': + continue + category_urls.append(f"{two_url}#{one_name}/{two_name}") + two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + if two_name == '메인': + continue + category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}") + + # three_menu_eles = two_menu_ele.eles('xpath=ul/li') + # for three_menu_ele in three_menu_eles: + # three_link_ele = three_menu_ele.ele('@tag()=a') + # three_url = three_link_ele.attr('href') + # three_name = three_link_ele.text.replace('/', '-') + # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(category_url) + goods_urls = [] + goods_eles = self.tab.eles('xpath=//*[@id="prodList"]/div') + for goods_ele in goods_eles: + goods_link_ele = goods_ele.ele('.prod-link') + goods_urls.append(goods_link_ele.attr('href')) + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(tab): + image_urls = [] + image_eles = tab.ele('#prod-img').eles('@tag()=img') + for image_ele in image_eles: + image_urls.append(image_ele.attr('src')) + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + brand = '' + # try: + # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text + # except: + # pass + # if brand == '': + # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text + # else: + title = tab.ele('.page-title').text + + try: + desc = formats.clean_html(tab.ele('@@class=tab-content@@id=tabContent0202').html) + except: + desc = '' + + price = float(tab.ele('.price-group').ele('.val').text.replace(',', '')) + try: + old_price = float(tab.ele('.price-group origin').text.replace(',', '')) + except: + old_price = price + + main_images = get_image_urls(tab) + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + images=main_images + ) + + m_data = [] + color_data = {} + color_eles = [] + # 颜色 + tab.ele('.opt-color') + m_data.append('컬러') + color_eles = tab.ele('.color-list-wrap').eles('.prod-color-rdo') + + m_data.append('사이즈') + + for color_ele in color_eles: + color_input_onclick = color_ele.ele('@tag()=input').attr('onclick') + have_url = False + if 'location.href=' in color_input_onclick: + color_url = 'https://dk-on.com' + formats.re_search(color_input_onclick, r"location\.href\s*=\s*'([^']*)'") + color_name = color_url.split('/')[-1] + have_url = True + elif 'fnColorClicked' in color_input_onclick: + color_url = goods_info.url + color_name = color_ele.ele('@tag()=input').attr('value') + color_ele.click() + else: + color_url = goods_info.url + color_name = goods_info.title.split(' / ')[-1] + + color_data[color_name] = {'url': color_url, 'have_url': have_url, 'ele': color_ele} + goods_info.p_urls.append(color_url) + + real_color_data = {} + + + for color_name, color_item in color_data.items(): + color_tab = tab + if color_item['have_url']: + color_tab = self.browser.new_tab() + + size_data = [] + if color_item['have_url']: + color_tab.get(color_item['url']) + else: + color_item['ele'].click() + color_tab.wait(1) + color_item['price'] = str(color_tab.ele('.price-group').ele('.val').text.replace(',', '')) + try: + color_item['old_price'] = str(color_tab.ele('.price-group origin').text.replace(',', '')) + except: + color_item['old_price'] = color_item['price'] + + # 颜色对应商品图 + color_item['images'] = get_image_urls(tab = color_tab) + + # 颜色对应尺码 + radio_eles = color_tab.ele('.prod-size-list').eles('.prod-size') + for radio_ele in radio_eles: + value = radio_ele.ele('@tag()=label').text + if value == None: + continue + size_data.append(value) + color_item['size'] = size_data + + real_color_data[color_tab.ele('.page-title').text.split(' / ')[-1]] = color_item + color_data = real_color_data + + goods_info.attr = 'M' + goods_info.title = goods_info.title.split(' / ')[0] + goods_info.attr_items = m_data + for color_name, color_item in color_data.items(): + for color_image in color_item['images']: + if color_image in goods_info.images: + continue + goods_info.images.append(color_image) + + # if color_data: + item_data = [] + for color_name, color_item in color_data.items(): + for size in color_item['size']: + item_data.append([color_name, size]) + for attribute_items in item_data: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + attr_items=attribute_items, + price=color_data[attribute_items[0]]['price'], + old_price=color_data[attribute_items[0]]['price'], + images=color_data[attribute_items[0]]['images'], + )) + + except Exception as e: + try: + tab.close() + color_tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + color_tab.close() + except: + pass + return goods_info diff --git a/lib/module/example.py b/lib/module/example.py new file mode 100644 index 0000000..e8d6b45 --- /dev/null +++ b/lib/module/example.py @@ -0,0 +1,86 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "example" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 10, + worker_num = 1, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + pass + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(f"{category_url}") + goods_urls = [] + + pass + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + pass + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + desc = "简介" + brand = "品牌" + title = "标题" + price = 0.00 + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + + if len(attr_items) == 0: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'] + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/friskybusiness.py b/lib/module/friskybusiness.py new file mode 100644 index 0000000..9a1575c --- /dev/null +++ b/lib/module/friskybusiness.py @@ -0,0 +1,141 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "friskybusiness" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 5, + switch_clash = False, + reflush_browser = False + ) + self.tab.get('https://friskybusiness.sg/') + # input('wait:') + + def get_category_urls(self) -> list[str]: + category_urls = [] + + one_menu_eles = self.tab.eles('xpath=/html/body/header/height-observer/x-header/nav[2]/ul/li') + for one_index, one_menu_ele in enumerate(one_menu_eles[:-1]): + + one_link_ele = one_menu_ele.ele('@tag()=summary') + one_url = one_link_ele.attr('data-follow-link') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + if one_index == 0 or one_index == 6: + two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/div/a') + else: + two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/ul/li', timeout=1) + for two_index, two_menu_ele in enumerate(two_menu_eles): + if one_index == 0 or one_index == 6: + two_link_ele = two_menu_ele + else: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + try: + three_menu_eles = two_menu_ele.eles(f'xpath=div/ul/li') + except: + continue + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('@tag()=a') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + page = 1 + self.tab.get(f"{category_url}?page={page}") + goods_urls = [] + + #####获取页数#### + + ################# + + goods_urls = [] + while True: + self.tab.scroll.to_bottom() + try: + goods_eles = self.tab.eles('#:snize-product-', timeout=5) + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" + goods_urls.append(goods_url) + if len(goods_eles) == 0: + break + except: + break + page += 1 + self.tab.get(f"{category_url}?page={page}") + + messages.sendInfo(f"{len(goods_urls)} 个") + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + image_eles = tab.eles('xpath=/html/body/main/section[1]/div/div/product-rerender/div/product-gallery/safe-sticky/product-gallery-navigation/button') + + for image_ele in image_eles[-1:]: + image_url = image_ele.ele('xpath=img').attr('src') + image_urls.append(image_url) + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + h1_ele = tab.ele('@tag()=h1') + brand = '' + title = h1_ele.text + + desc = "" + try: + desc = formats.clean_html(tab.ele('.liquid').html) + except: + pass + + price_list = tab.ele('.price-list price-list--product') + price = price_list.ele('@tag()=sale-price').ele('.money').text.replace('$', '').replace(',', '') + try: + old_price = price_list.ele('@tag()=compare-at-price').ele('.money').text.replace('$', '').replace(',', '') + except: + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/peek_cloppenburg.py b/lib/module/peek_cloppenburg.py new file mode 100644 index 0000000..ca21863 --- /dev/null +++ b/lib/module/peek_cloppenburg.py @@ -0,0 +1,217 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "peek-cloppenburg" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 5, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.peek-cloppenburg.at/') + one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + one_link_ele.click() + self.tab.wait(3) + + two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/a', timeout=1) + for two_index, two_menu_ele in enumerate(two_menu_eles): + two_link_ele = two_menu_ele + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + try: + three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') + except: + continue + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('@tag()=a') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(f"{category_url}?page=1") + goods_urls = [] + + max_page = 1 + #####获取页数#### + try: + max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text) + except: + pass + ################# + + goods_urls = [] + if max_page > 1: + for page in range(1, max_page+1): + if page > 1: + self.tab.get(f"{category_url}?page={page}&userId=123456") + + goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" + goods_urls.append(goods_url) + messages.sendInfo(f"{page}/{max_page}") + else: + goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" + goods_urls.append(goods_url) + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a') + + for image_ele in image_eles: + image_url = image_ele.attr('href') + image_urls.append(image_url) + return image_urls + + def get_size_eles(): + try: + eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li') + except: + pass + try: + eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li') + except: + pass + try: + eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li') + except: + pass + try: + eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li') + except: + pass + return eles + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + h1_eles = tab.ele('@tag()=h1', timeout=10).eles('@tag()=span') + brand = '' + try: + brand = h1_eles[0].text + title = h1_eles[1].text + except: + title = tab.ele('@tag()=h1').text + + desc = "" + try: + desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}") + except: + pass + + price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '.') + try: + old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '.') + except: + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + color_urls = [] + try: + color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li') + m_data.append('Farbe') + for color_ele in color_eles: + color_url = color_ele.ele('@tag()=a').attr('href') + color_urls.append(color_url) + except: + pass + m_data.append('Größe') + + if len(color_urls) > 0: + for color_url in color_urls: + color_images = main_images + if color_url != url: + tab.get(color_url) + color_images = get_image_urls() + + price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '') + try: + old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '') + except: + old_price = price + + color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]') + + size_eles = get_size_eles() + for size_ele in size_eles: + size_name = size_ele.ele('.text-cp-md truncate').text + + attr_items.append({ + 'url': color_url, + 'items': [color_name, size_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + + if len(attr_items) == 0: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + url=attr_item['url'], + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'], + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + goods_info.p_urls = color_urls + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/peek_cloppenburg_nl.py b/lib/module/peek_cloppenburg_nl.py new file mode 100644 index 0000000..6be9f3b --- /dev/null +++ b/lib/module/peek_cloppenburg_nl.py @@ -0,0 +1,213 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "peek-cloppenburg-nl" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 5, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.peek-cloppenburg.nl/') + one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + one_link_ele.click() + self.tab.wait(3) + + two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/a', timeout=1) + for two_index, two_menu_ele in enumerate(two_menu_eles): + two_link_ele = two_menu_ele + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + try: + three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') + except: + continue + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('@tag()=a') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(f"{category_url}?page=1") + goods_urls = [] + + max_page = 1 + #####获取页数#### + try: + max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text) + except: + pass + ################# + + goods_urls = [] + if max_page > 1: + for page in range(1, max_page+1): + if page > 1: + self.tab.get(f"{category_url}?page={page}&userId=123456") + + goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" + goods_urls.append(goods_url) + messages.sendInfo(f"{page}/{max_page}") + else: + goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" + goods_urls.append(goods_url) + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a') + + for image_ele in image_eles: + image_url = image_ele.attr('href') + image_urls.append(image_url) + return image_urls + + def get_size_eles(): + try: + eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li') + except: + pass + try: + eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li') + except: + pass + try: + eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li') + except: + pass + try: + eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li') + except: + pass + return eles + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + h1_eles = tab.ele('@tag()=h1').eles('@tag()=span') + brand = h1_eles[0].text + title = h1_eles[1].text + + desc = "" + try: + desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}") + except: + pass + + price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.') + try: + old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.') + except: + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + color_urls = [] + try: + color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li') + m_data.append('Kleur') + for color_ele in color_eles: + color_url = color_ele.ele('@tag()=a').attr('href') + color_urls.append(color_url) + except: + pass + m_data.append('Maat') + + if len(color_urls) > 0: + for color_url in color_urls: + color_images = main_images + if color_url != url: + tab.get(color_url) + color_images = get_image_urls() + + price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.') + try: + old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.') + except: + old_price = price + + color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]') + + size_eles = get_size_eles() + for size_ele in size_eles: + size_name = size_ele.ele('.text-cp-md truncate').text + + attr_items.append({ + 'url': color_url, + 'items': [color_name, size_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + + if len(attr_items) == 0: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + url=attr_item['url'], + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'], + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + goods_info.p_urls = color_urls + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/ssfshop.py b/lib/module/ssfshop.py new file mode 100644 index 0000000..7dd4674 --- /dev/null +++ b/lib/module/ssfshop.py @@ -0,0 +1,371 @@ +from core import spiders, types +from utils import formats, messages, files +import requests + +class SpiderModule(spiders.Spiders): + project_name = "ssfshop" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 300, + worker_num = 24, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.ssfshop.com/') + one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('@tag()=a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + if one_name == '라이프': + continue + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + if two_name == '메인': + continue + category_urls.append(f"{two_url}#{one_name}/{two_name}") + two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + if two_name == '메인': + continue + category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}") + + # three_menu_eles = two_menu_ele.eles('xpath=ul/li') + # for three_menu_ele in three_menu_eles: + # three_link_ele = three_menu_ele.ele('@tag()=a') + # three_url = three_link_ele.attr('href') + # three_name = three_link_ele.text.replace('/', '-') + # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + page = 1 + self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N") + goods_urls = [] + + url_index = {} + while True: + self.tab.scroll.to_bottom() + page_urls = [] + goods_eles = self.tab.eles('.god-item') + for goods_ele in goods_eles: + goods_link_ele = goods_ele.ele('xpath=a') + if goods_link_ele.attr('href') in url_index: + continue + page_urls.append(goods_link_ele.attr('href')) + url_index[goods_link_ele.attr('href')] = None + + if len(page_urls) == 0: + break + if len(goods_urls) > 5000: + break + goods_urls += page_urls + page += 1 + self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N") + self.tab.wait(1) + + return goods_urls + + def get_goods_info(self, url: str) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + + for _ in range(10): + image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div') + if len(image_eles) > 0: + break + tab.wait(3) + for image_ele in image_eles: + image_link_ele = image_ele + image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data')) + + return image_urls + + def get_options(godNo, optValCd1, optValCd2 = ''): + headers = { + 'Accept': 'application/json, text/javascript, */*; q=0.01', + 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'Connection': 'keep-alive', + 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', + 'Origin': 'https://www.ssfshop.com', + 'Referer': url, + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Sec-Fetch-Site': 'same-origin', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0', + 'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0', + 'X-Requested-With': 'XMLHttpRequest', + 'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + # 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950', + } + + data = { + 'optValCd1': optValCd1, + 'optValCd2': optValCd2, + 'optValCd3': '', + 'godNo': godNo, + } + + response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'}) + json_data = response.json() + + options = {} + if 'opt' not in json_data: + return options + + for option in json_data['opt']: + for i in range(1, 6): + key = f"optValNm{i}" + if option[key]: + options[option[key]] = {} + break + return options + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + brand = url.split('/')[3] + godNo = url.split('/')[4] + + title = tab.ele('#goodDtlTitle').text + + desc = "" + try: + desc = tab.ele('#godsTabView').html + except: + pass + + price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', '')) + try: + old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', '')) + except: + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + color_urls = [] + try: + tab.ele('.gods-option').ele('text=색상').click() + color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label') + for color_ele in color_eles: + color_url = color_ele.attr('onclick') + if color_url: + color_url = color_url.split("'")[1] + else: + color_url = url + if 'https://' not in color_url: + color_url = 'https://www.ssfshop.com'+color_url + color_urls.append(color_url) + except: + pass + + if len(color_urls) > 1: + m_data = ["색상"] + + try: + option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv') + except: + option_eles = [] + for option_ele in option_eles: + m_data.append(option_ele.ele('@tag()=input').attr('opt-nm')) + m_data = m_data[:3] + + option_data = {} + if len(option_eles) > 0: + try: + option_eles[0].click() + except: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + try: + tab.close() + except: + pass + return goods_info + tab.wait(0.5) + option1_eles = option_eles[0].eles('@tag()=li') + for option1_ele in option1_eles: + option_data[option1_ele.text] = get_options(godNo, option1_ele.text) + for option2_name, items in option_data[option1_ele.text].items(): + option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name) + + if len(color_urls) > 1: + for color_url in color_urls: + color_images = main_images + if color_url != url: + tab.get(color_url) + title = tab.ele('#goodDtlTitle').text + color_images = get_image_urls() + + godNo = color_url.split('/')[4] + try: + option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied') + except: + option_eles = [] + option_data = {} + if len(option_eles) > 0: + option1_eles = option_eles[0].eles('xpath=div/ul/li') + for option1_ele in option1_eles: + option_data[option1_ele.text] = get_options(godNo, option1_ele.text) + for option2_name, items in option_data[option1_ele.text].items(): + option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name) + + try: + color_name = title.split('-')[1] + except: + color_name = tab.ele('.buy-txts').text + + price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', '')) + try: + old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', '')) + except: + old_price = price + + if len(option_data) > 0: + for option1_name, items1 in option_data.items(): + if len(items1) > 0: + for option2_name, items2 in items1.items(): + attr_items.append({ + 'url': color_url, + 'items': [color_name, option1_name, option2_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + attr_items.append({ + 'url': color_url, + 'items': [color_name, option1_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + attr_items.append({ + 'url': color_url, + 'items': [color_name], + 'price': price, + 'old_price': old_price, + 'images': color_images + }) + else: + for option1_name, items1 in option_data.items(): + if len(items1) > 0: + for option2_name, items2 in items1.items(): + if len(items2) > 0: + for option3_name, items3 in items2.items(): + attr_items.append({ + 'url': url, + 'items': [option1_name, option2_name, option3_name], + 'price': price, + 'old_price': old_price, + 'images': [] + }) + else: + attr_items.append({ + 'url': url, + 'items': [option1_name, option2_name], + 'price': price, + 'old_price': old_price, + 'images': [] + }) + else: + attr_items.append({ + 'url': url, + 'items': [option1_name], + 'price': price, + 'old_price': old_price, + 'images': [] + }) + + if len(attr_items) == 0: + try: + tab.ele('.gods-option').ele('text=사이즈').click() + size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li') + goods_info.attr = 'M' + goods_info.attr_items = ['사이즈'] + goods_info.images = main_images + for size_ele in size_eles: + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + attr_items=[size_ele.text], + price=price, + old_price=old_price, + images=[], + )) + except: + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + else: + goods_info.attr = 'M' + goods_info.attr_items = m_data + for attr_item in attr_items: + if len(attr_item['items']) != len(m_data): + attr_item['items'] += m_data[len(attr_item['items']):] + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + url=attr_item['url'], + attr_items=attr_item['items'], + price=attr_item['price'], + old_price=attr_item['old_price'], + images=attr_item['images'], + )) + for image_url in attr_item['images']: + if image_url not in main_images: + main_images.append(image_url) + goods_info.images = main_images + goods_info.p_urls = color_urls + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info + + def get_home(self): + self.tab.get('https://www.ssfshop.com/main') + li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li') + categorys = [] + for li_ele in li_eles: + a_ele = li_ele.ele('@tag()=a') + categorys.append(f"{a_ele.attr('href')}#{a_ele.text}") + files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys) diff --git a/lib/module/taketoys.py b/lib/module/taketoys.py new file mode 100644 index 0000000..286744e --- /dev/null +++ b/lib/module/taketoys.py @@ -0,0 +1,131 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "taketoys" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 30, + worker_num = 5, + switch_clash = False, + reflush_browser = False + ) + self.tab.get('https://taketoys.sg/') + # input('wait:') + + def get_category_urls(self) -> list[str]: + category_urls = [] + + one_menu_eles = self.tab.eles('xpath=/html/body/header/div[2]/div/nav/ul/li') + for one_menu_ele in one_menu_eles[1:-1]: + one_link_ele = one_menu_ele.ele('xpath=a') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=ul/li', timeout=1) + for two_index, two_menu_ele in enumerate(two_menu_eles): + two_link_ele = two_menu_ele.ele('xpath=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + # try: + # three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') + # except: + # continue + # for three_menu_ele in three_menu_eles: + # three_link_ele = three_menu_ele.ele('@tag()=a') + # three_url = three_link_ele.attr('href') + # three_name = three_link_ele.text.replace('/', '-') + # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + self.tab.get(f"{category_url}?page=1") + goods_urls = [] + + #####获取页数#### + + ################# + + goods_urls = [] + while True: + self.tab.scroll.to_bottom() + try: + self.tab.ele('text=Show more', timeout=5).click() + self.tab.wait(1) + except: + break + + goods_eles = self.tab.eles('.col-xs-6 col-sm-4 col-lg-3 item-wrapper') + for goods_ele in goods_eles: + goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" + goods_urls.append(goods_url) + messages.sendInfo(f"{len(goods_urls)} 个") + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + image_eles = tab.eles('xpath=/html/body/main/div[2]/div[2]/div[2]/div[2]/div/div/div[3]/div[2]/div/div') + + for image_ele in image_eles: + image_url = image_ele.ele('xpath=img').attr('src') + image_urls.append(image_url) + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + h1_ele = tab.ele('@tag()=h1') + brand = '' + try: + brand = h1_ele.ele('@tag()=strong').text + except: + pass + title = h1_ele.ele('@tag()=span').text + + desc = "" + try: + desc = formats.clean_html(tab.ele('#product-info-tab').html) + except: + pass + + price = tab.ele('.price').attr('title').replace('SGD', '').replace(',', '') + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + goods_info.attr = 'S' + goods_info.attr_items = [] + goods_info.images = main_images + goods_info.p_lists = [] + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/module/tfashion.py b/lib/module/tfashion.py new file mode 100644 index 0000000..31e668c --- /dev/null +++ b/lib/module/tfashion.py @@ -0,0 +1,140 @@ +from core import spiders, types +from utils import formats, messages + +class SpiderModule(spiders.Spiders): + project_name = "t-fashion" + + def __init__(self): + super().__init__( + self.project_name, + bitch = 300, + worker_num = 24, + switch_clash = False, + reflush_browser = False + ) + + def get_category_urls(self) -> list[str]: + category_urls = [] + + self.tab.get('https://www.t-fashion.jp/') + one_menu_eles = self.tab.ele('.m-aside-menu-body').eles('.js-collest-li') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('@tag()=a') + # one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-') + # category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + two_menu_eles = one_menu_ele.eles('xpath=div/ul/li') + for two_menu_ele in two_menu_eles[1:]: + two_link_ele = two_menu_ele.ele('@tag()=a') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + # three_menu_eles = two_menu_ele.eles('xpath=ul/li') + # for three_menu_ele in three_menu_eles: + # three_link_ele = three_menu_ele.ele('@tag()=a') + # three_url = three_link_ele.attr('href') + # three_name = three_link_ele.text.replace('/', '-') + # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") + + return category_urls + + def get_goods_urls(self, category_url: str) -> list[str]: + page = 1 + self.tab.get(f"{category_url}?page={page}") + goods_urls = [] + + while True: + try: + goods_eles = self.tab.eles('xpath=/html/body/div[1]/div[3]/div/section[1]/div/div[1]/div[1]/div[3]/ol/li') + for goods_ele in goods_eles: + goods_link_ele = goods_ele.ele('xpath=a') + goods_urls.append(goods_link_ele.attr('href')) + if len(goods_eles) == 0: + break + except: + break + page += 1 + self.tab.get(f"{category_url}?page={page}") + + return goods_urls + + def get_goods_info(self, url) -> types.GoodsInfo: + def get_image_urls(): + image_urls = [] + + image_eles = tab.eles('xpath=//*[@id="item-gallery-parent"]/li') + for image_ele in image_eles: + image_link_ele = image_ele.ele('xpath=div/img') + image_urls.append(image_link_ele.attr('src')) + + return image_urls + + tab = self.browser.new_tab(url) + tab.set.window.max() + spu = formats.url_to_spu(url) + + try: + brand = '' + try: + brand = tab.ele('xpath=//*[@id="current-tgoods-section"]/div/div[2]/div/div[1]/a[2]', timeout=10).text + except: + pass + + title = tab.ele('@tag()=h1').text + + desc = "" + try: + desc = formats.clean_html(tab.ele('xpath=//*[@id="slide-down-contents-parent"]')) + except: + pass + + try: + price = float(tab.ele('.is-fz28-36f is-fw-b ').text.replace(',', '').replace('¥', '').replace('（税込）', '')) + except: + price = float(tab.ele('.is-fz28-36f is-fw-b is-color-notice').text.replace(',', '').replace('¥', '').replace('（税込）', '')) + old_price = price + + main_images = get_image_urls() + + goods_info = types.GoodsInfo( + title=title, + desc=desc, + brand=brand, + url=url, + spu=spu, + price=price, + old_price=old_price, + ) + + m_data = [] + attr_items = [] + goods_info.attr = 'M' + goods_info.attr_items = ['Color', 'Size'] + goods_info.images = main_images + color_eles = tab.eles('xpath=//*[@id="sku-list-box"]/div') + for color_ele in color_eles: + color_image = color_ele.ele('xpath=div/a').attr('href') + color_name = color_ele.ele('xpath=div/a/span').text + size_eles = color_ele.eles('xpath=ol/li') + for size_ele in size_eles: + size_name = size_ele.ele('xpath=div[1]/text()') + goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( + attr_items=[color_name, size_name], + price=price, + old_price=old_price, + images=[color_image] + )) + except Exception as e: + try: + tab.close() + except: + pass + raise Exception(e) + try: + tab.close() + except: + pass + return goods_info diff --git a/lib/shopyy.py b/lib/shopyy.py new file mode 100644 index 0000000..a44c802 --- /dev/null +++ b/lib/shopyy.py @@ -0,0 +1,371 @@ +import requests +import lib +from core import excels, browsers +from utils import files, formats + +class Shopyy: + def __init__(self, save_folder): + self.save_folder = save_folder + + self.__album_template_path__ = 'sources/专辑模板.xlsx' + self.__navigation_template_path__ = 'sources/导航模板.xlsx' + + # 导航 + def generate_navigation(self, category_path): + index = {} + category_urls = files.load_lines(category_path) + + WorkBook = excels.WorkBook( + template=self.__navigation_template_path__, + save_path=f"{self.save_folder}/导航表格.xlsx" + ) + for category_url in category_urls: + category = category_url.split('#')[-1] + if category in index: + continue + category_lists = category.split('/') + category_lists += ['', ''] + row_data = [ + '顶部导航菜单', + category_lists[0], + category_lists[1], + category_lists[2], + 'collection', + category + ] + WorkBook.add_row_of_list(row_data) + index[category] = None + WorkBook.save() + + return self + + # 专辑 + def generate_album(self, goods_url_path): + def open_workbook(bitch_index) -> excels.WorkBook: + WorkBook = excels.WorkBook( + template=self.__album_template_path__, + save_path=f"{self.save_folder}/专辑表格({bitch_index}).xlsx" + ) + return WorkBook + + album_data = {} + count = 0 + bitch_index = 1 + WorkBook = open_workbook(bitch_index) + + goods_urls = files.load_lines(goods_url_path) + for goods_url in goods_urls: + if goods_url == '': + continue + goods_url_split = goods_url.split('#') + goods_categorys = goods_url_split[-1].split('/') + goods_url = '#'.join(goods_url_split[:-1]) + goods_url = goods_url.split('?')[0] + + for category_index, goods_category in enumerate(goods_categorys): + goods_category = '/'.join(goods_categorys[:category_index + 1]) + + if goods_category in album_data: + mode = '' + sort = '' + else: + mode = 'add' + sort = 'collections_position' + album_data[goods_category] = [] + + row_data = [ + '', + goods_category, + mode, + sort, + '', + formats.url_to_spu(goods_url) + ] + album_data[goods_category].append(row_data) + count += 1 + + save_bitch = count/3+1 + for goods_category, row_datas in album_data.items(): + for row_data in row_datas: + WorkBook.add_row_of_list(row_data) + + if WorkBook.start_row > save_bitch and bitch_index <= 2: + WorkBook.save() + WorkBook.close() + + bitch_index += 1 + WorkBook = open_workbook(bitch_index) + WorkBook.save() + WorkBook.close() + + return self + +class ShopyyVerify: + def __init__(self, url): + self.url = url + browser = browsers.getBrowsers() + self.tab = browser.get_tab(0) + self.login() + + def login(self): + self.tab.get(f"{self.url}/admin/login") + print('等待登录') + while True: + token = self.tab.local_storage('CARDADMIN_TOKEN') + if token: + print('登录成功') + self.token = token.replace('"', '') + break + self.tab.wait(1) + + def get_zero_collections(self, delete: bool = False): + def get_collections(page): + headers = { + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'priority': 'u=1, i', + # 'referer': 'https://cisalfasport.zenshop.cn/admin/collections?page=1&pagesize=50', + 'sec-ch-ua': '"Not(A:Brand";v="8", "Chromium";v="144", "Microsoft Edge";v="144"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'token': self.token, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36 Edg/144.0.0.0', + # 'cookie': '__stripe_mid=89b6ec3f-bbac-483d-b8ab-9938859a0b340b72d8; _aiihbe_vid=18FC7E3E-1CA7-FDC2-C52D-37A4BB0E1DF0; _aiihbe_cvid=75CFD325-E408-2C25-C126-07A349ED46C5; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1770185710%2C%22expire_time%22%3A1772777710%7D%5D; landing_page=aHR0cHM6Ly9jaXNhbGZhc3BvcnQuemVuc2hvcC5jbi9wcm9kdWN0cy90b3VyLWRlLWZyYW5jZS1qcj9wcmV2aWV3PTE%3D; first_http_referer=https%3A%2F%2Fcisalfasport.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1770185710; PHPSESSID=6329e9a44018771ce40064bc50fbe189; _aiihbe_vs=36B35E5C-874E-4D6C-8945-F5CF6F323F99; theme_id=411278; _AIIHBE_ua=Mozilla%252F5.0%2520(Windows%2520NT%252010.0%253B%2520Win64%253B%2520x64)%2520AppleWebKit%252F537.36%2520(KHTML%252C%2520like%2520Gecko)%2520Chrome%252F144.0.0.0%2520Safari%252F537.36%2520Edg%252F144.0.0.0; _AIIHBE_pu=https%253A%252F%252Fcisalfasport.zenshop.cn%252Fadmin%252Ftheme; _AIIHBE_tz=Etc%2FGMT-8; _AIIHBE_ss=1920X1080; _AIIHBE_lang=zh-CN; _AIIHBE_vs=1874X930; _AIIHBE_dt=2026-02-06%2016%3A30%3A44; acw_tc=a3b54ee217704477374193466e23fb58e387e54f5b84504527a4beab6b; cdn_sec_tc=a3b54ee217704477374193466e23fb58e387e54f5b84504527a4beab6b; __stripe_sid=49bcd8c1-4ae4-46ba-8e95-a29205606cb2980c18; visit_token=4be69765c90de228978e9738878367fbkj083fsy; _aiihbe_ka=A298A3E1-379E-4DA0-2B28-A5961B741FA9; is_theme_edit=1', + } + + params = { + 'page': str(page), + 'pagesize': '600', + } + + response = requests.get(f'{self.url}/api/collections', params=params, headers=headers) + response_data = response.json() + if response_data['code'] != 0: + raise Exception(response_data['msg']) + return response_data['data'] + + def delete_collections(ids: list): + headers = { + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'cache-control': 'no-cache', + 'content-type': 'application/json', + 'origin': 'https://peekcloppenburgat.zenshop.cn', + 'pragma': 'no-cache', + 'priority': 'u=1, i', + # 'referer': 'https://peekcloppenburgat.zenshop.cn/admin/collections?page=1&pagesize=600', + 'sec-ch-ua': '"Not:A-Brand";v="99", "Microsoft Edge";v="145", "Chromium";v="145"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'token': self.token, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0', + # 'cookie': '__stripe_mid=49d8f368-8d82-4b14-9c42-63e0d33542ef6372ca; _bzcece_vid=E823CDA9-7EDE-3617-71BB-80F879AD878E; _bzcece_cvid=36F627EC-095F-9323-5047-43325149C808; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1773370701%2C%22expire_time%22%3A1775962701%7D%5D; landing_page=aHR0cHM6Ly9wZWVrY2xvcHBlbmJ1cmdhdC56ZW5zaG9wLmNuL3Byb2R1Y3RzL3Zlcm8tbW9kYS13aWRlLWxlZy1qZWFucy1hdXMtYmF1bXdvbGwtbWl4LW1vZGVsbC10ZXNzYS1pbi1oZWxsZ3JhdT9wcmV2aWV3PTE%3D; first_http_referer=https%3A%2F%2Fpeekcloppenburgat.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1773370701; PHPSESSID=fba65fe62a66071f174801bb432b942c; theme_id=418468; __stripe_sid=323c1341-4108-4c33-b331-ea6b782f9a2349679f; visit_token=f1d5170d234867f2ca490821f454239bmwnlrqr7; _bzcece_vs=5F572333-803E-D038-11EC-6CFCBA6F32D3; _bzcece_ka=B2503B21-4B44-A5A0-F261-71EC86D52A6E; is_theme_edit=1; acw_tc=9b66b49a17734819180038754ede11478ee763b95bb1c9acff317f521f; cdn_sec_tc=9b66b49a17734819180038754ede11478ee763b95bb1c9acff317f521f', + } + + json_data = { + 'ids': ids, + } + + response = requests.post( + f'{self.url}/api/collections/batchdelete', + headers=headers, + json=json_data, + ) + print(response.json()) + + print('开始检查专辑数量') + collection_items = [] + data = get_collections(1) + page_total = data['paginate']['pageTotal'] + collection_items += data['collections'] + print(f"1/{page_total}") + + if page_total > 1: + for page in range(2, page_total+1): + data = get_collections(page) + collection_items += data['collections'] + print(f"{page}/{page_total}") + + lost_collection = [] + zero_ids = [] + for collection_item in collection_items: + if collection_item['collection_products_count'] == 0: + lost_collection.append(collection_item['title']) + print(f"{collection_item['title']} 数量为0") + zero_ids.append(collection_item['id']) + if delete: + delete_collections(zero_ids) + + if len(lost_collection) > 0: + with open('商品数量为0专辑.txt', 'w', encoding='utf-8') as f: + f.write('\n'.join(lost_collection)) + return self + + def get_error_images(self, delete: bool = False): + def get_errors(page): + headers = { + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'cache-control': 'no-cache', + 'dnt': '1', + 'pragma': 'no-cache', + 'priority': 'u=1, i', + # 'referer': 'https://vanuffelenmode.zenshop.cn/admin/resources', + 'sec-ch-ua': '"Not:A-Brand";v="99", "Microsoft Edge";v="145", "Chromium";v="145"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'token': self.token, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0', + # 'cookie': 'acw_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; cdn_sec_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; __stripe_mid=cc09d025-360b-466e-bef4-ba92d045fdc94448e2; __stripe_sid=511d0972-d39d-4aa1-a031-5258db651ad1d4124e; visit_token=9b8172c220d5be292045835470d38951px0lh404', + } + + params = { + 'file_name': '', + 'page_type': 'fail', + 'page': str(page), + 'pagesize': '600', + } + + response = requests.get(f'{self.url}/api/resources', params=params, headers=headers) + response_data = response.json() + if response_data['code'] != 0: + raise Exception(response_data['msg']) + return response_data['data'] + + def post_retry(ids: list): + headers = { + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'content-type': 'application/json', + # 'origin': 'https://ssfshop.zenshop.cn', + 'priority': 'u=1, i', + # 'referer': 'https://ssfshop.zenshop.cn/admin/resources', + 'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'token': self.token, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0', + # 'cookie': '__stripe_mid=a8059a05-4d7e-476d-b376-9e54f222efbe3c23c7; _bzgigi_vid=85F94DA2-7BD9-525E-E4A5-8D686E51B2C6; _bzgigi_cvid=EEFE28A2-FDC4-DC54-E76A-2BC1B2929243; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1775195861%2C%22expire_time%22%3A1777787861%7D%5D; landing_page=aHR0cHM6Ly9zc2ZzaG9wLnplbnNob3AuY24vcHJvZHVjdHMvJUU4JTg3JUFBJUU3JTk0JUIxJUU1JThDJUJBLWwtMTklRTUlOEYlQjclRTMlODIlQjUlRTMlODIlQTQlRTMlODIlQkElRTMlODElQkUlRTMlODElQTclRTUlQjElOTUlRTklOTYlOEItJUUzJTgyJUFCJUUzJTgyJUJGJUUzJTgzJUFEJUUzJTgyJUIwJUU2JThFJUIyJUU4JUJDJTg5LSVFMyU4MiVCQiVFMyU4MyU4MyVFMyU4MyU4OCVFMyU4MiVBMiVFMyU4MyU4MyVFMyU4MyU5NyVFNSVBRiVCRSVFNSVCRiU5Qy0lRTMlODIlQkIlRTMlODMlQUMlRTMlODMlQTIlRTMlODMlOEIlRTMlODMlQkMtJUU5JTgwJTlBJUU1JThCJUE0LSVFMyU4MyU4OCVFMyU4MyVBQSVFMyU4MiVBMiVFMyU4MiVCQiVFMyU4MyU4MCVFMyU4MyU5NiVFMyU4MyVBQiVFMyU4MiVBRiVFMyU4MyVBRCVFMyU4MiVCOSVFMyU4MyU4NiVFMyU4MyVCQyVFMyU4MyVBOSVFMyU4MyVCQyVFMyU4MyU4OS0lRTMlODIlQjglRTMlODMlQTMlRTMlODIlQjElRTMlODMlODMlRTMlODMlODhfOGNhZjQ4Y2U%2FcHJldmlldz0x; first_http_referer=https%3A%2F%2Fssfshop.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1775195861; visit_token=41be25876fa245bb99018d68f7338d89y49l7utp; _bzgigi_vs=E5BE5924-EEF9-E277-6AFC-5F70BB548D0F; PHPSESSID=b08ff5f29b9469e1e92740efef87c09e; _BZGIGI_ua=Mozilla%252F5.0%2520(Windows%2520NT%252010.0%253B%2520Win64%253B%2520x64)%2520AppleWebKit%252F537.36%2520(KHTML%252C%2520like%2520Gecko)%2520Chrome%252F146.0.0.0%2520Safari%252F537.36%2520Edg%252F146.0.0.0; _BZGIGI_pu=https%253A%252F%252Fssfshop.zenshop.cn%252Fadmin%252Fproducts; _BZGIGI_tz=Etc%2FGMT-8; _BZGIGI_ss=2560X1440; _BZGIGI_lang=zh-CN; _BZGIGI_vs=2514X1316; _BZGIGI_dt=2026-04-07%2014%3A48%3A20; theme_id=429104; is_theme_edit=1; acw_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; cdn_sec_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; __stripe_sid=a0f6f277-4a95-4d43-bd6f-c38f108e17f52fc752', + } + response = requests.post(f'{self.url}/api/resources/retry', headers=headers, json={'ids':ids}) + print(response.json()) + + def delete_images(ids: list): + headers = { + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'content-type': 'application/json', + # 'origin': 'https://ssfshop.zenshop.cn', + 'priority': 'u=1, i', + # 'referer': 'https://ssfshop.zenshop.cn/admin/resources', + 'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'token': self.token, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0', + # 'cookie': '__stripe_mid=a8059a05-4d7e-476d-b376-9e54f222efbe3c23c7; _bzgigi_vid=85F94DA2-7BD9-525E-E4A5-8D686E51B2C6; _bzgigi_cvid=EEFE28A2-FDC4-DC54-E76A-2BC1B2929243; utm_source=direct; utm_medium=default; order_utm_history=%5B%7B%22utm_source%22%3A%22direct%22%2C%22utm_medium%22%3A%22default%22%2C%22utm_term%22%3A%22%22%2C%22utm_campaign%22%3A%22%22%2C%22utm_content%22%3A%22%22%2C%22source_device%22%3A%22computer%22%2C%22create_time%22%3A1775195861%2C%22expire_time%22%3A1777787861%7D%5D; landing_page=aHR0cHM6Ly9zc2ZzaG9wLnplbnNob3AuY24vcHJvZHVjdHMvJUU4JTg3JUFBJUU3JTk0JUIxJUU1JThDJUJBLWwtMTklRTUlOEYlQjclRTMlODIlQjUlRTMlODIlQTQlRTMlODIlQkElRTMlODElQkUlRTMlODElQTclRTUlQjElOTUlRTklOTYlOEItJUUzJTgyJUFCJUUzJTgyJUJGJUUzJTgzJUFEJUUzJTgyJUIwJUU2JThFJUIyJUU4JUJDJTg5LSVFMyU4MiVCQiVFMyU4MyU4MyVFMyU4MyU4OCVFMyU4MiVBMiVFMyU4MyU4MyVFMyU4MyU5NyVFNSVBRiVCRSVFNSVCRiU5Qy0lRTMlODIlQkIlRTMlODMlQUMlRTMlODMlQTIlRTMlODMlOEIlRTMlODMlQkMtJUU5JTgwJTlBJUU1JThCJUE0LSVFMyU4MyU4OCVFMyU4MyVBQSVFMyU4MiVBMiVFMyU4MiVCQiVFMyU4MyU4MCVFMyU4MyU5NiVFMyU4MyVBQiVFMyU4MiVBRiVFMyU4MyVBRCVFMyU4MiVCOSVFMyU4MyU4NiVFMyU4MyVCQyVFMyU4MyVBOSVFMyU4MyVCQyVFMyU4MyU4OS0lRTMlODIlQjglRTMlODMlQTMlRTMlODIlQjElRTMlODMlODMlRTMlODMlODhfOGNhZjQ4Y2U%2FcHJldmlldz0x; first_http_referer=https%3A%2F%2Fssfshop.zenshop.cn%2Fadmin%2Fproducts; first_visit_time=1775195861; visit_token=41be25876fa245bb99018d68f7338d89y49l7utp; _bzgigi_vs=E5BE5924-EEF9-E277-6AFC-5F70BB548D0F; PHPSESSID=b08ff5f29b9469e1e92740efef87c09e; _BZGIGI_ua=Mozilla%252F5.0%2520(Windows%2520NT%252010.0%253B%2520Win64%253B%2520x64)%2520AppleWebKit%252F537.36%2520(KHTML%252C%2520like%2520Gecko)%2520Chrome%252F146.0.0.0%2520Safari%252F537.36%2520Edg%252F146.0.0.0; _BZGIGI_pu=https%253A%252F%252Fssfshop.zenshop.cn%252Fadmin%252Fproducts; _BZGIGI_tz=Etc%2FGMT-8; _BZGIGI_ss=2560X1440; _BZGIGI_lang=zh-CN; _BZGIGI_vs=2514X1316; _BZGIGI_dt=2026-04-07%2014%3A48%3A20; theme_id=429104; is_theme_edit=1; acw_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; cdn_sec_tc=9b66b4a617756120379694001e5c7efed3faa08e9cf1c621bbfeb27c77; __stripe_sid=60ef73cf-e5ed-483a-a342-9e581a0387d64df267', + } + json_data = { + 'ids': ids, + 'delete_product': 0 + } + response = requests.delete('https://ssfshop.zenshop.cn/api/resources/batchdelete', headers=headers, json=json_data) + print(response.json()) + + print('开始检查失败图片') + image_items = [] + data = get_errors(1) + page_total = data['paginate']['pageTotal'] + image_items += data['list'] + print(f"1/{page_total}") + + if page_total > 1: + for page in range(2, page_total+1): + ids = [] + data = get_errors(1) + for item in data['list']: + ids.append(item['id']) + if delete: + delete_images(ids) + else: + post_retry(ids) + page_total = data['paginate']['pageTotal'] + print(f"{page}/{page_total}") + + # lost_image_urls = [] + # for image_item in image_items: + # lost_image_urls.append(image_item['remote_url']) + + # if len(lost_image_urls) > 0: + # files.save_list('采集失败图片链接.txt', lost_image_urls) + # print(f"共有: {len(lost_image_urls)} 条失败链接") + + def get_all_images(self): + def get_images(page): + headers = { + 'accept': 'application/json, text/plain, */*', + 'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', + 'cache-control': 'no-cache', + 'dnt': '1', + 'pragma': 'no-cache', + 'priority': 'u=1, i', + # 'referer': 'https://vanuffelenmode.zenshop.cn/admin/resources', + 'sec-ch-ua': '"Not:A-Brand";v="99", "Microsoft Edge";v="145", "Chromium";v="145"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'empty', + 'sec-fetch-mode': 'cors', + 'sec-fetch-site': 'same-origin', + 'token': self.token, + 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36 Edg/145.0.0.0', + # 'cookie': 'acw_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; cdn_sec_tc=a3b54dab17734824723112557e921e5eb8672de1b4bf2e0bfb7de57fe1; __stripe_mid=cc09d025-360b-466e-bef4-ba92d045fdc94448e2; __stripe_sid=511d0972-d39d-4aa1-a031-5258db651ad1d4124e; visit_token=9b8172c220d5be292045835470d38951px0lh404', + } + + params = { + 'file_name': '', + 'page_type': 'all', + 'page': str(page), + 'pagesize': '600', + } + + response = requests.get(f'{self.url}/api/resources', params=params, headers=headers) + response_data = response.json() + if response_data['code'] != 0: + raise Exception(response_data['msg']) + return response_data['data'] + + print('开始获取图片url') + image_items = [] + data = get_images(1) + page_total = data['paginate']['pageTotal'] + image_items += data['list'] + print(f"1/{page_total}") + + if page_total > 1: + for page in range(2, page_total+1): + data = get_images(page) + image_items += data['list'] + print(f"{page}/{page_total}") + + image_urls = [] + for image_item in image_items: + image_urls.append(image_item['remote_url']) + + if len(image_urls) > 0: + files.save_list('图片链接.txt', image_urls) + print(f"共有: {len(image_urls)} 张图片") diff --git a/main.py b/main.py new file mode 100644 index 0000000..b109ed6 --- /dev/null +++ b/main.py @@ -0,0 +1,49 @@ +import lib +from lib import gether, shopyy +from lib.model import goods +from utils import formats, files, wpdata + +if __name__ == '__main__': + from lib.module import example, ssfshop, tfashion + + Server:example.SpiderModule + + Server = ssfshop.SpiderModule() + lib.Server = Server + + # category_urls = Server.get_category_urls() + # files.save_list(f"{Server.__urls_folder__}/categorys.txt", category_urls) + # Server.get_home() + # gether.get_goods_urls(2) + # formats.de_repeat_urls(r'data\bricomarche\urls\goods(home).txt') + # gether.get_goods_info() + + GoodsModel = goods.GoodsModel(Server.__database_path__) + # GoodsModel.to_shopyy_xlsx(f"{Server.__excels_folder__}/goods.xlsx") + # GoodsModel.to_woo_csv(r'data\fs\urls\goods_2026-01-23.txt_old.txt', f"data/fs/goods.csv") + # shopyy.Shopyy(Server.__data_path__).generate_album( + # f"{Server.__urls_folder__}/goods.txt_old.txt" + # ) + # shopyy.Shopyy(Server.__data_path__).generate_navigation( + # f"{Server.__urls_folder__}/categorys.txt" + # ) + # shopyy.Shopyy(Server.__data_path__).generate_album( + # f"{Server.__urls_folder__}/goods(home).txt" + # ).generate_navigation( + # f"{Server.__urls_folder__}/categorys(home).txt" + # ) + + # wpdata.Wpdata( + # 'data/condom69', + # r'data\condom69\excels\goods_1.xlsx', + # r'data\condom69\urls\goods.txt_old.txt', + # ).run() + + # Verify = shopyy.ShopyyVerify('https://ssfshop.zenshop.cn') + # Verify.get_zero_collections(True) + # Verify.get_error_images(True) + + # GoodsModel.splice_db('D:/Workers/Cerys/spiders/database/ssfshop') + GoodsModel.Db.close() + + Server.browser.quit() diff --git a/server.py b/server.py new file mode 100644 index 0000000..d94bf16 --- /dev/null +++ b/server.py @@ -0,0 +1,205 @@ +import os +import re +import platform +import subprocess +import psutil +import winreg +from flask import Flask, jsonify, request +from flask_cors import CORS + +app = Flask(__name__) + +# <--- 2. 启用全局跨域支持 +# 默认允许所有域名跨域访问所有接口。 +# 如果想只允许特定前端域名，可以改成 CORS(app, origins=["http://localhost:8080", "http://你的域名.com"]) +CORS(app, supports_credentials=True) + +# Windows 下分离进程的标志位，确保 exe 独立运行 +DETACHED_PROCESS = 0x00000008 + +def get_cpu_model(): + """ + 通过读取 Windows 注册表获取准确的 CPU 型号 + """ + try: + # 打开注册表路径 + key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"HARDWARE\DESCRIPTION\System\CentralProcessor\0") + # 读取 ProcessorNameString 的值 + cpu_model, _ = winreg.QueryValueEx(key, "ProcessorNameString") + return cpu_model.strip() + except Exception: + return "Unknown CPU" + +def is_valid_task_id(task_id): + """ + 校验 task_id，只允许字母、数字、下划线和连字符，防止目录穿越攻击 + """ + return re.match(r'^[\w\-]+$', task_id) is not None + +@app.route('/api/system_info', methods=['GET']) +def get_system_info(): + """ + 接口 1：获取当前设备(Windows)的负载情况和硬件信息（附带字段说明） + """ + try: + # 获取基础监控数据 + cpu_usage = psutil.cpu_percent(interval=0.5) + cpu_cores_physical = psutil.cpu_count(logical=False) + cpu_cores_logical = psutil.cpu_count(logical=True) + cpu_freq = psutil.cpu_freq() + virtual_mem = psutil.virtual_memory() + disk_usage = psutil.disk_usage('C:\\') + + # 构建返回数据，为每个返回值添加 desc 注释说明 + info = { + # "os": { + # "system": platform.system(), # 操作系统名称 (如 Windows) + # "release": platform.release(), # 操作系统主要版本号 (如 10, 11) + # "version": platform.version(), # 操作系统的详细内部构建版本号 + # "machine": platform.machine() # 系统架构 (如 AMD64 代表 64位) + # }, + "os": f"{platform.system()} {platform.version()} ({platform.machine()})", # 综合描述操作系统的字符串 + "hardware": { + "cpu": f"{get_cpu_model()} ({cpu_cores_physical}P/{cpu_cores_logical}L) {cpu_freq.max if cpu_freq else None}GHz", # CPU 型号和核心数的综合描述字符串 + # "cpu_model": get_cpu_model(), # CPU 具体型号名称 + # "cpu_cores_physical": cpu_cores_physical, # CPU 物理核心数 (真实核心) + # "cpu_cores_logical": cpu_cores_logical, # CPU 逻辑核心数 (包含超线程) + # "cpu_max_freq_mhz": cpu_freq.max if cpu_freq else None, # CPU 最大设计频率 (单位: MHz) + "ram_total_gb": round(virtual_mem.total / (1024 ** 3), 2), # 物理内存 (RAM) 总容量 (单位: GB) + "disk_c_total_gb": round(disk_usage.total / (1024 ** 3), 2) # 系统盘 (C盘) 总容量 (单位: GB) + }, + "load": { + "cpu_usage_percent": cpu_usage, # 当前 CPU 总利用率 (单位: %) + "ram_usage_percent": virtual_mem.percent, # 当前物理内存使用率 (单位: %) + "ram_available_gb": round(virtual_mem.available / (1024 ** 3), 2), # 当前可用的物理内存大小 (单位: GB) + "disk_c_usage_percent": disk_usage.percent, # 系统盘 (C盘) 存储空间使用率 (单位: %) + "disk_c_free_gb": round(disk_usage.free / (1024 ** 3), 2) # 系统盘 (C盘) 剩余可用空间 (单位: GB) + } + } + return jsonify({"code": 200, "status": "success", "data": info}), 200 + + except Exception as e: + return jsonify({"code": 500, "status": "error", "message": str(e)}), 500 + +@app.route('/api/run_exe', methods=['POST']) +def run_exe(): + """ + 接口 2：非阻塞地运行一个 exe 程序 + """ + data = request.get_json() + if not data or 'exe_path' not in data: + return jsonify({"status": "error", "message": "Missing 'exe_path' in request body."}), 400 + + exe_path = data['exe_path'] + args = data.get('args', []) + + if not os.path.exists(exe_path): + return jsonify({"status": "error", "message": f"File not found: {exe_path}"}), 404 + + try: + command = [exe_path] + args + process = subprocess.Popen( + command, + creationflags=DETACHED_PROCESS, + close_fds=True, + shell=False + ) + + return jsonify({ + "code": 200, + "status": "success", + "message": "Executable started successfully.", + "pid": process.pid + }), 200 + + except Exception as e: + return jsonify({"code": 500, "status": "error", "message": str(e)}), 500 + +@app.route('/api/upload', methods=['POST']) +def upload_files(): + """ + 接口 3：上传文件 + 接收 form-data 格式数据： + - task_id: 任务ID (字符串) + - file: 上传的文件 (可以同时上传多个文件) + """ + # 1. 获取并校验 task_id + task_id = request.form.get('task_id') + if not task_id: + return jsonify({"status": "error", "message": "Missing 'task_id' in form data."}), 400 + + if not is_valid_task_id(task_id): + return jsonify({"status": "error", "message": "Invalid 'task_id'. Only alphanumeric, dash, and underscore are allowed."}), 400 + + # 2. 准备目录 + # os.path.join 会自动根据当前系统(Windows)处理路径分隔符 + data_dir = os.path.join("data", task_id) + lib_dir = "lib/module" + + # 自动创建不存在的目录 (exist_ok=True 避免目录已存在时报错) + os.makedirs(data_dir, exist_ok=True) + os.makedirs(lib_dir, exist_ok=True) + + saved_files = [] + + # 3. 遍历并保存文件 + # request.files 是一个字典，通过 .getlist('file') 获取所有上传的文件（假设表单键名为 file） + # 为了兼容不同的前端传法，我们遍历 request.files 中的所有文件对象 + for key in request.files: + for file in request.files.getlist(key): + if file and file.filename: + # 获取文件后缀，转换为小写 + ext = os.path.splitext(file.filename)[1].lower() + + if ext == '.txt': + # .txt 储存至 data/{task_id}/goods.txt + save_path = os.path.join(data_dir, "goods.txt") + file.save(save_path) + saved_files.append({"original_name": file.filename, "saved_path": save_path}) + + elif ext == '.py': + # .py 储存至 lib/{task_id}.py + save_path = os.path.join(lib_dir, f"{task_id}.py") + file.save(save_path) + saved_files.append({"original_name": file.filename, "saved_path": save_path}) + else: + # 忽略不支持的文件格式 + pass + + if not saved_files: + return jsonify({"status": "error", "message": "No valid .txt or .py files were uploaded."}), 400 + + return jsonify({ + "status": "success", + "message": "Files uploaded successfully.", + "data": saved_files + }), 200 + +@app.route('/api/get_status', methods=['GET']) +def get_status(): + params = request.args + task_id = params.get('task_id') + + progress = 0 + status = 0 + if os.path.exists(f"data/{task_id}/progress"): + with open(f"data/{task_id}/progress", "r", encoding="utf-8") as f: + progress = f.read().strip() + if os.path.exists(f"data/{task_id}/status"): + with open(f"data/{task_id}/status", "r", encoding="utf-8") as f: + status = f.read().strip() + + return jsonify({ + "status": "success", + "message": f"Task '{task_id}' is ready.", + "data": { + "task_id": task_id, + "progress": progress, + "status": status + } + }), 200 + +if __name__ == '__main__': + # 解决 jsonify 返回中文字符时变成 Unicode 编码的问题 + app.config['JSON_AS_ASCII'] = False + app.run(host='127.0.0.1', port=5000, debug=True) \ No newline at end of file diff --git a/sources/amazon商品模板.xlsx b/sources/amazon商品模板.xlsx new file mode 100644 index 0000000..f48e106 Binary files /dev/null and b/sources/amazon商品模板.xlsx differ diff --git a/sources/shopyy商品模板.xlsx b/sources/shopyy商品模板.xlsx new file mode 100644 index 0000000..4c870d7 Binary files /dev/null and b/sources/shopyy商品模板.xlsx differ diff --git a/sources/woo商品模板.csv b/sources/woo商品模板.csv new file mode 100644 index 0000000..cbe3432 --- /dev/null +++ b/sources/woo商品模板.csv @@ -0,0 +1 @@ +Type,SKU,Name,Published,Description,In Stock?,Stock,Sale Price,Regular Price,Categories,Images,Parent,Attribute 1 Name,Attribute 1 Value(s),Attribute 1 visible,Attribute 2 Name,Attribute 2 Value(s),Attribute 2 visible,Attribute 3 Name,Attribute 3 Value(s),Attribute 3 visible diff --git a/sources/专辑模板.xlsx b/sources/专辑模板.xlsx new file mode 100644 index 0000000..0607d2d Binary files /dev/null and b/sources/专辑模板.xlsx differ diff --git a/sources/导航模板.xlsx b/sources/导航模板.xlsx new file mode 100644 index 0000000..e22e85d Binary files /dev/null and b/sources/导航模板.xlsx differ diff --git a/utils/__pycache__/clash.cpython-312.pyc b/utils/__pycache__/clash.cpython-312.pyc new file mode 100644 index 0000000..bd80d6e Binary files /dev/null and b/utils/__pycache__/clash.cpython-312.pyc differ diff --git a/utils/__pycache__/db.cpython-312.pyc b/utils/__pycache__/db.cpython-312.pyc new file mode 100644 index 0000000..4d451f9 Binary files /dev/null and b/utils/__pycache__/db.cpython-312.pyc differ diff --git a/utils/__pycache__/files.cpython-312.pyc b/utils/__pycache__/files.cpython-312.pyc new file mode 100644 index 0000000..ea5ca0e Binary files /dev/null and b/utils/__pycache__/files.cpython-312.pyc differ diff --git a/utils/__pycache__/formats.cpython-312.pyc b/utils/__pycache__/formats.cpython-312.pyc new file mode 100644 index 0000000..8cd6137 Binary files /dev/null and b/utils/__pycache__/formats.cpython-312.pyc differ diff --git a/utils/__pycache__/messages.cpython-312.pyc b/utils/__pycache__/messages.cpython-312.pyc new file mode 100644 index 0000000..7f58684 Binary files /dev/null and b/utils/__pycache__/messages.cpython-312.pyc differ diff --git a/utils/__pycache__/redis.cpython-312.pyc b/utils/__pycache__/redis.cpython-312.pyc new file mode 100644 index 0000000..1246633 Binary files /dev/null and b/utils/__pycache__/redis.cpython-312.pyc differ diff --git a/utils/__pycache__/uploads.cpython-312.pyc b/utils/__pycache__/uploads.cpython-312.pyc new file mode 100644 index 0000000..92425d1 Binary files /dev/null and b/utils/__pycache__/uploads.cpython-312.pyc differ diff --git a/utils/__pycache__/wpdata.cpython-312.pyc b/utils/__pycache__/wpdata.cpython-312.pyc new file mode 100644 index 0000000..027e280 Binary files /dev/null and b/utils/__pycache__/wpdata.cpython-312.pyc differ diff --git a/utils/clash.py b/utils/clash.py new file mode 100644 index 0000000..908f709 --- /dev/null +++ b/utils/clash.py @@ -0,0 +1,64 @@ +import requests +import random + + + +def change_proxy(): + port = '17650' + secret = '7ce2f241-d6a1-4b69-a742-96201072187a' + selector = '🚀 节点选择' + + port = port + secret = secret + + headers_secret = { + 'Authorization': 'Bearer {}'.format(secret) + } + + url_all_proxies_info = 'http://127.0.0.1:{}/proxies'.format(port) + url_all_proxies_info = requests.get(url_all_proxies_info,headers=headers_secret).json() + url_all_proxies = 'http://127.0.0.1:{}/proxies/{}'.format(port,selector) + + res_proxies = requests.get(url_all_proxies,headers=headers_secret).json() + # res_proxies = res_proxies['proxies'][selector] + proxy_name_list = res_proxies['all'][3:-3] + now_proxy = res_proxies['now'] + print('现在的代理是{}'.format(now_proxy)) + # if now_proxy not in ['DIRECT','REJECT','账号邮箱看最新的地址','NETV2','自动选择','故障转移']: + # proxy_name_list.remove(now_proxy) + + shift = [] + continue_proxys = ['♻️ 自动选择', '仅海外用户'] + for proxy_name in proxy_name_list: + is_continue = False + for continue_proxy in continue_proxys: + if continue_proxy in proxy_name: + is_continue = True + break + if is_continue: + continue + # if '香港' not in proxy_name or '日本' not in proxy_name or '俄罗斯' not in proxy_name: + # continue + shift.append(proxy_name) + proxy_name_list = shift + + while True: + random_proxy = random.choice(proxy_name_list) + delay = url_all_proxies_info['proxies'][random_proxy]['history'][-1]['delay'] + if delay!=0: + break + print('随机选择的代理为{}'.format(random_proxy)) + data = { + "name": random_proxy + } + headers = { + "content-type": "application/json", + 'Authorization': 'Bearer {}'.format(secret) + } + + res = requests.put(url=url_all_proxies,json=data,headers=headers) + print('切换代理请求的状态码为{}'.format(res.status_code)) + if res.status_code == 204: + print('切换代理成功！现在的代理为{}'.format(random_proxy)) +if __name__ == '__main__': + change_proxy() diff --git a/utils/db.py b/utils/db.py new file mode 100644 index 0000000..17435c3 --- /dev/null +++ b/utils/db.py @@ -0,0 +1,672 @@ +############################################################################# + +# Author: Cerys +# Update: 2026-03-06 + +############################################################################# + +import sqlite3 +import pymysql +import copy +import functools +import re +from pymysql.constants import CLIENT +from pymysql.cursors import DictCursor +from typing import List, Dict, Any, Optional, Union +from contextlib import contextmanager + +# ================= 统一执行装饰器 ================= +def db_execute_wrapper(func): + """ + 统一执行装饰器： + 1. 负责自动获取/检查连接 + 2. 自动处理非事务状态下的 commit 和 rollback + 3. 针对 MySQL 处理 Gone Away 断连重连重试机制 + """ + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + self.__connect__() + try: + res = func(self, *args, **kwargs) + return res + except Exception as e: + # 针对 MySQL 的长连接断开 (OperationalError) 尝试进行一次重试 + if getattr(self, '_is_mysql', False) and not self._in_transaction: + if isinstance(e, pymysql.err.OperationalError): + try: + # 【修复】废弃 reconnect=True，直接销毁并重新初始化连接 + self.conn = None + self.__connect__() + + res = func(self, *args, **kwargs) + return res + except Exception: + if self.conn: + self.conn.rollback() + raise # 【修复】直接 raise，保留完整的错误堆栈 Traceback + + # 常规错误回滚 + if getattr(self, 'conn', None) and not getattr(self, '_in_transaction', False): + self.conn.rollback() + raise # 【修复】直接 raise，保留完整的错误堆栈 Traceback + return wrapper +# ================================================= + +class BaseDriver: + """ + 基类：负责定义接口类型，让 VS Code 能识别返回值 + """ + def __init__(self): + self.conn = None + # 子类需设置这两个属性 + self.param_mark = '?' + self._is_mysql = False + self._transaction_depth = 0 + + # -------------- 核心执行方法 (清晰直白) -------------- + + @db_execute_wrapper + def fetch_all(self, sql: str, params: tuple = ()) -> List[dict]: + """查询多条：返回字典列表""" + cursor = self.conn.cursor() + try: + cursor.execute(sql, params) + rows = cursor.fetchall() + if not rows: + return [] + # 兼容性处理：如果是对象(SQLite Row)则转dict，如果是dict(MySQL)则直接用 + return [dict(row) for row in rows] if rows else [] + finally: + cursor.close() + + @db_execute_wrapper + def fetch_one(self, sql: str, params: tuple = ()) -> Optional[dict]: + """查询单条：返回字典 或 None""" + cursor = self.conn.cursor() + try: + cursor.execute(sql, params) + row = cursor.fetchone() + return dict(row) if row else None + finally: + cursor.close() + + @db_execute_wrapper + def execute(self, sql: str, params: tuple = ()) -> int: + """增删改：返回影响行数""" + cursor = self.conn.cursor() + try: + cursor.execute(sql, params) + if not self._in_transaction: + self.conn.commit() + return cursor.rowcount + finally: + cursor.close() + + @db_execute_wrapper + def execute_insert(self, sql: str, params: tuple = ()) -> int: + """插入：返回自增ID""" + cursor = self.conn.cursor() + try: + cursor.execute(sql, params) + if not self._in_transaction: + self.conn.commit() + return cursor.lastrowid + finally: + cursor.close() + + @db_execute_wrapper + def execute_many(self, sql: str, params_list: List[tuple]) -> int: + """批量执行：返回影响行数""" + cursor = self.conn.cursor() + try: + cursor.executemany(sql, params_list) + if not self._in_transaction: + self.conn.commit() + return cursor.rowcount + finally: + cursor.close() + + @db_execute_wrapper + def execute_raw(self, sql: str, params: tuple = ()) -> Union[List[dict], int]: + """混合执行：根据是否返回结果集自动判断""" + cursor = self.conn.cursor() + try: + cursor.execute(sql, params) + # 如果有 description 说明是 SELECT 类查询 + if cursor.description: + rows = cursor.fetchall() + return [dict(row) for row in rows] if rows else [] + else: + if not self._in_transaction: + self.conn.commit() + return cursor.rowcount + finally: + cursor.close() + + @db_execute_wrapper + def execute_script(self, sql_script: str) -> None: + """简单的脚本执行，不支持复杂的存储过程分隔符""" + self.__connect__() + cursor = self.conn.cursor() + try: + # 开启多语句支持通常需要在 connect 时指定 client_flag + # 或者手动分割 + statements = [s.strip() for s in sql_script.split(';') if s.strip()] + for sql in statements: + cursor.execute(sql) + if not self._in_transaction: + self.conn.commit() + finally: + cursor.close() + +class MysqlDriver(BaseDriver): + def __init__(self, host, port, user, password, database, charset='utf8mb4'): + super().__init__() + self.param_mark = '%s' # MySQL 占位符 + self.quote_mark = '`' + self.config = { + 'host': host, + 'port': port, + 'user': user, + 'password': password, + 'database': database, + 'charset': charset, + 'autocommit': False, + 'cursorclass': DictCursor, + 'client_flag': CLIENT.MULTI_STATEMENTS + } + self._is_mysql = True + self.__connect__() + + @property + def _in_transaction(self) -> bool: + """判断当前是否处于事务中""" + return self._transaction_depth > 0 + + def __connect__(self) -> None: + if self.conn is None: + self.conn = pymysql.connect(**self.config) + else: + try: + # 【修复】去除已在 PyMySQL 新版废弃的 reconnect=True 参数 + self.conn.ping(reconnect=False) + except Exception: + self.conn = pymysql.connect(**self.config) + + # =============== 事务控制方法 =============== + def begin(self) -> None: + self.__connect__() + if self._transaction_depth == 0: + self.conn.begin() + self._transaction_depth += 1 + + def commit(self) -> None: + if self._transaction_depth > 0: + self._transaction_depth -= 1 + if self._transaction_depth == 0 and self.conn: + self.conn.commit() + + def rollback(self) -> None: + # 安全清理嵌套层级，防止局部回滚后引发后续误提交 + if self._transaction_depth > 0: + self._transaction_depth = 0 + if self.conn: + self.conn.rollback() + # ============================================ + + def close(self) -> None: + if self.conn: + self.conn.close() + self.conn = None + + +class SqliteDriver(BaseDriver): + def __init__(self, db_path: str): + super().__init__() + self.param_mark = '?' # SQLite 占位符 + self.quote_mark = '"' + self.db_path = db_path + self._is_mysql = False + self.__connect__() + + @property + def _in_transaction(self) -> bool: + return self._transaction_depth > 0 + + def __connect__(self) -> None: + if not self.conn: + self.conn = sqlite3.connect(self.db_path, check_same_thread=False, timeout=10.0) + self.conn.row_factory = sqlite3.Row + self.conn.execute('PRAGMA journal_mode=WAL;') + self.conn.execute('PRAGMA synchronous=NORMAL;') + + @db_execute_wrapper + def execute_script(self, sql_script: str) -> None: + """SQLite 原生支持脚本执行，更安全且无需手动分割""" + # 注意：sqlite3.Cursor.executescript 不遵循 commit 逻辑，它会直接提交 + # 所以这里不需要 conn.commit()，但为了保持一致性逻辑，还是走装饰器 + cursor = self.conn.cursor() + try: + cursor.executescript(sql_script) + # SQLite executescript 会自动 commit，不需要手动再 commit + finally: + cursor.close() + + # =============== 事务控制方法 =============== + def begin(self) -> None: + self.__connect__() + self._transaction_depth += 1 + + def commit(self) -> None: + if self._transaction_depth > 0: + self._transaction_depth -= 1 + if self._transaction_depth == 0 and self.conn: + self.conn.commit() + + def rollback(self) -> None: + if self._transaction_depth > 0: + self._transaction_depth = 0 + if self.conn: + self.conn.rollback() + # ============================================ + + def close(self) -> None: + if self.conn: + self.conn.close() + self.conn = None + + +class Query: + """查询构建器：负责拼装 SQL，且支持链式调用不可变对象""" + def __init__(self, driver, table_name: str): + self.driver = driver + self.table_name = table_name + self._wheres: List[tuple] = [] # 【修改】存为 tuple: (逻辑符, 语句) 支持 OR + self._params: List[Any] = [] + self._orders: List[str] = [] + self._joins: List[str] = [] + self._limit: Optional[int] = None + self._offset: Optional[int] = None + + self.mark = getattr(driver, 'param_mark', '?') + self.quote = getattr(driver, 'quote_mark', '') + + def __copy_instance(self) -> 'Query': + """【核心】创建当前对象的深拷贝，用于链式调用不污染原对象""" + new_query = copy.copy(self) + new_query._wheres = self._wheres[:] + new_query._params = self._params[:] + new_query._orders = self._orders[:] + new_query._joins = self._joins[:] + return new_query + + def _q(self, field: str) -> str: + field = field.strip() + if field == '*' or '(' in field or ')' in field: + return field + + # 简单的防止注入校验 + if re.search(r'[;\'"\s\-]', field): + raise ValueError(f"非法字段名: {field}") + + # 支持 table.column 格式 + if '.' in field: + table, col = field.split('.', 1) + return f"{self.quote}{table}{self.quote}.{self.quote}{col}{self.quote}" + + return f"{self.quote}{field}{self.quote}" + + # ================= 链式构建方法 (返回新对象) ================= + + def where(self, key: str, value: Any, operator: str = '=') -> 'Query': + """ + 单条件判断 + + :param key: 字段名 + :param value: 值 + :param operator: 运算符 + + """ + new_q = self.__copy_instance() + condition = f"{self._q(key)} {operator} {self.mark}" + new_q._wheres.append(("AND", condition)) + new_q._params.append(value) + return new_q + + def or_where(self, key: str, value: Any, operator: str = '=') -> 'Query': + """ + OR 条件判断 + 方法同where + """ + new_q = self.__copy_instance() + condition = f"{self._q(key)} {operator} {self.mark}" + new_q._wheres.append(("OR", condition)) + new_q._params.append(value) + return new_q + + def where_all(self, condition: Dict[str, Any], operator: str = '=', logic: str = 'AND') -> 'Query': + """ + 多条件判断 + + :param condition:param condition: { 字段名: 值 } + :param operator: 运算符 (传入字典建议保持默认 `=`) + :param logic: 逻辑运算符内部连接(AND/OR) + + """ + new_q = self.__copy_instance() + clauses = [] + for key, value in condition.items(): + clauses.append(f"{self._q(key)} {operator} {self.mark}") + new_q._params.append(value) + + if clauses: + full_condition = f"({f' {logic.upper()} '.join(clauses)})" + # 作为一个整体条件加入，默认外部通过 AND 连接 + new_q._wheres.append(("AND", full_condition)) + return new_q + + def where_in(self, key: str, values: List[Any]) -> 'Query': + """ + IN 语句支持 + + :param key: 字段名 + :param values: 匹配值列表 + """ + if not values: + raise ValueError("whereIn 的 values 参数不能为空列表") + new_q = self.__copy_instance() + placeholders = ", ".join([self.mark] * len(values)) + condition = f"{self._q(key)} IN ({placeholders})" + new_q._wheres.append(("AND", condition)) + new_q._params.extend(values) + return new_q + + def where_raw(self, sql: str, params: tuple = ()) -> 'Query': + """ + 原生条件语句 (支持复杂逻辑查询) + 注意：调用方需自行通过 params 传递参数以防注入 + + :param sql: SQL 字符串，如 "age > ? AND (status = ? OR role = ?)" + :param params: 对应占位符的参数元组 + """ + new_q = self.__copy_instance() + new_q._wheres.append(("AND", f"({sql})")) + new_q._params.extend(params) + return new_q + + def join(self, table: str, on: str, join_type: str = 'LEFT') -> 'Query': + """ + 链表查询 + + :param table: 要链接的表名 + :param on: 链接条件 + :param join_type: 链接类型(LEFT/INNER/RIGHT) + + """ + new_q = self.__copy_instance() + # 注意: join 的 on 条件由开发者自己写 raw sql，需自行注意安全 + new_q._joins.append(f"{join_type.upper()} JOIN {self._q(table)} ON {on}") + return new_q + + def order(self, field: str, direction: str = 'ASC') -> 'Query': + """ + 排序 + + :param field: 排序字段 + :param direction: 排序方向(ASC/DESC) + + """ + direction = direction.upper() + if direction not in ('ASC', 'DESC'): + raise ValueError("排序方向只能是 ASC 或 DESC") + + new_q = self.__copy_instance() + # _q() 内部已经加上了正则校验，防止 order by 注入 + new_q._orders.append(f"{self._q(field)} {direction}") + return new_q + + def limit(self, limit: int, offset: int = 0) -> 'Query': + """ + 分页 + + :param limit: 每页数量 + :param offset: 第几页 (从0开始) + + """ + new_q = self.__copy_instance() + new_q._limit = int(limit) + new_q._offset = int(offset) + return new_q + + # ================= 终结方法 (执行并返回结果) ================= + + def select(self, fields: str = '*') -> List[dict]: + """ + 执行查询返回列表 + + :param fields: 查询字段 + + """ + sql = self._build_select_sql(fields) + return self.driver.fetch_all(sql, tuple(self._params)) + + def find(self, fields: str = '*') -> Optional[dict]: + """ + 执行查询返回单条 + + :param fields: 查询字段 + + """ + temp_q = self.__copy_instance() + temp_q._limit = 1 + temp_q._offset = 0 + + sql = temp_q._build_select_sql(fields) + return self.driver.fetch_one(sql, tuple(temp_q._params)) + + def count(self) -> int: + """统计数量""" + sql = self._build_select_sql("COUNT(*) AS cnt") + res = self.driver.fetch_one(sql, tuple(self._params)) + return res.get('cnt', 0) if res else 0 + + def insert(self, data: Dict[str, Any], replace: bool = False) -> int: + """ + 插入单条数据 + + :param data: { 字段名: 值 } + :param replace: 是否使用 REPLACE INTO(存在则覆盖) + :return: 影响行数/自增主键ID + + """ + keys = [self._q(k) for k in data.keys()] + values = list(data.values()) + placeholders = ",".join([self.mark] * len(values)) + columns = ",".join(keys) + + action = "REPLACE" if replace else "INSERT" + sql = f"{action} INTO {self._q(self.table_name)} ({columns}) VALUES ({placeholders})" + return self.driver.execute_insert(sql, tuple(values)) + + def insert_all(self, data: List[Dict[str, Any]], replace: bool = False) -> int: + """ + 批量插入 + 每条数据key结构必须相同 + + :param data: [{ 字段名: 值 }, ...] + :param replace: 是否使用 REPLACE INTO(存在则覆盖) + :return: 影响行数 + + """ + if not data: return 0 + + raw_keys = list(data[0].keys()) + keys = [self._q(k) for k in raw_keys] + columns = ",".join(keys) + placeholders = ",".join([self.mark] * len(keys)) + + params_list = [tuple(d.get(k) for k in raw_keys) for d in data] + + action = "REPLACE" if replace else "INSERT" + sql = f"{action} INTO {self._q(self.table_name)} ({columns}) VALUES ({placeholders})" + + return self.driver.execute_many(sql, params_list) + + def update(self, data: Dict[str, Any]) -> int: + """ + 更新符合条件的所有行数据 + + :param data: { 字段名: 值 } + :return: 影响行数 + + """ + if not data: + return 0 # 避免生成无效 SQL + + set_clauses = [] + set_params = [] + + for key, value in data.items(): + set_clauses.append(f"{self._q(key)} = {self.mark}") + set_params.append(value) + + set_sql = ", ".join(set_clauses) + where_sql = self._build_where_sql() + if not where_sql: + raise ValueError("WHERE 条件不存在，为防止全表更新操作已被拦截!") + + full_params = set_params + self._params + sql = f"UPDATE {self._q(self.table_name)} SET {set_sql} {where_sql}" + return self.driver.execute(sql, tuple(full_params)) + + def delete(self) -> int: + """ + 删除符合条件的所有行数据 + :return: 影响行数 + + """ + where_sql = self._build_where_sql() + if not where_sql: + raise ValueError("WHERE 条件不存在，为防止全表删除操作已被拦截!") + sql = f"DELETE FROM {self._q(self.table_name)} {where_sql}" + return self.driver.execute(sql, tuple(self._params)) + + # ================= 内部辅助方法 ================= + + def _build_where_sql(self) -> str: + if not self._wheres: + return "" + + # 【修改】适配新的 Tuple(logic, condition) 结构以支持 OR + where_parts = [] + for i, (logic, condition) in enumerate(self._wheres): + if i == 0: + where_parts.append(condition) # 首个条件不需要 AND/OR 前缀 + else: + where_parts.append(f"{logic} {condition}") + + return "WHERE " + " ".join(where_parts) + + def _build_select_sql(self, fields: str) -> str: + parts = [f"SELECT {fields} FROM {self._q(self.table_name)}"] + + if self._joins: + parts.extend(self._joins) + + where_sql = self._build_where_sql() + if where_sql: + parts.append(where_sql) + + if self._orders: + parts.append("ORDER BY " + ", ".join(self._orders)) + + if self._limit is not None: + parts.append(f"LIMIT {self._limit}") + + if self._offset is not None: + parts.append(f"OFFSET {self._offset}") + + return " ".join(parts) + + +class Db: + """数据库入口""" + def __init__(self, db_path: str, config: dict = None, db_type: str = None): + config = config or {} + + if db_type is None: + if db_path.lower() == 'mysql' or 'host' in config: + db_type = 'mysql' + else: + db_type = 'sqlite' + + if db_type == 'mysql': + self.driver = MysqlDriver(**config) + elif db_type == 'sqlite': + self.driver = SqliteDriver(db_path) # 【修复】采用正确的命名 + else: + raise ValueError("db_type 参数错误: 应为 'mysql' 或 'sqlite'") + + @contextmanager + def transaction(self): + """ + 事务上下文管理器 + with db.transaction(): + db.table('A').insert(...) + db.table('B').update(...) + """ + self.driver.begin() + try: + yield self + self.driver.commit() + except Exception: + self.driver.rollback() + raise # 【修复】直接 raise 避免截断真实报错的 Traceback + + def table(self, table_name: str) -> Query: + """ + 获取表操作对象 + + :param table_name: 表名 + + """ + return Query(self.driver, table_name) + + def execute_raw(self, sql: str, params: tuple = ()) -> Union[List[dict], int]: + """ + 执行原生 SQL + + :param sql: SQL 语句 + :param params: 参数 + + """ + return self.driver.execute_raw(sql, params) + + def execute_many_raw(self, sql: str, params_list: List[tuple]) -> int: + """ + 批量执行原生 SQL + + :param sql: SQL 语句 + :param params_list: 参数列表 + + """ + return self.driver.execute_many(sql, params_list) + + def execute_script(self, sql_script: str) -> None: + """ + 执行多条 SQL 语句（如建表） + + :param sql_script: SQL 脚本 + + """ + self.driver.execute_script(sql_script) + + def close(self) -> None: + self.driver.close() + + # Context Manager 支持 + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() \ No newline at end of file diff --git a/utils/files.py b/utils/files.py new file mode 100644 index 0000000..57e79f2 --- /dev/null +++ b/utils/files.py @@ -0,0 +1,24 @@ +def save_line(filepath, text): + with open(filepath, 'a+', encoding='utf-8') as f: + f.writelines(text+'\n') + +def load_lines(filepath): + with open(filepath, 'r', encoding='utf-8') as f: + text = f.read() + return text.split('\n') + +def save_text(filepath, text): + with open(filepath, 'w', encoding='utf-8') as f: + f.write(text) + +def add_text(filepath, text): + with open(filepath, 'a+', encoding='utf-8') as f: + f.writelines(text+'\n') + +def save_list(filepath, lists: list): + with open(filepath, 'w', encoding='utf-8') as f: + f.write('\n'.join(lists)) + +def add_list(filepath, lists: list): + with open(filepath, 'a+', encoding='utf-8') as f: + f.write('\n'.join(lists)+'\n') diff --git a/utils/formats.py b/utils/formats.py new file mode 100644 index 0000000..41a93f0 --- /dev/null +++ b/utils/formats.py @@ -0,0 +1,104 @@ +import hashlib +from bs4 import BeautifulSoup +import time +import random +import datetime +import re + +def url_to_spu(url: str, length: int = 36) -> str: + h = hashlib.sha256(url.encode('utf-8')).digest() + int_val = int.from_bytes(h, 'big') + chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + base36 = '' + while int_val > 0: + int_val, idx = divmod(int_val, 36) + base36 = chars[idx] + base36 + base36 = base36.zfill(50) + return base36[:length] + +# 工具函数，几乎每个站点的采集都会用到 +def remove_unwanted_tags(html_content): + html_content = str(html_content) + soup = BeautifulSoup(html_content, 'html.parser') + for tag in soup(['button', 'img', 'a', 'script', 'svg', 'video']): + tag.decompose() + return str(soup) + +def clean_html(html): + soup = BeautifulSoup(html, 'html.parser') + + # 遍历所有标签 + for tag in soup(['button', 'a', 'script']): + tag.attrs = {} # 直接清空所有属性 + + return str(soup) + +def de_repeat_urls(filepath): + index = {} + with open(filepath, 'r', encoding='utf-8') as f: + text = f.read() + old_data = text.split('\n') + with open(f"{filepath}_old.txt", 'w', encoding='utf-8') as f: + f.write('\n'.join(old_data)) + + new_data = [] + for line in old_data: + line = '#'.join(line.split('#')[:-1]) + if line in index: + continue + new_data.append(line) + index[line] = None + with open(filepath, 'w', encoding='utf-8') as f: + f.write('\n'.join(new_data)) + +def getTime(timeStamp = False, format: str = '%Y-%m-%d %H:%M:%S'): + """ + 获取时间 + Author Cerys + ChangeTime 2023-11-10 + + @param timeStamp 指定时间戳 + @param fotmat 指定格式 + + return str + """ + + if timeStamp != False: + time = datetime.datetime.fromtimestamp(timeStamp) + else: + time = datetime.datetime.now() + formatTime = time.strftime(format) + + return formatTime + +def formatName(name: str, types: bool = False): + """ + 下划线字符串命名转大驼峰或小驼峰 + Author Cerys + ChangeTime 2023-11-10 + + @param name 要转换的字符串 + @param types 是否转为大驼峰 + + return str + """ + + strLists = name.split('_') + if types: + return strLists[0].title() + ''.join(x.title() for x in strLists[1:]) + else: + return strLists[0] + ''.join(x.title() for x in strLists[1:]) + +def formatMd5(text: str = '', is_random: bool = False, attach: str = ''): + if is_random: + text = f"{time.time()}_{random.randint(10000, 99999)}" + md5 = hashlib.md5(f"{text}_{attach}".encode()).hexdigest() + return md5 + +def re_search(text: str, rule: str) -> str: + match = re.search(rule, text) + if match: + url_path = match.group(1) + return url_path + else: + return "" diff --git a/utils/messages.py b/utils/messages.py new file mode 100644 index 0000000..a70292d --- /dev/null +++ b/utils/messages.py @@ -0,0 +1,18 @@ +import datetime +from colorama import Fore, Style, init + +init() + +def __getTime__(): + time = datetime.datetime.now() + formatTime = time.strftime('%Y-%m-%d %H:%M:%S') + return formatTime + +def sendInfo(message: str): + print(f"[{__getTime__()}] {message}") + +def sendWarn(message): + print(Fore.YELLOW + f"[{__getTime__()}] 提示: {message}" + Style.RESET_ALL) + +def sendError(message: str): + print(Fore.RED + f"[{__getTime__()}] 错误: {message}" + Style.RESET_ALL) diff --git a/utils/uploads.py b/utils/uploads.py new file mode 100644 index 0000000..4a8a73f --- /dev/null +++ b/utils/uploads.py @@ -0,0 +1,27 @@ +import os +import io +import time +import random +import requests + +def upload_image(image2_data, domain: str): + image_stream = io.BytesIO(image2_data) + headers = { + 'Token': 'fangzhouxinghe' + } + data = { + 'domain': domain + } + files = { + 'file': (f'{time.time()}_{random.randint(10000, 99999)}.jpg', image_stream, 'image/jpeg') + } + + response = requests.post('https://fangzhouxinghe.com/uploads.php', data=data, headers=headers, files=files) + response_data = response.json() + + if response_data['code'] != 200: + raise Exception(response_data['msg']) + return response_data['data'] + +if __name__ == '__main__': + print(upload_image(open(r"test\50137.jpg", 'rb').read(), 'test')) diff --git a/utils/wpdata.py b/utils/wpdata.py new file mode 100644 index 0000000..d6a73e9 --- /dev/null +++ b/utils/wpdata.py @@ -0,0 +1,828 @@ +import pandas as pd +from bs4 import BeautifulSoup +import os +import re +import requests +from io import BytesIO +import hashlib +import concurrent.futures +import time + +class Wpdata: + def __init__(self, data_path: str, input_excel: str, goods_url_path: str, upload_workers: int = 10, upload_image_domain:str = "https://www.fzcaiji.com/upload.php", column_name:str = "商品图片*"): + self.__data_path__ = data_path + self.__input_excel__ = input_excel + self.__goods_url_path__ = goods_url_path + + self.__upload_workers__ = upload_workers + self.__upload_image_domain__ = upload_image_domain + self.__column_name__ = column_name + + self.__output_csv__ = f"{input_excel}.csv" + self.__images_txt__ = f"{data_path}_images.txt" + + # 原始数据转换 + def transform_excel_to_csv(self): + def match_variant_count(number, group_df, variant_1, title, text, spu, data_rows, variant_2=None,variant_3=None): + # 提取当前spu分组的价格 + sale_prices = group_df[(group_df['商品属性*'] == 'P') | (group_df['商品属性*'] == 'S')]['商品售价*'].tolist() + origin_prices = group_df[(group_df['商品属性*'] == 'P') | (group_df['商品属性*'] == 'S')]['商品原价'].tolist() + + # 提取对应属性的图片，并统计 + M_images = group_df[(group_df['商品属性*'] == 'M') | (group_df['商品属性*'] == 'S')]['商品图片*'].tolist() + image_list = [url.strip() for url in M_images[0].split(',')] + + group_df + # 获取当前分组的URL + if 'url' in group_df: + url = group_df['url'].tolist()[0] + else: + url = group_df['URL'].tolist()[0] + + match number: + case 3: + # 提取当前spu分组属性为P的值 + all_variants_first = group_df[(group_df['商品属性*'] == 'P')]['款式1'].tolist() + all_variants_second = group_df[(group_df['商品属性*'] == 'P')]['款式2'].tolist() + all_variants_three = group_df[(group_df['商品属性*'] == 'P')]['款式3'].tolist() + + all_variants_count = max(len(all_variants_first), len(all_variants_second), len(all_variants_three)) + + if all_variants_count >= len(image_list): + + # 计算需要扩大的长度，并进行扩大 + expand_length = all_variants_count - len(image_list) + expanded_image_list = image_list + [None] * expand_length + + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + for k, variant in enumerate(all_variants_first): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": variant_1, + "Option1 Value": all_variants_first[k], + "Option2 Name": variant_2, + "Option2 Value": all_variants_second[k], + "Option3 Name": variant_3, + "Option3 Value": all_variants_second[k], + "Variant SKU": f"{spu}{str(k)}", + "Variant Grams": "0", + "Variant Inventory Tracker": "shopify", + "Variant Inventory Qty": "1000", + "Variant Inventory Policy": "continue", + "Variant Fulfillment Service": "manual", + "Variant Price": sale_prices[k], + "Variant Compare At Price": origin_prices[k], + "Variant Requires Shipping": "TRUE", + "Variant Taxable": "TRUE", + "Variant Barcode": "", + "Image Src": expanded_image_list[k], + "Image Position": (image_counter := image_counter + 1) if expanded_image_list[ + k] is not None else "", # 海象运算符 + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if all_variants_first[k] else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + + else: # 变体少于图片数量 + # 计算扩大长度 + expand_length = len(image_list) - all_variants_count + + expanded_all_variants_first_count = all_variants_first + [None] * expand_length + expanded_all_variants_two_count = all_variants_second + [None] * expand_length + expanded_all_variants_three_count = all_variants_three + [None] * expand_length + # 扩大价格长度（为了统一价格） + expanded_sale_prices = sale_prices + [None] * expand_length + expanded_origin_prices = origin_prices + [None] * expand_length + + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + + for k, variant in enumerate(image_list): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": variant_1 if expanded_all_variants_first_count[k] is not None else "", + "Option1 Value": expanded_all_variants_first_count[k], + "Option2 Name": variant_2 if expanded_all_variants_two_count[k] is not None else "", + "Option2 Value": expanded_all_variants_two_count[k], + "Option3 Name": variant_3 if expanded_all_variants_three_count[k] is not None else "", + "Option3 Value": expanded_all_variants_three_count[k], + "Variant SKU": f"{spu}{str(k)}" if expanded_all_variants_first_count[k] is not None else "", + "Variant Grams": "0" if expanded_all_variants_first_count[k] is not None else "", + "Variant Inventory Tracker": "shopify" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Inventory Qty": "1000" if expanded_all_variants_first_count[k] is not None else "", + "Variant Inventory Policy": "continue" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Fulfillment Service": "manual" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Price": expanded_sale_prices[k], + "Variant Compare At Price": expanded_origin_prices[k], + "Variant Requires Shipping": "TRUE" if expanded_all_variants_first_count[k] is not None else "", + "Variant Taxable": "TRUE" if expanded_all_variants_first_count[k] is not None else "", + "Variant Barcode": "", + "Image Src": image_list[k], + "Image Position": (image_counter := image_counter + 1) if image_list[k] is not None else "", + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if k == 0 else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + + case 2: + # 提取当前spu分组属性为P的值 + all_variants_first = group_df[(group_df['商品属性*'] == 'P')]['款式1'].tolist() + all_variants_second = group_df[(group_df['商品属性*'] == 'P')]['款式2'].tolist() + + all_variants_count = max(len(all_variants_first), len(all_variants_second)) + + if all_variants_count >= len(image_list): + + # 计算需要扩大的长度，并进行扩大 + expand_length = all_variants_count - len(image_list) + expanded_image_list = image_list + [None] * expand_length + + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + for k, variant in enumerate(all_variants_first): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": variant_1, + "Option1 Value": all_variants_first[k], + "Option2 Name": variant_2, + "Option2 Value": all_variants_second[k], + "Option3 Name": "", + "Option3 Value": "", + "Variant SKU": f"{spu}{str(k)}", + "Variant Grams": "0", + "Variant Inventory Tracker": "shopify", + "Variant Inventory Qty": "1000", + "Variant Inventory Policy": "continue", + "Variant Fulfillment Service": "manual", + "Variant Price": sale_prices[k], + "Variant Compare At Price": origin_prices[k], + "Variant Requires Shipping": "TRUE", + "Variant Taxable": "TRUE", + "Variant Barcode": "", + "Image Src": expanded_image_list[k], + "Image Position": (image_counter := image_counter + 1) if expanded_image_list[ + k] is not None else "", # 海象运算符 + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if all_variants_first[k] else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + + else: # 变体少于图片数量 + # 计算扩大长度 + expand_length = len(image_list) - all_variants_count + + expanded_all_variants_first_count = all_variants_first + [None] * expand_length + expanded_all_variants_two_count = all_variants_second + [None] * expand_length + + # 扩大价格长度（为了统一价格） + expanded_sale_prices = sale_prices + [None] * expand_length + expanded_origin_prices = origin_prices + [None] * expand_length + + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + + for k, variant in enumerate(image_list): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": variant_1 if expanded_all_variants_first_count[k] is not None else "", + "Option1 Value": expanded_all_variants_first_count[k], + "Option2 Name": variant_2 if expanded_all_variants_two_count[k] is not None else "", + "Option2 Value": expanded_all_variants_two_count[k], + "Option3 Name": "", + "Option3 Value": "", + "Variant SKU": f"{spu}{str(k)}" if expanded_all_variants_first_count[k] is not None else "", + "Variant Grams": "0" if expanded_all_variants_first_count[k] is not None else "", + "Variant Inventory Tracker": "shopify" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Inventory Qty": "1000" if expanded_all_variants_first_count[k] is not None else "", + "Variant Inventory Policy": "continue" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Fulfillment Service": "manual" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Price": expanded_sale_prices[k], + "Variant Compare At Price": expanded_origin_prices[k], + "Variant Requires Shipping": "TRUE" if expanded_all_variants_first_count[k] is not None else "", + "Variant Taxable": "TRUE" if expanded_all_variants_first_count[k] is not None else "", + "Variant Barcode": "", + "Image Src": image_list[k], + "Image Position": (image_counter := image_counter + 1) if image_list[k] is not None else "", + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if k == 0 else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + + case 1: + all_variants = group_df[(group_df['商品属性*'] == 'P')]['款式1'].tolist() + + all_variants_count = len(all_variants) + + if all_variants_count >= len(image_list): + # 计算扩大长度 + expand_length = all_variants_count - len(image_list) + expanded_list = image_list + [None] * expand_length + + # 扩大价格 + expanded_sale_prices = sale_prices + [None] * expand_length + expanded_origin_prices = origin_prices + [None] * expand_length + + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + for k, variant in enumerate(all_variants): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": variant_1, + "Option1 Value": all_variants[k], + "Option2 Name": "", + "Option2 Value": "", + "Option3 Name": "", + "Option3 Value": "", + "Variant SKU": f"{spu}{str(k)}", + "Variant Grams": "0", + "Variant Inventory Tracker": "shopify", + "Variant Inventory Qty": "1000", + "Variant Inventory Policy": "continue", + "Variant Fulfillment Service": "manual", + "Variant Price": expanded_sale_prices[k], + "Variant Compare At Price": expanded_origin_prices[k], + "Variant Requires Shipping": "TRUE", + "Variant Taxable": "TRUE", + "Variant Barcode": "", + "Image Src": expanded_list[k], + "Image Position": (image_counter := image_counter + 1) if expanded_list[k] is not None else "", + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if all_variants[k] else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + + else: # 变体少于图片数量 + # 计算扩大长度 + expand_length = len(image_list) - all_variants_count + expanded_all_variants_first_count = all_variants + [None] * expand_length + + # 扩大价格 + expanded_sale_prices = sale_prices + [None] * expand_length + expanded_origin_prices = origin_prices + [None] * expand_length + + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + + for k, variant in enumerate(image_list): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": variant_1 if expanded_all_variants_first_count[k] is not None else "", + "Option1 Value": expanded_all_variants_first_count[k], + "Option2 Name": "", + "Option2 Value": "", + "Option3 Name": "", + "Option3 Value": "", + "Variant SKU": f"{spu}{str(k)}" if expanded_all_variants_first_count[k] is not None else "", + "Variant Grams": "0" if expanded_all_variants_first_count[k] is not None else "", + "Variant Inventory Tracker": "shopify" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Inventory Qty": "1000" if expanded_all_variants_first_count[k] is not None else "", + "Variant Inventory Policy": "continue" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Fulfillment Service": "manual" if expanded_all_variants_first_count[ + k] is not None else "", + "Variant Price": expanded_sale_prices[k], + "Variant Compare At Price": expanded_origin_prices[k], + "Variant Requires Shipping": "TRUE" if expanded_all_variants_first_count[k] is not None else "", + "Variant Taxable": "TRUE" if expanded_all_variants_first_count[k] is not None else "", + "Variant Barcode": "", + "Image Src": image_list[k], + "Image Position": (image_counter := image_counter + 1) if image_list[k] is not None else "", + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if k == 0 else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + + case 0: + Handle = title[0].replace(' ', '-').replace('/', '-').lower() + image_counter = 0 + for k, variant in enumerate(image_list): + rows = { + "Handle": Handle, + "Title": title[0] if k == 0 else "", + "Body (HTML)": text if k == 0 else "", + "Vendor": "", + "Type": "", + "Tags": "", + "Published": "TRUE" if k == 0 else "", + "Option1 Name": "variant", + "Option1 Value": "default", + "Option2 Name": "", + "Option2 Value": "", + "Option3 Name": "", + "Option3 Value": "", + "Variant SKU": f"{spu}" if k == 0 else "", + "Variant Grams": "0" if k == 0 else "", + "Variant Inventory Tracker": "shopify" if k == 0 else "", + "Variant Inventory Qty": "1000" if k == 0 else "", + "Variant Inventory Policy": "continue" if k == 0 else "", + "Variant Fulfillment Service": "manual" if k == 0 else "", + "Variant Price": sale_prices[0] if k == 0 else "", + "Variant Compare At Price": origin_prices[0] if k == 0 else "", + "Variant Requires Shipping": "TRUE" if k == 0 else "", + "Variant Taxable": "TRUE" if k == 0 else "", + "Variant Barcode": "", + "Image Src": image_list[k], + "Image Position": (image_counter := image_counter + 1) if image_list[k] is not None else "", + "Image Alt Text": "", + "Gift Card": "FALSE" if k == 0 else "", + "SEO Title": title[0] if k == 0 else "", + "SEO Description": title[0] if k == 0 else "", + "Google Shopping / Google Product Category": "", + "Google Shopping / Gender": "", + "Google Shopping / Age Group": "", + "Google Shopping / MPN": "", + "Google Shopping / AdWords Grouping": "", + "Google Shopping / AdWords Labels": "", + "Google Shopping / Condition": "", + "Google Shopping / Custom Product": "", + "Google Shopping / Custom Label 0": "", + "Google Shopping / Custom Label 测试.txt": "", + "Google Shopping / Custom Label 2": "", + "Google Shopping / Custom Label 3": "", + "Google Shopping / Custom Label 4": "", + "Variant Image": "", + "IsDraft": "", + "Variant Weight Unit": "Kg" if k == 0 else "", + "Variant Tax Code": "", + "Cost per item": "", + "Status": "activate" if k == 0 else "", + "Collection": "", + "url": url + } + data_rows.append(rows) + data_rows = [] + df = pd.read_excel(self.__input_excel__) + + # 以相同商品spu为一组 + grouped = df.groupby('商品spu') + + data_rows = [] + + for spu, group_df in grouped: + print(spu) + group_df = group_df.fillna('') + # 获取商品属性为M或S的商品标题 + title = group_df[(group_df['商品属性*'] == 'M') | (group_df['商品属性*'] == 'S')]['商品标题*'].tolist() + description = group_df[(group_df['商品属性*'] == 'M') | (group_df['商品属性*'] == 'S')]['商品描述'].tolist() + description.append('') + text = BeautifulSoup(description[0], 'html.parser').get_text() + + variant_name_one = group_df[(group_df['商品属性*'] == 'M') | (group_df['商品属性*'] == 'S')]['款式1'].tolist()[ + 0] + variant_name_two = group_df[(group_df['商品属性*'] == 'M') | (group_df['商品属性*'] == 'S')]['款式2'].tolist()[ + 0] + variant_name_three = \ + group_df[(group_df['商品属性*'] == 'M') | (group_df['商品属性*'] == 'S')]['款式3'].tolist()[0] + variant_count = 0 + + if variant_name_one: + variant_count += 1 + if variant_name_two: + variant_count += 1 + if variant_name_three: + variant_count += 1 + + # 统计个数 + match_variant_count(variant_count, group_df, variant_name_one, title, text, spu, data_rows, variant_name_two,variant_name_three) + new_df = pd.DataFrame(data_rows) + new_df.to_csv(self.__output_csv__, index=False, encoding="utf-8") + + return self + + # 数据分割 + def split_csv(self, csv_path: str = "", goods_url_path: str = ""): + if csv_path == "": + csv_path = self.__output_csv__ + if goods_url_path == "": + goods_url_path = self.__goods_url_path__ + + with open(goods_url_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + + dict_all = {} + for i in lines: + url = i.strip() + if url == '': + continue + album = url.rsplit('#', 1)[1] + third_album = album.split('/')[-1].lstrip() # 最后一个专辑 + # 全部存进字典 + if third_album not in dict_all: + dict_all[third_album] = [] + + dict_all[third_album].append(url.rsplit('#',1)[0]) + # print(dict_all) + + # 测试.txt. 读取 CSV 文件 + df = pd.read_csv(csv_path) # 替换为你的 CSV 文件路径 + + output_dir = f"{self.__data_path__}/已分类数据" + os.makedirs(output_dir, exist_ok=True) + # 2. 遍历字典的每个键（分类） + for category, urls in dict_all.items(): + # 3. 筛选出 url 列中包含在当前分类的 URL 的行 + filtered_df = df[df['url'].isin(urls)] + + # 4. 如果筛选后的 DataFrame 不为空，则删除 'url' 列并保存到 CSV + if not filtered_df.empty: + filtered_df = filtered_df.drop(columns=['url']) # 删除 url 列 + output_filename = fr"{output_dir}/{category}.csv" + filtered_df.to_csv(output_filename, index=False) + print(f"已保存 {len(filtered_df)} 条数据到 {output_filename}（已删除 url 列）") + else: + print(f"分类 {category} 无匹配数据，跳过保存") + return self + + # 获取原始图片 + def get_excel_images_to_txt(self): + def split_image_urls(cell: str): + """按 ',http' / ',https' 分割字符串，自动补上 'http'。不校验后缀。""" + if not isinstance(cell, str) or not cell.strip(): + return [] + + s = re.sub(r',\s*(https?://)', r'||SEP||\1', cell.strip(), flags=re.IGNORECASE) + parts = s.split('||SEP||') + + urls = [] + for part in parts: + part = part.strip().lstrip(',').strip() + if not part: + continue + m = re.search(r'https?://', part, re.IGNORECASE) + if m: + part = part[m.start():] + urls.append(part) + return urls + + """从 Excel 中读取指定列，返回每行的图片 URL 列表。""" + df = pd.read_excel(self.__input_excel__) + urls_list = [] + for val in df[self.__column_name__].fillna(''): + urls_list.append(split_image_urls(val)) + + """展开 + 去重 + 保存到 txt 文件""" + all_urls = set() # 用 set 去重 + for row_urls in urls_list: + for url in row_urls: + all_urls.add(url.strip()) + + # 写入文件 + with open(self.__images_txt__, "w", encoding="utf-8") as f: + for url in sorted(all_urls): + f.write(url + "\n") + + print(f"✅ 共提取 {len(all_urls)} 张图片，已保存到 {self.__images_txt__}") + return self + + # 上传原始图片 + def update_images(self): + def upload_image(img_url: str): + """下载并上传单张图片""" + HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", + "Accept": "image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", + "Accept-Encoding": "gzip, deflate, br", + "Referer": "https://www.google.com/" + } + try: + # 通过 requests 下载图片（带代理和请求头） + resp_img = requests.get(img_url, timeout=15, headers=HEADERS) + resp_img.raise_for_status() + img_data = BytesIO(resp_img.content) + + # 生成哈希文件名 + hash_name = hashlib.sha1(img_url.encode("utf-8")).hexdigest() + + # 获取扩展名（安全处理） + ext = img_url.split("?")[0].split(".")[-1].lower() + if len(ext) > 5 or "/" in ext: + ext = "jpg" + + # 上传 + files = {"file": (f"{hash_name}.{ext}", img_data, f"image/{ext}")} + data = {"source_url": img_url} + resp = requests.post(self.__upload_image_domain__, files=files, data=data, timeout=30) + + # 打印结果 + try: + js = resp.json() + print(f"✅ {img_url} -> {js.get('url') or js}") + except Exception: + print(f"❌ {img_url} -> 上传失败: {resp.text[:100]}") + + except Exception as e: + print(f"⚠️ 下载或上传异常: {img_url} -> {e}") + # 读取本地 txt 文件 + with open(self.__images_txt__, "r", encoding="utf-8") as f: + urls = [x.strip() for x in f if x.strip()] + + print(f"共读取到 {len(urls)} 张图片。开始上传...") + + start = time.time() + # 使用线程池并发上传 + with concurrent.futures.ThreadPoolExecutor(max_workers=self.__upload_workers__) as executor: + list(executor.map(upload_image, urls)) + + print(f"\n全部上传完成，用时 {time.time() - start:.2f} 秒。") + + return self + + # 替换原始图片 + def replace_excel_image(self): + """ + 读取 Excel 和 txt，将原图片 URL 替换为新图片 URL，保存到新 Excel + """ + def get_uploaded_image_url_php_style(img_url: str) -> str: + """ + 模拟 PHP 上传脚本的路径生成逻辑： + - hash = sha1(source_url) + - 一级目录 = hash 前 1 字符 + - 二级目录 = hash 第 2 字符 + - 文件名 = hash + 扩展名 + """ + # 计算 sha1 + hash_name = hashlib.sha1(img_url.encode('utf-8')).hexdigest() + + # 取扩展名（与 PHP 一致：从文件名里提取，默认 jpg） + ext = os.path.splitext(img_url.split('?')[0])[1].lstrip('.').lower() + if not ext or len(ext) > 5 or "/" in ext: + ext = 'jpg' + + # 两层目录 + dir1 = hash_name[0] + dir2 = hash_name[1] + + # 拼接最终 URL + new_url = f"{self.__upload_image_domain__}/{dir1}/{dir2}/{hash_name}.{ext}" + + return new_url + + def split_image_urls(cell: str): + """按 ',http' / ',https' 分割字符串，自动补上 'http'。不校验后缀。""" + if not isinstance(cell, str) or not cell.strip(): + return [] + + s = re.sub(r',\s*(https?://)', r'||SEP||\1', cell.strip(), flags=re.IGNORECASE) + parts = s.split('||SEP||') + + urls = [] + for part in parts: + part = part.strip().lstrip(',').strip() + if not part: + continue + m = re.search(r'https?://', part, re.IGNORECASE) + if m: + part = part[m.start():] + urls.append(part) + # cell = ',' + cell + # cell = cell.split(',https://') + # urls = [] + # for image_url in cell: + # if '.svg' in image_url: + # continue + # if image_url == "": + # continue + # urls.append(('https://'+image_url).strip()) + return urls + + # 读取 txt 中的原图片链接 + with open(self.__images_txt__, "r", encoding="utf-8") as f: + original_urls = [line.strip() for line in f if line.strip()] + + # 生成原图片到新图片的映射 + url_mapping = {} + for old_url in original_urls: + new_url = get_uploaded_image_url_php_style(old_url) + url_mapping[old_url] = new_url + + print(f"📋 共加载 {len(url_mapping)} 个图片映射关系") + + # 读取 Excel + df = pd.read_excel(self.__input_excel__) + + # 替换图片列 + replaced_count = 0 + for idx, cell_value in enumerate(df[self.__column_name__].fillna('')): + if not cell_value: + continue + + # 分割出该单元格的所有图片 URL + old_urls = split_image_urls(cell_value) + + # 替换为新 URL + new_urls = [] + for old_url in old_urls: + if old_url in url_mapping: + new_urls.append(url_mapping[old_url]) + replaced_count += 1 + else: + new_urls.append(old_url) # 如果找不到映射，保持原样 + + # 拼接回去（用逗号分隔） + df.at[idx, self.__column_name__] = ','.join(new_urls) + + output_excel = f"{self.__input_excel__}(new_image).xlsx" + # 保存到新 Excel + df.to_excel(output_excel, index=False) + print(f"✅ 替换完成！共替换 {replaced_count} 个图片链接") + print(f"✅ 已保存到 {output_excel}") + self.__input_excel__ = output_excel + + # 一键运行 + def run(self, replace_image: bool = False): + if replace_image: + self.get_excel_images_to_txt().update_images().replace_excel_image() + self.transform_excel_to_csv().split_csv() + +if __name__ == "__main__": + wpdata = Wpdata( + data_path="data", # 已分类数据文件夹保存路径 + input_excel=r"data\jamesavery\info\goods(sort)_2025-12-24.xlsx", # 原始excel路径 + goods_url_path=r"data\jamesavery\urls\goods_2025-12-19.txt_old.txt" # 商品txt路径 + ) + wpdata.run( + # replace_image=True # 是否上传图片 + )