初始化

core/__init__.py

@@ -0,0 +1,3 @@
+from core import types
+
+Config = types.Config()

core/browsers.py

@@ -0,0 +1,25 @@
+import socket
+import random
+from DrissionPage import Chromium, ChromiumOptions
+
+def getBrowsers(chrome_zoom = 1.0, save_path: str = "") -> Chromium:
+    while True:
+        port = random.randint(9000, 65535)
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            if s.connect_ex(('127.0.0.1', port)) != 10061:
+                continue
+        break
+
+    if save_path:
+        options = ChromiumOptions().set_local_port(port).set_user_data_path(save_path)
+    else:
+        options = ChromiumOptions().auto_port()
+    options = options.set_argument('--force-device-scale-factor', str(chrome_zoom))
+    # options = options.set_user_agent('Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)')
+    while True:
+        try:
+            browser = Chromium(options)
+            break
+        except:
+            pass
+    return browser

core/excels.py

@@ -0,0 +1,84 @@
+import csv
+import traceback
+from openpyxl import load_workbook
+from core import types
+from utils import messages
+
+def read_excel_to_dict(excel_path: str, sheet_name: str = 'Sheet1') -> dict[str, types.GoodsInfo]:
+    """
+    从 Excel 文件中读取商品数据，返回 {url: GoodsInfo} 的字典。
+    
+    :param excel_path: Excel 文件路径
+    :param sheet_name: 工作表名称，默认 'Sheet1'
+    :return: dict{GoodsInfo.url: GoodsInfo}
+    """
+    wb = load_workbook(excel_path, data_only=True)
+    ws = wb[sheet_name]
+
+    goods_dict = {}
+
+    # 从第3行开始读取（假设第1、2行为标题/说明）
+    for row in ws.iter_rows(min_row=3, values_only=True):
+        # 过滤空行（如果整行都为空则跳过）
+        if all(cell is None or str(cell).strip() == '' for cell in row):
+            continue
+
+        # 转为字符串列表，None 转为空字符串
+        row_data = [str(cell) if cell is not None else '' for cell in row]
+
+        try:
+            goods = types.GoodsInfo.from_row_data(row_data)
+            if goods.url and (goods.attribute == "M" or goods.attribute == "S"):  # 只有 url 非空才加入字典
+                goods_dict[goods.url] = goods
+            elif goods.url and goods.attribute == "P":
+                if goods.url in goods_dict:
+                    goods_dict[goods.url].p_lists.append(goods)
+        except Exception as e:
+            messages.sendError(f"解析行失败（跳过）: {row_data[:5]}... 错误: {e}\n{traceback.format_exc()}")
+            continue
+    wb.close()
+
+    return goods_dict
+
+def save_lists_to_csv(data: list, save_path: str, template_path: str = ''):
+    header = []
+    if template_path:
+        with open(template_path, 'r', encoding='utf-8-sig') as f:
+            template = csv.reader(f)
+            for row in template:
+                header = list(row)
+        data.insert(0, header)
+    with open(save_path, 'w', encoding='utf-8', newline="") as f:
+        writer = csv.writer(f)
+        writer.writerows(data)
+
+class WorkBook:
+    def __init__(self, template: str, save_path: str, sheet_name: str = 'Sheet1'):
+        wb = load_workbook(template)
+        ws = wb[sheet_name]
+        last_row = ws.max_row
+        if ws.cell(row=last_row, column=1).value is None and "此行导入时不可删除" in str(ws.cell(row=2, column=1).value):
+            start_row = last_row + 1
+        else:
+            while ws.cell(row=last_row, column=1).value is not None or any(ws.cell(row=last_row, column=c).value for c in range(2, 30)):
+                last_row += 1
+            start_row = last_row
+        
+        self.wb = wb
+        self.ws = ws
+        self.start_row = start_row
+        self.save_path = save_path
+    
+    def add_row_of_list(self, data: list):
+        for col_idx, value in enumerate(data, start=1):
+            try:
+                self.ws.cell(row=self.start_row, column=col_idx, value=value)
+            except:
+                pass
+        self.start_row += 1
+    
+    def save(self):
+        self.wb.save(self.save_path)
+    
+    def close(self):
+        self.wb.close()

core/spiders.py

@@ -0,0 +1,31 @@
+import os
+from core import browsers
+
+class Spiders:
+    def __init__(self, project_name: str, bitch: int = 5, worker_num: int = 1, switch_clash: bool = False, reflush_browser: bool = False):
+        self.bitch = bitch
+        self.worker_num = worker_num
+        self.switch_clash = switch_clash
+        self.reflush_browsers = reflush_browser
+
+        self.__data_path__ = f"data/{project_name}"
+        self.__database_path__ = f"database/{project_name}.db"
+        self.__urls_folder__ = f"{self.__data_path__}/urls"
+        self.__excels_folder__ = f"{self.__data_path__}/excels"
+        self.__errors_folder__ = f"{self.__data_path__}/errors"
+
+        if os.path.exists(self.__data_path__) == False:
+            os.makedirs(self.__data_path__)
+            os.makedirs(self.__urls_folder__)
+            os.makedirs(self.__excels_folder__)
+            os.makedirs(self.__errors_folder__)
+        
+        self.browser = browsers.getBrowsers()
+        self.browser.set.timeouts(3)
+        self.tab = self.browser.get_tab(0)
+        self.tab.set.window.max()
+    
+    def reflush_browser(self):
+        self.browser.quit()
+        self.browser = browsers.getBrowsers()
+        self.browser.set.timeouts(3)

core/types.py

@@ -0,0 +1,263 @@
+import orjson
+from bs4 import BeautifulSoup
+from pydantic import BaseModel
+
+class Config(BaseModel):
+    home_mode: bool = True
+    skip_category: bool = True
+
+class GoodsInfo(BaseModel):
+    """
+    商品类(包含处理)
+
+    :param spu: [必填]
+    :param title: 商品标题 [必填]
+    :param brand: 品牌
+    :param desc: 简介
+    :param attr: 属性 S|M
+    :param attr_items: 变体[名称]列表 [Color, Size]
+    :param price: 价格
+    :param old_price: 原价
+    :param images: 图片列表
+    :param url: 主商品链接
+    :param p_urls: 变体链接列表
+    :param p_lists: 变体类列表
+    :param other_data: 其他参数
+    
+    """
+    class GoodsInfoP(BaseModel):
+        """
+        变体类
+
+        :param attr_items: 变体[值]列表 [red, 30]
+
+        """
+        attr_items: list = [] # 变体名称1,2,3
+        price: float = 0.00
+        old_price: float = 0.00
+        images: list[str] = []
+        url: str = '' # 商品链接
+        other_data: dict = {} # 其他参数
+
+    spu: str
+    title: str
+    brand: str = '' # 品牌
+    desc: str = '' # 简介
+    attr: str = 'S' # 属性 S M
+    attr_items: list = [] # 变体名称1,2,3
+    price: float = 0.00
+    old_price: float = 0.00
+    images: list[str] = []
+    url: str = '' # 商品链接
+    p_urls: list[str] = [] # 变体链接列表(筛选用,可略)
+    p_lists: list[GoodsInfoP] = [] # 变体列表
+    other_data: dict = {} # 其他参数
+
+    def to_db_data(self):
+        """转为goods_info.add_goods()所需格式"""
+
+        main_info = {
+            'spu': self.spu,
+            'title': self.title,
+            'brand': self.brand,
+            'attr': self.attr,
+            'desc': self.desc,
+            'price': self.price,
+            'old_price': self.old_price,
+            'images': orjson.dumps(self.images),
+            'url': self.url,
+            'other_data': orjson.dumps(self.other_data)
+        }
+        for index, item_name in enumerate(self.attr_items):
+            if item_name:
+                main_info[f"item{index+1}"] = item_name
+        
+        son_infos = []
+        for p_info in self.p_lists:
+            item = {
+                'spu': self.spu,
+                'price': p_info.price,
+                'old_price': p_info.old_price,
+                'images': orjson.dumps(p_info.images),
+                'url': p_info.url,
+                'other_data': orjson.dumps(p_info.other_data)
+            }
+            p_info.attr_items += [None, None]
+            for index, item_name in enumerate(p_info.attr_items[:3]):
+                if item_name:
+                    item[f"item{index+1}"] = item_name
+            son_infos.append(item)
+
+        return main_info, son_infos
+
+    def to_shopyy_row_data(self):
+        """转为shopyy行数据"""
+        prices = [self.price, self.old_price]
+        for p_info in self.p_lists:
+            prices.append(p_info.price)
+            prices.append(p_info.old_price)
+        old_price = max(prices)
+
+        rows_data = []
+        rows_data.append([
+            "",  # 商品ID（系统生成，留空）
+            "",
+            f"{self.brand} {self.title}" if self.brand != "" else self.title,
+            self.attr,
+            "",
+            "",
+            self.desc,
+            "",
+            "",
+            "",
+            "",
+            "",
+            "",
+            "Y",
+            "Y",
+            self.spu,
+            0,
+            "N",
+            2,
+            self.brand,  # 专辑名称
+            "",
+            "",
+            *(self.attr_items[:3] + [''] * (3 - len(self.attr_items)))[:3],  # 款式1/2/3
+            self.price,
+            old_price,
+            "",
+            "",
+            "",
+            50,
+            '',  # 变体备注1（未在模型中定义，可扩展）
+            '',  # 变体备注2
+            ','.join(self.images) if self.images else '',
+            self.url,
+            ','.join(self.p_urls) if self.p_urls else ''
+        ])
+
+        p_index = []
+        for p_info in self.p_lists:
+            if p_info.attr_items in p_index:
+                continue
+            p_index.append(p_info.attr_items)
+            rows_data.append([
+                "",  # 商品ID（系统生成，留空）
+                "",
+                f"{self.brand} {self.title}" if self.brand != "" else self.title,
+                "P",
+                "",
+                "",
+                self.desc,
+                "",
+                "",
+                "",
+                "",
+                "",
+                "",
+                "Y",
+                "Y",
+                self.spu,
+                0,
+                "N",
+                2,
+                self.brand,  # 专辑名称
+                "",
+                "",
+                *(p_info.attr_items[:3] + [''] * (3 - len(p_info.attr_items)))[:3],  # 款式1/2/3
+                p_info.price,
+                old_price,
+                "",
+                "",
+                "",
+                50,
+                '',  # 变体备注1（未在模型中定义，可扩展）
+                '',  # 变体备注2
+                ','.join(p_info.images) if p_info.images else '',
+                p_info.url,
+                ""
+            ])
+        return rows_data
+
+    def to_woo_row_data(self, category_name):
+        """转为woo行数据"""
+        def get_lists(attr = 'variable',spu = '', price = '', old_price = '', item_name1: str = '', item_value1: str = '', item_name2: str = '', item_value2: str = '', item_name3: str = '', item_value3: str = '', images: list = []):
+            return [
+                attr,
+                self.spu if attr == 'variable' or attr == 'simple' else "",
+                self.title if attr == 'variable' or attr == 'simple' else "",
+                1,
+                desc if attr == 'variable' or attr == 'simple' else "",
+                1,
+                100,
+                price,
+                old_price,
+                category_name if attr == 'variable' or attr == 'simple' else "",
+                ','.join(images) if images else '',
+                self.spu if attr == 'variation' else "",
+                item_name1,
+                item_value1,
+                1,
+                item_name2,
+                item_value2,
+                1,
+                item_name3,
+                item_value3,
+                1,
+            ]
+        
+        desc = BeautifulSoup(self.desc, 'html.parser').get_text()
+        rows_data = []
+        if self.attr == 'S':
+            rows_data.append(get_lists(
+                attr='simple',
+                spu=self.spu,
+                price=self.price,
+                old_price=self.old_price,
+                images=self.images
+            ))
+        else:
+            
+            item_values1 = set()
+            item_values2 = set()
+            item_values3 = set()
+            for p_info in self.p_lists:
+                item_value1 = p_info.attr_items[0] if len(p_info.attr_items) >= 1 else ""
+                item_value2 = p_info.attr_items[1] if len(p_info.attr_items) >= 2 else ""
+                item_value3 = p_info.attr_items[2] if len(p_info.attr_items) >= 3 else ""
+
+                item_value1 = item_value1.replace(',', '，').replace('|', ' ')
+                item_values1.add(item_value1)
+                if item_value2:
+                    item_value2 = item_value2.replace(',', '，').replace('|', ' ')
+                    item_values2.add(item_value2)
+                if item_value3:
+                    item_value3 = item_value3.replace(',', '，').replace('|', ' ')
+                    item_values3.add(item_value3)
+
+                rows_data.append(get_lists(
+                    attr='variation',
+                    spu='',
+                    price=p_info.price,
+                    old_price=p_info.old_price,
+                    images=p_info.images,
+                    item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "",
+                    item_value1=item_value1,
+                    item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "",
+                    item_value2=item_value2,
+                    item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "",
+                    item_value3=item_value3
+                ))
+            rows_data.insert(0, get_lists(
+                spu=self.spu,
+                price='',
+                old_price='',
+                images=self.images,
+                item_name1=self.attr_items[0] if len(self.attr_items) >= 1 else "",
+                item_value1='|'.join(item_values1),
+                item_name2=self.attr_items[1] if len(self.attr_items) >= 2 else "",
+                item_value2='|'.join(item_values2),
+                item_name3=self.attr_items[2] if len(self.attr_items) >= 3 else "",
+                item_value3='|'.join(item_values3)
+            ))
+        return rows_data