diff --git a/lib/module/__pycache__/agatameble.cpython-312.pyc b/lib/module/__pycache__/agatameble.cpython-312.pyc deleted file mode 100644 index 2c88467..0000000 Binary files a/lib/module/__pycache__/agatameble.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/ancientnutrition.cpython-312.pyc b/lib/module/__pycache__/ancientnutrition.cpython-312.pyc deleted file mode 100644 index 1ea323f..0000000 Binary files a/lib/module/__pycache__/ancientnutrition.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/bricomarche.cpython-312.pyc b/lib/module/__pycache__/bricomarche.cpython-312.pyc deleted file mode 100644 index b171261..0000000 Binary files a/lib/module/__pycache__/bricomarche.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/condom69.cpython-312.pyc b/lib/module/__pycache__/condom69.cpython-312.pyc deleted file mode 100644 index 4013302..0000000 Binary files a/lib/module/__pycache__/condom69.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/dk_one.cpython-312.pyc b/lib/module/__pycache__/dk_one.cpython-312.pyc deleted file mode 100644 index 28b52cc..0000000 Binary files a/lib/module/__pycache__/dk_one.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/example.cpython-312.pyc b/lib/module/__pycache__/example.cpython-312.pyc deleted file mode 100644 index a3e80c2..0000000 Binary files a/lib/module/__pycache__/example.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/friskybusiness.cpython-312.pyc b/lib/module/__pycache__/friskybusiness.cpython-312.pyc deleted file mode 100644 index 45aecbb..0000000 Binary files a/lib/module/__pycache__/friskybusiness.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/peek_cloppenburg.cpython-312.pyc b/lib/module/__pycache__/peek_cloppenburg.cpython-312.pyc deleted file mode 100644 index e700ddb..0000000 Binary files a/lib/module/__pycache__/peek_cloppenburg.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/peek_cloppenburg_nl.cpython-312.pyc b/lib/module/__pycache__/peek_cloppenburg_nl.cpython-312.pyc deleted file mode 100644 index 6de47d9..0000000 Binary files a/lib/module/__pycache__/peek_cloppenburg_nl.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/ssfshop.cpython-312.pyc b/lib/module/__pycache__/ssfshop.cpython-312.pyc deleted file mode 100644 index fc05670..0000000 Binary files a/lib/module/__pycache__/ssfshop.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/taketoys.cpython-312.pyc b/lib/module/__pycache__/taketoys.cpython-312.pyc deleted file mode 100644 index c72f3b9..0000000 Binary files a/lib/module/__pycache__/taketoys.cpython-312.pyc and /dev/null differ diff --git a/lib/module/__pycache__/tfashion.cpython-312.pyc b/lib/module/__pycache__/tfashion.cpython-312.pyc deleted file mode 100644 index efab6ad..0000000 Binary files a/lib/module/__pycache__/tfashion.cpython-312.pyc and /dev/null differ diff --git a/lib/module/agatameble.py b/lib/module/agatameble.py deleted file mode 100644 index 47bf346..0000000 --- a/lib/module/agatameble.py +++ /dev/null @@ -1,176 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "agatameble" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 1, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.agatameble.pl/') - one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele.ele('@tag()=a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - three_menu_eles = two_menu_ele.eles('xpath=ul/li') - for three_menu_ele in three_menu_eles: - three_link_ele = three_menu_ele.ele('@tag()=a') - three_url = three_link_ele.attr('href') - three_name = three_link_ele.text.replace('/', '-') - category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - self.tab.get(f"{category_url}") - goods_urls = [] - - while True: - try: - self.tab.scroll.to_bottom() - self.tab.ele('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/div/div/a').click() - except: - break - - goods_eles = self.tab.eles('xpath=//*[@id="contentContainer"]/div/article/div[2]/div[2]/section/div/div/div') - for goods_ele in goods_eles: - goods_link_ele = goods_ele.ele('xpath=a') - goods_urls.append(goods_link_ele.attr('href')) - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - - tab.ele('xpath=//*[@id="contentContainer"]/form/div[1]/section/div/div[2]/button[1]').click() - image_eles = tab.eles('xpath=//*[@id="root"]/div/section/div[2]/div/aside/div[2]/div/div/div/div[1]/div/div/div/div') - for image_ele in image_eles: - image_link_ele = image_ele.ele('xpath=div/img[2]') - image_urls.append(image_link_ele.attr('src').replace('width=', '').replace('height=', '')) - - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - brand = '' - try: - brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text - except: - pass - if brand == '': - title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text - else: - title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text - - desc = "" - desc_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[1]/div/div')[:3] - for desc_ele in desc_eles: - desc += formats.clean_html(desc_ele.html) - - price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - color_urls = [url] - color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li') - for color_ele in color_eles[1:]: - color_link_ele = color_ele.ele('xpath=a') - color_urls.append(color_link_ele.attr('href')) - if len(color_urls) > 1: - m_data = ["Color"] - - if len(color_urls) > 1: - for color_url in color_urls: - color_images = main_images - if color_url != url: - tab.get(color_url) - if brand == '': - title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text - else: - title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text - color_images = get_image_urls() - - price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") - old_price = price - - try: - item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text - except: - continue - attr_items.append({ - 'url': color_url, - 'items': [item_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - - if len(attr_items) == 0: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - url=attr_item['url'], - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'], - )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - goods_info.p_urls = color_urls - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/ancientnutrition.py b/lib/module/ancientnutrition.py deleted file mode 100644 index 41970d2..0000000 --- a/lib/module/ancientnutrition.py +++ /dev/null @@ -1,325 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "ancientnutrition" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 1, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://ancientnutrition.com/') - one_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[4]/div[1]/div[1]/div') - for index, one_menu_ele in enumerate(one_menu_eles[1:]): - one_link_ele = one_menu_ele.ele('xpath=div/a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - print(index+2) - two_menu_eles = self.tab.eles(f'xpath=//*[@id="__next"]/div[4]/div[{index+2}]/div/div/div[2]/div/a') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - # three_menu_eles = two_menu_ele.eles('xpath=ul/li') - # for three_menu_ele in three_menu_eles: - # three_link_ele = three_menu_ele.ele('@tag()=a') - # three_url = three_link_ele.attr('href') - # three_name = three_link_ele.text.replace('/', '-') - # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - self.tab.get(f"{category_url}") - goods_urls = [] - - while True: - try: - self.tab.scroll.to_bottom() - self.tab.ele('text=Load More').click() - except: - break - - goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[9]/div/div[2]/div[2]/div') - for goods_ele in goods_eles: - try: - goods_link_ele = goods_ele.ele('@tag()=a') - except Exception as e: - continue - goods_urls.append(goods_link_ele.attr('href')) - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - - image_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[1]/div/div[2]/div/div[1]/div') - for image_ele in image_eles: - image_link_ele = image_ele.ele('xpath=div/span/img') - image_urls.append(image_link_ele.attr('src').replace('_150x150', '_1450x1450')) - - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - brand = '' - # try: - # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text - # except: - # pass - # if brand == '': - # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text - # else: - title = tab.ele('@tag()=h1').text - - desc = formats.clean_html(tab.ele('#productDescription').html) - - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - try: - tab.ele('text=Select Flavor').click() - m_data.append('Flavor') - except: - pass - try: - tab.ele('text=Select Size').click() - m_data.append('Size') - except: - pass - try: - tab.ele('text=Select Quantity').click() - m_data.append('Quantity') - except: - pass - - if 'Flavor' in m_data: - tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click() - color_eles = tab.eles('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div[2]/div') - color_xpaths = [] - for color_ele in color_eles: - color_xpaths.append(color_ele.xpath) - - for color_xpath in color_xpaths: - while True: - tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[3]/div/div').click() - try: - color_ele = tab.ele(f"xpath={color_xpath}") - color_name = color_ele.text - break - except: - tab.get(url) - color_ele.click() - tab.wait(3) - color_images = get_image_urls() - if 'Size' in m_data: - try: - size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}") - except: - size_name = 'Default' - if 'Quantity' in m_data: - quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") - for quantity_index, quantity_ele in enumerate(quantity_eles): - try: - quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") - quantity_name = quantity_ele.text - except: - continue - quantity_ele.click() - tab.wait(2) - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [color_name, size_name, quantity_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [color_name, size_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - size_xpaths = [] - for size_ele in size_eles: - try: - size_xpaths.append(size_ele.xpath) - except: - continue - - for size_xpath in size_xpaths: - try: - size_ele = tab.ele(f"xpath={size_xpath}") - size_name = size_ele.text - except: - continue - size_ele.click() - tab.wait(3) - color_images = get_image_urls() - if 'Quantity' in m_data: - quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") - for quantity_index, quantity_ele in enumerate(quantity_eles): - try: - quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") - quantity_name = quantity_ele.text - except: - continue - quantity_ele.click() - tab.wait(2) - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [color_name, size_name, quantity_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [color_name, size_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [color_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - color_images = [] - if 'Size' in m_data: - size_eles = tab.eles(f"xpath={tab.ele('text=Select Size').xpath.replace('/span[1]', '/div/div')}") - size_xpaths = [] - for size_ele in size_eles: - try: - size_xpaths.append(size_ele.xpath) - except: - continue - - for size_xpath in size_xpaths: - try: - size_ele = tab.ele(f"xpath={size_xpath}") - size_name = size_ele.text - except: - continue - size_ele.click() - tab.wait(2) - color_images = get_image_urls() - if 'Quantity' in m_data: - quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") - for quantity_index, quantity_ele in enumerate(quantity_eles): - try: - quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") - quantity_name = quantity_ele.text - except: - continue - quantity_ele.click() - tab.wait(2) - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [size_name, quantity_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - if 'Quantity' in m_data: - quantity_eles = tab.eles(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', '/div/div')}") - for quantity_index, quantity_ele in enumerate(quantity_eles): - try: - quantity_ele = tab.ele(f"xpath={tab.ele('text=Select Quantity').xpath.replace('/span[1]', f'/div/div[{quantity_index+1}]')}") - quantity_name = quantity_ele.text - except: - continue - quantity_ele.click() - tab.wait(2) - price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[1]').text.replace('$', '')) - old_price = float(tab.ele('xpath=//*[@id="__next"]/div[8]/div[1]/div/div[2]/div[2]/div/div[1]/div/span/span[2]/span[1]').text.replace('$', '')) - attr_items.append({ - 'url': tab.url, - 'items': [quantity_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - - if len(attr_items) == 0: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - url=attr_item['url'], - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'], - )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/bricomarche.py b/lib/module/bricomarche.py deleted file mode 100644 index 97c616d..0000000 --- a/lib/module/bricomarche.py +++ /dev/null @@ -1,201 +0,0 @@ -from core import spiders, types -from utils import formats, messages -from DrissionPage.common import Keys - -class SpiderModule(spiders.Spiders): - project_name = "bricomarche" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 5, - worker_num = 1, - switch_clash = False, - reflush_browser = False - ) - self.tab.get('https://www.bricomarche.pl/') - # input('wait:') - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.bricomarche.pl/') - one_menu_eles = self.tab.eles('xpath=//*[@id="root"]/main/div[5]/div[3]/nav/div') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele.ele('@tag()=a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=div/div[1]/div') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - three_menu_eles = two_menu_ele.eles('xpath=ul/li') - for three_menu_ele in three_menu_eles: - three_link_ele = three_menu_ele.ele('@tag()=a') - three_url = three_link_ele.attr('href') - three_name = three_link_ele.text.replace('/', '-') - category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - page = 1 - limit = 100 - self.tab.get(f"{category_url}?page={page}&limit={limit}") - if self.tab.ele('@tag()=h2').text == '执行安全验证': - self.tab.wait(10) - self.tab.actions.type((Keys.TAB, Keys.SPACE)) - goods_urls = [] - - while True: - try: - self.tab.scroll.to_bottom() - goods_eles = self.tab.eles('xpath=//*[@id="spark"]/div[2]/div/section/div/div/main/div[2]/section/div') - for goods_ele in goods_eles: - goods_link_ele = goods_ele.ele('@tag()=a') - goods_urls.append(goods_link_ele.attr('href')) - if len(goods_eles) == 0: - break - except: - break - page += 1 - self.tab.get(f"{category_url}?page={page}&limit={limit}") - if self.tab.ele('@tag()=h2').text == '执行安全验证': - self.tab.wait(10) - self.tab.actions.type((Keys.TAB, Keys.SPACE)) - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - - image_eles = tab.eles('xpath=//*[@id="spark"]/div[2]/div/main/div/div[2]/section[1]/div/div[1]/div/div/div/div/div') - for image_ele in image_eles: - image_link_ele = image_ele.ele('xpath=button/div/img') - image_urls.append(image_link_ele.attr('src').replace('gallery_100_91', 'gallery')) - - return image_urls - - tab = self.browser.new_tab(url) - while True: - try: - if tab.ele('@tag()=h2').text == '执行安全验证': - tab.wait(10) - tab.actions.type((Keys.TAB, Keys.SPACE)) - tab.wait(5) - else: - break - except: - break - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - brand = '' - # try: - # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text - # except: - # pass - - title = tab.ele('@tag()=h1', timeout=10).text - - desc = "" - try: - desc = formats.clean_html(tab.ele('.attributes').html) - except: - pass - - # try: - price_ele = tab.ele('.main-price') - price = float(f"{price_ele.ele('.whole').text}{price_ele.ele('.cents').text.replace(' zł', '')}") - # except: - # price = 0.0 - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - # color_urls = [url] - # color_eles = tab.eles('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li') - # for color_ele in color_eles[1:]: - # color_link_ele = color_ele.ele('xpath=a') - # color_urls.append(color_link_ele.attr('href')) - # if len(color_urls) > 1: - # m_data = ["Color"] - - # if len(color_urls) > 1: - # for color_url in color_urls: - # color_images = main_images - # if color_url != url: - # tab.get(color_url) - # if brand == '': - # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text - # else: - # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[2]').text - # color_images = get_image_urls() - - # price = float(f"{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[1]').text}.{tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/section[2]/div[1]/div[2]/span[3]').text}") - # old_price = price - - # try: - # item_name = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[2]/div/ul/li[1]/div[2]/p').text - # except: - # continue - # attr_items.append({ - # 'url': color_url, - # 'items': [item_name], - # 'price': price, - # 'old_price': old_price, - # 'images': color_images - # }) - - if len(attr_items) == 0: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - url=attr_item['url'], - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'], - )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - # goods_info.p_urls = color_urls - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/condom69.py b/lib/module/condom69.py deleted file mode 100644 index 51a6893..0000000 --- a/lib/module/condom69.py +++ /dev/null @@ -1,136 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "condom69" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 5, - switch_clash = False, - reflush_browser = False - ) - self.tab.get('https://www.condom69.net/') - # input('wait:') - - def get_category_urls(self) -> list[str]: - category_urls = [] - - one_menu_eles = self.tab.eles('xpath=//*[@id="nav-one"]/li') - for one_menu_ele in one_menu_eles[:-1]: - one_link_ele = one_menu_ele.ele('xpath=a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=div/ul/li', timeout=1) - for two_index, two_menu_ele in enumerate(two_menu_eles): - two_link_ele = two_menu_ele.ele('xpath=div/a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - # try: - # three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') - # except: - # continue - # for three_menu_ele in three_menu_eles: - # three_link_ele = three_menu_ele.ele('@tag()=a') - # three_url = three_link_ele.attr('href') - # three_name = three_link_ele.text.replace('/', '-') - # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - page = 1 - self.tab.get(f"{category_url}&pg={page}") - - #####获取页数#### - - ################# - - goods_urls = [] - url_index = {} - while True: - self.tab.scroll.to_bottom() - goods_eles = self.tab.eles('.divProdItemImg2Inner') - page_urls = [] - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('xpath=a').attr('href')}" - if goods_url not in url_index: - page_urls.append(goods_url) - url_index[goods_url] = None - if len(page_urls) > 0: - goods_urls += page_urls - else: - break - page += 1 - self.tab.get(f"{category_url}&pg={page}") - - messages.sendInfo(f"{len(goods_urls)} 个") - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - image_eles = tab.eles('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[1]/div[2]/div') - - for image_ele in image_eles: - image_url = image_ele.ele('xpath=div/div/a/img').attr('src') - image_urls.append(image_url) - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - h1_ele = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[1]') - brand = '' - try: - brand = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[1]/table/tbody/tr/td[3]/a').text - except: - pass - title = h1_ele.text - - desc = "" - try: - desc = formats.clean_html(tab.ele('.ctl00_cphContent_ucUsrIndProduct_pnlIndProdDesc').html) - except: - pass - - price = tab.ele('xpath=/html/body/form/div[3]/div[2]/div/div[2]/div/div/div[2]/div[2]/div[3]/div/div[2]/div/div/div[1]/div[2]/div/div[2]/div[3]/div[1]/div[2]/span').text.replace(',', '') - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/dk_one.py b/lib/module/dk_one.py deleted file mode 100644 index e037a4f..0000000 --- a/lib/module/dk_one.py +++ /dev/null @@ -1,210 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "dk_one" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 1, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.dk_one.com/') - one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele.ele('@tag()=a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - if one_name == '라이프': - continue - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - if two_name == '메인': - continue - category_urls.append(f"{two_url}#{one_name}/{two_name}") - two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - if two_name == '메인': - continue - category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}") - - # three_menu_eles = two_menu_ele.eles('xpath=ul/li') - # for three_menu_ele in three_menu_eles: - # three_link_ele = three_menu_ele.ele('@tag()=a') - # three_url = three_link_ele.attr('href') - # three_name = three_link_ele.text.replace('/', '-') - # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - self.tab.get(category_url) - goods_urls = [] - goods_eles = self.tab.eles('xpath=//*[@id="prodList"]/div') - for goods_ele in goods_eles: - goods_link_ele = goods_ele.ele('.prod-link') - goods_urls.append(goods_link_ele.attr('href')) - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(tab): - image_urls = [] - image_eles = tab.ele('#prod-img').eles('@tag()=img') - for image_ele in image_eles: - image_urls.append(image_ele.attr('src')) - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - brand = '' - # try: - # brand = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1/div[1]', timeout=10).text - # except: - # pass - # if brand == '': - # title = tab.ele('xpath=//*[@id="contentContainer"]/form/div[2]/div[1]/h1').text - # else: - title = tab.ele('.page-title').text - - try: - desc = formats.clean_html(tab.ele('@@class=tab-content@@id=tabContent0202').html) - except: - desc = '' - - price = float(tab.ele('.price-group').ele('.val').text.replace(',', '')) - try: - old_price = float(tab.ele('.price-group origin').text.replace(',', '')) - except: - old_price = price - - main_images = get_image_urls(tab) - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - images=main_images - ) - - m_data = [] - color_data = {} - color_eles = [] - # 颜色 - tab.ele('.opt-color') - m_data.append('컬러') - color_eles = tab.ele('.color-list-wrap').eles('.prod-color-rdo') - - m_data.append('사이즈') - - for color_ele in color_eles: - color_input_onclick = color_ele.ele('@tag()=input').attr('onclick') - have_url = False - if 'location.href=' in color_input_onclick: - color_url = 'https://dk-on.com' + formats.re_search(color_input_onclick, r"location\.href\s*=\s*'([^']*)'") - color_name = color_url.split('/')[-1] - have_url = True - elif 'fnColorClicked' in color_input_onclick: - color_url = goods_info.url - color_name = color_ele.ele('@tag()=input').attr('value') - color_ele.click() - else: - color_url = goods_info.url - color_name = goods_info.title.split(' / ')[-1] - - color_data[color_name] = {'url': color_url, 'have_url': have_url, 'ele': color_ele} - goods_info.p_urls.append(color_url) - - real_color_data = {} - - - for color_name, color_item in color_data.items(): - color_tab = tab - if color_item['have_url']: - color_tab = self.browser.new_tab() - - size_data = [] - if color_item['have_url']: - color_tab.get(color_item['url']) - else: - color_item['ele'].click() - color_tab.wait(1) - color_item['price'] = str(color_tab.ele('.price-group').ele('.val').text.replace(',', '')) - try: - color_item['old_price'] = str(color_tab.ele('.price-group origin').text.replace(',', '')) - except: - color_item['old_price'] = color_item['price'] - - # 颜色对应商品图 - color_item['images'] = get_image_urls(tab = color_tab) - - # 颜色对应尺码 - radio_eles = color_tab.ele('.prod-size-list').eles('.prod-size') - for radio_ele in radio_eles: - value = radio_ele.ele('@tag()=label').text - if value == None: - continue - size_data.append(value) - color_item['size'] = size_data - - real_color_data[color_tab.ele('.page-title').text.split(' / ')[-1]] = color_item - color_data = real_color_data - - goods_info.attr = 'M' - goods_info.title = goods_info.title.split(' / ')[0] - goods_info.attr_items = m_data - for color_name, color_item in color_data.items(): - for color_image in color_item['images']: - if color_image in goods_info.images: - continue - goods_info.images.append(color_image) - - # if color_data: - item_data = [] - for color_name, color_item in color_data.items(): - for size in color_item['size']: - item_data.append([color_name, size]) - for attribute_items in item_data: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - attr_items=attribute_items, - price=color_data[attribute_items[0]]['price'], - old_price=color_data[attribute_items[0]]['price'], - images=color_data[attribute_items[0]]['images'], - )) - - except Exception as e: - try: - tab.close() - color_tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - color_tab.close() - except: - pass - return goods_info diff --git a/lib/module/friskybusiness.py b/lib/module/friskybusiness.py deleted file mode 100644 index 9a1575c..0000000 --- a/lib/module/friskybusiness.py +++ /dev/null @@ -1,141 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "friskybusiness" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 5, - switch_clash = False, - reflush_browser = False - ) - self.tab.get('https://friskybusiness.sg/') - # input('wait:') - - def get_category_urls(self) -> list[str]: - category_urls = [] - - one_menu_eles = self.tab.eles('xpath=/html/body/header/height-observer/x-header/nav[2]/ul/li') - for one_index, one_menu_ele in enumerate(one_menu_eles[:-1]): - - one_link_ele = one_menu_ele.ele('@tag()=summary') - one_url = one_link_ele.attr('data-follow-link') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - if one_index == 0 or one_index == 6: - two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/div/a') - else: - two_menu_eles = one_menu_ele.eles('xpath=mega-menu-disclosure/details/div/ul/li', timeout=1) - for two_index, two_menu_ele in enumerate(two_menu_eles): - if one_index == 0 or one_index == 6: - two_link_ele = two_menu_ele - else: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - try: - three_menu_eles = two_menu_ele.eles(f'xpath=div/ul/li') - except: - continue - for three_menu_ele in three_menu_eles: - three_link_ele = three_menu_ele.ele('@tag()=a') - three_url = three_link_ele.attr('href') - three_name = three_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') - category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - page = 1 - self.tab.get(f"{category_url}?page={page}") - goods_urls = [] - - #####获取页数#### - - ################# - - goods_urls = [] - while True: - self.tab.scroll.to_bottom() - try: - goods_eles = self.tab.eles('#:snize-product-', timeout=5) - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" - goods_urls.append(goods_url) - if len(goods_eles) == 0: - break - except: - break - page += 1 - self.tab.get(f"{category_url}?page={page}") - - messages.sendInfo(f"{len(goods_urls)} 个") - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - image_eles = tab.eles('xpath=/html/body/main/section[1]/div/div/product-rerender/div/product-gallery/safe-sticky/product-gallery-navigation/button') - - for image_ele in image_eles[-1:]: - image_url = image_ele.ele('xpath=img').attr('src') - image_urls.append(image_url) - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - h1_ele = tab.ele('@tag()=h1') - brand = '' - title = h1_ele.text - - desc = "" - try: - desc = formats.clean_html(tab.ele('.liquid').html) - except: - pass - - price_list = tab.ele('.price-list price-list--product') - price = price_list.ele('@tag()=sale-price').ele('.money').text.replace('$', '').replace(',', '') - try: - old_price = price_list.ele('@tag()=compare-at-price').ele('.money').text.replace('$', '').replace(',', '') - except: - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/peek_cloppenburg.py b/lib/module/peek_cloppenburg.py deleted file mode 100644 index ca21863..0000000 --- a/lib/module/peek_cloppenburg.py +++ /dev/null @@ -1,217 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "peek-cloppenburg" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 5, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.peek-cloppenburg.at/') - one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - one_link_ele.click() - self.tab.wait(3) - - two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/a', timeout=1) - for two_index, two_menu_ele in enumerate(two_menu_eles): - two_link_ele = two_menu_ele - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - try: - three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div[2]/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') - except: - continue - for three_menu_ele in three_menu_eles: - three_link_ele = three_menu_ele.ele('@tag()=a') - three_url = three_link_ele.attr('href') - three_name = three_link_ele.text.replace('/', '-') - category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - self.tab.get(f"{category_url}?page=1") - goods_urls = [] - - max_page = 1 - #####获取页数#### - try: - max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text) - except: - pass - ################# - - goods_urls = [] - if max_page > 1: - for page in range(1, max_page+1): - if page > 1: - self.tab.get(f"{category_url}?page={page}&userId=123456") - - goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" - goods_urls.append(goods_url) - messages.sendInfo(f"{page}/{max_page}") - else: - goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" - goods_urls.append(goods_url) - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a') - - for image_ele in image_eles: - image_url = image_ele.attr('href') - image_urls.append(image_url) - return image_urls - - def get_size_eles(): - try: - eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li') - except: - pass - try: - eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li') - except: - pass - try: - eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li') - except: - pass - try: - eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li') - except: - pass - return eles - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - h1_eles = tab.ele('@tag()=h1', timeout=10).eles('@tag()=span') - brand = '' - try: - brand = h1_eles[0].text - title = h1_eles[1].text - except: - title = tab.ele('@tag()=h1').text - - desc = "" - try: - desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}") - except: - pass - - price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '.') - try: - old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '.') - except: - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - color_urls = [] - try: - color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li') - m_data.append('Farbe') - for color_ele in color_eles: - color_url = color_ele.ele('@tag()=a').attr('href') - color_urls.append(color_url) - except: - pass - m_data.append('Größe') - - if len(color_urls) > 0: - for color_url in color_urls: - color_images = main_images - if color_url != url: - tab.get(color_url) - color_images = get_image_urls() - - price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[0].replace(',', '') - try: - old_price = tab.ele('.line-through').text.split(' ')[0].replace(',', '') - except: - old_price = price - - color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]') - - size_eles = get_size_eles() - for size_ele in size_eles: - size_name = size_ele.ele('.text-cp-md truncate').text - - attr_items.append({ - 'url': color_url, - 'items': [color_name, size_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - - if len(attr_items) == 0: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - url=attr_item['url'], - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'], - )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - goods_info.p_urls = color_urls - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/peek_cloppenburg_nl.py b/lib/module/peek_cloppenburg_nl.py deleted file mode 100644 index 6be9f3b..0000000 --- a/lib/module/peek_cloppenburg_nl.py +++ /dev/null @@ -1,213 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "peek-cloppenburg-nl" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 5, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.peek-cloppenburg.nl/') - one_menu_eles = self.tab.ele('@data-testid=header-links').eles('@tag()=a') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - one_link_ele.click() - self.tab.wait(3) - - two_menu_eles = self.tab.eles('xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/a', timeout=1) - for two_index, two_menu_ele in enumerate(two_menu_eles): - two_link_ele = two_menu_ele - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - try: - three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') - except: - continue - for three_menu_ele in three_menu_eles: - three_link_ele = three_menu_ele.ele('@tag()=a') - three_url = three_link_ele.attr('href') - three_name = three_link_ele.text.replace('/', '-') - category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - self.tab.get(f"{category_url}?page=1") - goods_urls = [] - - max_page = 1 - #####获取页数#### - try: - max_page = int(self.tab.eles('@data-testid=pagination-item')[-1].text) - except: - pass - ################# - - goods_urls = [] - if max_page > 1: - for page in range(1, max_page+1): - if page > 1: - self.tab.get(f"{category_url}?page={page}&userId=123456") - - goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" - goods_urls.append(goods_url) - messages.sendInfo(f"{page}/{max_page}") - else: - goods_eles = self.tab.eles('xpath=//*[@id="__next"]/div[2]/div/section/div/div[2]/div[5]/div/section[1]/section/div/div') - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" - goods_urls.append(goods_url) - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - image_eles = tab.ele('#photoswipe-gallery').eles('@tag()=a') - - for image_ele in image_eles: - image_url = image_ele.attr('href') - image_urls.append(image_url) - return image_urls - - def get_size_eles(): - try: - eles = tab.ele('.option-picker mt-2 ').eles('@tag()=li') - except: - pass - try: - eles = tab.ele('.option-picker mt-2 long-label ').eles('@tag()=li') - except: - pass - try: - eles = tab.ele('.option-picker mt-2 out-of-stock').eles('@tag()=li') - except: - pass - try: - eles = tab.ele('.option-picker mt-2 long-label out-of-stock').eles('@tag()=li') - except: - pass - return eles - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - h1_eles = tab.ele('@tag()=h1').eles('@tag()=span') - brand = h1_eles[0].text - title = h1_eles[1].text - - desc = "" - try: - desc = formats.clean_html(f"{tab.ele('#accordion-panel-material-care').html}{tab.ele('#accordion-panel-details').html}") - except: - pass - - price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.') - try: - old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.') - except: - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - color_urls = [] - try: - color_eles = tab.ele('.pt-6 xl:pt-8 -mr-6 md:mr-0').eles('@tag()=li') - m_data.append('Kleur') - for color_ele in color_eles: - color_url = color_ele.ele('@tag()=a').attr('href') - color_urls.append(color_url) - except: - pass - m_data.append('Maat') - - if len(color_urls) > 0: - for color_url in color_urls: - color_images = main_images - if color_url != url: - tab.get(color_url) - color_images = get_image_urls() - - price = tab.ele('.:text-pdp-price-primary-xs font-bold').text.split(' ')[1].replace(',', '.') - try: - old_price = tab.ele('.line-through').text.split(' ')[1].replace(',', '.') - except: - old_price = price - - color_name = tab.ele('.text-cp-md font-bold').ele('xpath=text()[3]') - - size_eles = get_size_eles() - for size_ele in size_eles: - size_name = size_ele.ele('.text-cp-md truncate').text - - attr_items.append({ - 'url': color_url, - 'items': [color_name, size_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - - if len(attr_items) == 0: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - url=attr_item['url'], - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'], - )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - goods_info.p_urls = color_urls - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/ssfshop.py b/lib/module/ssfshop.py deleted file mode 100644 index 7dd4674..0000000 --- a/lib/module/ssfshop.py +++ /dev/null @@ -1,371 +0,0 @@ -from core import spiders, types -from utils import formats, messages, files -import requests - -class SpiderModule(spiders.Spiders): - project_name = "ssfshop" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 300, - worker_num = 24, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.ssfshop.com/') - one_menu_eles = self.tab.eles('xpath=/html/body/div[5]/header/div[2]/div[2]/div/div[1]/div[1]/ul/li') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele.ele('@tag()=a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - if one_name == '라이프': - continue - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=div/div/div[1]/ul/li') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - if two_name == '메인': - continue - category_urls.append(f"{two_url}#{one_name}/{two_name}") - two_menu_eles = one_menu_ele.eles('xpath=div/div/div[2]/ul/li') - for two_menu_ele in two_menu_eles: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - if two_name == '메인': - continue - category_urls.append(f"{two_url}#{one_name}/Top Brand/{two_name}") - - # three_menu_eles = two_menu_ele.eles('xpath=ul/li') - # for three_menu_ele in three_menu_eles: - # three_link_ele = three_menu_ele.ele('@tag()=a') - # three_url = three_link_ele.attr('href') - # three_name = three_link_ele.text.replace('/', '-') - # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - page = 1 - self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N") - goods_urls = [] - - url_index = {} - while True: - self.tab.scroll.to_bottom() - page_urls = [] - goods_eles = self.tab.eles('.god-item') - for goods_ele in goods_eles: - goods_link_ele = goods_ele.ele('xpath=a') - if goods_link_ele.attr('href') in url_index: - continue - page_urls.append(goods_link_ele.attr('href')) - url_index[goods_link_ele.attr('href')] = None - - if len(page_urls) == 0: - break - if len(goods_urls) > 5000: - break - goods_urls += page_urls - page += 1 - self.tab.get(f"{category_url}¤tPage={page}&sortColumn=SALE_QTY_SEQ&serviceType=DSP&ctgrySectCd=GNRL_CTGRY&fitPsbYn=N") - self.tab.wait(1) - - return goods_urls - - def get_goods_info(self, url: str) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - - for _ in range(10): - image_eles = tab.eles('xpath=//*[@id="godImgThumb"]/div') - if len(image_eles) > 0: - break - tab.wait(3) - for image_ele in image_eles: - image_link_ele = image_ele - image_urls.append('https://img.ssfshop.com'+image_link_ele.attr('data')) - - return image_urls - - def get_options(godNo, optValCd1, optValCd2 = ''): - headers = { - 'Accept': 'application/json, text/javascript, */*; q=0.01', - 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6', - 'Connection': 'keep-alive', - 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', - 'Origin': 'https://www.ssfshop.com', - 'Referer': url, - 'Sec-Fetch-Dest': 'empty', - 'Sec-Fetch-Mode': 'cors', - 'Sec-Fetch-Site': 'same-origin', - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0', - 'X-CSRF-TOKEN': 'e8e4bcd2-b976-4ca3-89b0-c85de0d1f3b0', - 'X-Requested-With': 'XMLHttpRequest', - 'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"', - 'sec-ch-ua-mobile': '?0', - 'sec-ch-ua-platform': '"Windows"', - # 'Cookie': 'PCID=17748347492661999807244; UTID=17748347492661999807244; MAI="eyJsb2dpbiI6Ik4ifQ=="; WMONID=mUUUMLseCyQ; AWSALBAPP-1=_remove_; AWSALBAPP-2=_remove_; AWSALBAPP-3=_remove_; visid_incap_2173964=lpNXES71Q5iAc3f2rzl/jTzUyWkAAAAAQUIPAAAAAABeBQZ8XijRe/SDlNUYlcCm; nlbi_2173964_3216183=TbQ1Ku6HAX5AN79F2g4AdQAAAADFBJVdMHKyRlRjq5BE8+sZ; storedUrl=Y; visid_incap_2794548=/x/uA+w/Rna/+lXitrmX20DUyWkAAAAAQUIPAAAAAACOgO5Lt68U4l70FJ35r3uf; incap_ses_725_2173964=A1WxV+agRSxFFqDq6bcPCtPnyWkAAAAAl1pNcoghbBUv0Idr250rQA==; visid_incap_2898463=WYviBGzFSwWHcJFJUGirBdXnyWkAAAAAQUIPAAAAAAAtfVuBlsqIe97ZjwN6QI2j; nlbi_2898463=wXQSYet0xyoFwAnd4MbCSgAAAAArzXmJlyqxxSh3+i0S9GbO; incap_ses_725_2898463=s8pzSCxlllS0GaDq6bcPCtXnyWkAAAAAu3cfqwOGa86YepyhOT6TvQ==; nth_pcid=3d5ffbba-60dd-a49a-6747-a542beef7538-1774839765504; incap_ses_725_2794548=D/rjK2Wq+zSjGqDq6bcPCtXnyWkAAAAAwfdIptRC0JfZMv8gf0dFHw==; incap_ses_724_2173964=dLR3AfGiVBRg7b2bbSoMCuEsymkAAAAAo4viBM5iG3asWiau6QX32w==; incap_ses_724_2898463=/V2oJwzj6BLV8L2bbSoMCuMsymkAAAAA9Ybn1njwVQ3r7KT6QLCZcg==; incap_ses_724_2794548=3i9aHRkHkTdh872bbSoMCugsymkAAAAAt9AH6HTki4ssrvCcPmSpIA==; AWSALBTG=AIb2slX0AEp+Mv3KJalsspDk7579UU9Jv2/3kjqEGNWjsIrDM+27+/1Ke7bKfPJDQPkvjZ2Sxz50ZoGgs7wax+6xPQZc22NzAPVhfCwSibc44xWGCNEUT9ixCDWhVrmeHJTF4AGiXACsmVs+73qXWgynz4bNZxFvwGcomdcZ09QX; AWSALB=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBCORS=1Lc2nZgyDqlHzKH2Ht9zUCbVzO3nmyjbvR6L8TbjmEzgIJXLWPHd2TKRrA4NweRQ4J3aqEC0O7/FVTTgdHmYYm6M1K3NeLkZbyrFTXyTGk7GFDh5nYLYl/aBRLKe; AWSALBAPP-0=AAAAAAAAAACeq+TZG7CFWkFz6UEyoFzfgzCso6p2one/fqTkDd1D8aJwmBMDcXt1z/4SFXLjp7cnG/VXtUeFI5VsG2vzUNb7CeolT4ZG4OzG2MYZTDISyFXF0Ftu3Y5EnlA+ClLy5QvoVKs=; nlbi_2173964_2029297=AebNbzFaqQMN0dyi2g4AdQAAAADw44LnFE5aKd0nWF3BiMTo; STID=1774941024138741600844420260331161024; INFLOW_SN=99993; N_INFLOW_SN=Y; PC_JSESSIONID=054FCD4B9E7ABB3B015CF732C022A41B; incap_ses_138_2173964=rIZHCDf5DCYIKM99lkbqAV9zy2kAAAAAhshd7O+yiHEnrcRAybEDaw==; incap_ses_138_2898463=zRn2E3KmY3qUMM99lkbqAWBzy2kAAAAAT0zfogyh0Cs4exDxqH0C5w==; nlbi_2794548=gojpVZcccHOhC/svvoS1vwAAAAB8htUpn549dG0/xu4xckFq; incap_ses_138_2794548=A1yKCZEQdFLyMc99lkbqAWBzy2kAAAAAcNViNS39K56YSoYGpK3p0A==; _TODAYALLLIST="[{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0026011527285\\",\\"regDt\\":\\"1774840170633\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQ3A26021060165\\",\\"regDt\\":\\"1774920131600\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GM0025122296270\\",\\"regDt\\":\\"1774920840210\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQJ924040149659\\",\\"regDt\\":\\"1774921134062\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GR4H25042395931\\",\\"regDt\\":\\"1774921515277\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945089\\",\\"regDt\\":\\"1774921868451\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945090\\",\\"regDt\\":\\"1774921870125\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GQZG26012945092\\",\\"regDt\\":\\"1774921872189\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602949\\",\\"regDt\\":\\"1774941024251\\"},{\\"todayGodSectCd\\":\\"GOD\\",\\"godNo\\":\\"GRRI26030602950\\",\\"regDt\\":1774941046056}]"; godClickInfo=GQY026032636412||GM0026011527285||GQ3A26021060165||GM0025122296270||GR4H25042395931||GR4H25042395931||GQJ924040149659||GR4H25042395931||GR4H25042395931||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GQZG26012945089||GQZG26012945090||GQZG26012945092||GRRI26030602949||GRRI26030602950', - } - - data = { - 'optValCd1': optValCd1, - 'optValCd2': optValCd2, - 'optValCd3': '', - 'godNo': godNo, - } - - response = requests.post('https://www.ssfshop.com/public/goods/selectGodDetailOpt', headers=headers, data=data, proxies={'http': 'http://127.0.0.1:7890','https': 'http://127.0.0.1:7890'}) - json_data = response.json() - - options = {} - if 'opt' not in json_data: - return options - - for option in json_data['opt']: - for i in range(1, 6): - key = f"optValNm{i}" - if option[key]: - options[option[key]] = {} - break - return options - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - brand = url.split('/')[3] - godNo = url.split('/')[4] - - title = tab.ele('#goodDtlTitle').text - - desc = "" - try: - desc = tab.ele('#godsTabView').html - except: - pass - - price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', '')) - try: - old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', '')) - except: - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - color_urls = [] - try: - tab.ele('.gods-option').ele('text=색상').click() - color_eles = tab.eles('xpath=//*[@id="content"]/section/div[2]/div[2]/div[6]/div[1]/div/span/label') - for color_ele in color_eles: - color_url = color_ele.attr('onclick') - if color_url: - color_url = color_url.split("'")[1] - else: - color_url = url - if 'https://' not in color_url: - color_url = 'https://www.ssfshop.com'+color_url - color_urls.append(color_url) - except: - pass - - if len(color_urls) > 1: - m_data = ["색상"] - - try: - option_eles = tab.ele('#prdInfoOptionArea').eles('#:optSelectPartmalDiv') - except: - option_eles = [] - for option_ele in option_eles: - m_data.append(option_ele.ele('@tag()=input').attr('opt-nm')) - m_data = m_data[:3] - - option_data = {} - if len(option_eles) > 0: - try: - option_eles[0].click() - except: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - try: - tab.close() - except: - pass - return goods_info - tab.wait(0.5) - option1_eles = option_eles[0].eles('@tag()=li') - for option1_ele in option1_eles: - option_data[option1_ele.text] = get_options(godNo, option1_ele.text) - for option2_name, items in option_data[option1_ele.text].items(): - option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name) - - if len(color_urls) > 1: - for color_url in color_urls: - color_images = main_images - if color_url != url: - tab.get(color_url) - title = tab.ele('#goodDtlTitle').text - color_images = get_image_urls() - - godNo = color_url.split('/')[4] - try: - option_eles = tab.ele('#prdInfoOptionArea', timeout=5).eles('.select lg unapplied') - except: - option_eles = [] - option_data = {} - if len(option_eles) > 0: - option1_eles = option_eles[0].eles('xpath=div/ul/li') - for option1_ele in option1_eles: - option_data[option1_ele.text] = get_options(godNo, option1_ele.text) - for option2_name, items in option_data[option1_ele.text].items(): - option_data[option1_ele.text][option2_name] = get_options(godNo, option1_ele.text, option2_name) - - try: - color_name = title.split('-')[1] - except: - color_name = tab.ele('.buy-txts').text - - price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span').ele('.price').text.replace(',', '')) - try: - old_price = float(tab.ele('xpath=//*[@id="content"]/section/div[2]/div[2]/div[3]/span/span[1]/del').text.replace(',', '')) - except: - old_price = price - - if len(option_data) > 0: - for option1_name, items1 in option_data.items(): - if len(items1) > 0: - for option2_name, items2 in items1.items(): - attr_items.append({ - 'url': color_url, - 'items': [color_name, option1_name, option2_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - attr_items.append({ - 'url': color_url, - 'items': [color_name, option1_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - attr_items.append({ - 'url': color_url, - 'items': [color_name], - 'price': price, - 'old_price': old_price, - 'images': color_images - }) - else: - for option1_name, items1 in option_data.items(): - if len(items1) > 0: - for option2_name, items2 in items1.items(): - if len(items2) > 0: - for option3_name, items3 in items2.items(): - attr_items.append({ - 'url': url, - 'items': [option1_name, option2_name, option3_name], - 'price': price, - 'old_price': old_price, - 'images': [] - }) - else: - attr_items.append({ - 'url': url, - 'items': [option1_name, option2_name], - 'price': price, - 'old_price': old_price, - 'images': [] - }) - else: - attr_items.append({ - 'url': url, - 'items': [option1_name], - 'price': price, - 'old_price': old_price, - 'images': [] - }) - - if len(attr_items) == 0: - try: - tab.ele('.gods-option').ele('text=사이즈').click() - size_eles = tab.eles('xpath=/html/body/div[5]/main/section/div[2]/div[2]/div[6]/div[1]/div/ul/li') - goods_info.attr = 'M' - goods_info.attr_items = ['사이즈'] - goods_info.images = main_images - for size_ele in size_eles: - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - attr_items=[size_ele.text], - price=price, - old_price=old_price, - images=[], - )) - except: - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - else: - goods_info.attr = 'M' - goods_info.attr_items = m_data - for attr_item in attr_items: - if len(attr_item['items']) != len(m_data): - attr_item['items'] += m_data[len(attr_item['items']):] - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - url=attr_item['url'], - attr_items=attr_item['items'], - price=attr_item['price'], - old_price=attr_item['old_price'], - images=attr_item['images'], - )) - for image_url in attr_item['images']: - if image_url not in main_images: - main_images.append(image_url) - goods_info.images = main_images - goods_info.p_urls = color_urls - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info - - def get_home(self): - self.tab.get('https://www.ssfshop.com/main') - li_eles = self.tab.ele('xpath=/html/body/div[5]/main/div[2]/section[2]/div/div[1]').eles('@tag()=li') - categorys = [] - for li_ele in li_eles: - a_ele = li_ele.ele('@tag()=a') - categorys.append(f"{a_ele.attr('href')}#{a_ele.text}") - files.save_list(f"{self.__data_path__}/categorys(home).txt", categorys) diff --git a/lib/module/taketoys.py b/lib/module/taketoys.py deleted file mode 100644 index 286744e..0000000 --- a/lib/module/taketoys.py +++ /dev/null @@ -1,131 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "taketoys" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 30, - worker_num = 5, - switch_clash = False, - reflush_browser = False - ) - self.tab.get('https://taketoys.sg/') - # input('wait:') - - def get_category_urls(self) -> list[str]: - category_urls = [] - - one_menu_eles = self.tab.eles('xpath=/html/body/header/div[2]/div/nav/ul/li') - for one_menu_ele in one_menu_eles[1:-1]: - one_link_ele = one_menu_ele.ele('xpath=a') - one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=ul/li', timeout=1) - for two_index, two_menu_ele in enumerate(two_menu_eles): - two_link_ele = two_menu_ele.ele('xpath=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-').replace(',', '，').replace('#', ' ') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - # try: - # three_menu_eles = self.tab.ele(f'xpath=//*[@id="__next"]/div[1]/header/div/div[2]/nav/div[{two_index+1}]/div/div[1]').eles('@tag()=li') - # except: - # continue - # for three_menu_ele in three_menu_eles: - # three_link_ele = three_menu_ele.ele('@tag()=a') - # three_url = three_link_ele.attr('href') - # three_name = three_link_ele.text.replace('/', '-') - # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - self.tab.get(f"{category_url}?page=1") - goods_urls = [] - - #####获取页数#### - - ################# - - goods_urls = [] - while True: - self.tab.scroll.to_bottom() - try: - self.tab.ele('text=Show more', timeout=5).click() - self.tab.wait(1) - except: - break - - goods_eles = self.tab.eles('.col-xs-6 col-sm-4 col-lg-3 item-wrapper') - for goods_ele in goods_eles: - goods_url = f"{goods_ele.ele('@tag()=a').attr('href')}" - goods_urls.append(goods_url) - messages.sendInfo(f"{len(goods_urls)} 个") - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - image_eles = tab.eles('xpath=/html/body/main/div[2]/div[2]/div[2]/div[2]/div/div/div[3]/div[2]/div/div') - - for image_ele in image_eles: - image_url = image_ele.ele('xpath=img').attr('src') - image_urls.append(image_url) - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - h1_ele = tab.ele('@tag()=h1') - brand = '' - try: - brand = h1_ele.ele('@tag()=strong').text - except: - pass - title = h1_ele.ele('@tag()=span').text - - desc = "" - try: - desc = formats.clean_html(tab.ele('#product-info-tab').html) - except: - pass - - price = tab.ele('.price').attr('title').replace('SGD', '').replace(',', '') - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - goods_info.attr = 'S' - goods_info.attr_items = [] - goods_info.images = main_images - goods_info.p_lists = [] - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info diff --git a/lib/module/tfashion.py b/lib/module/tfashion.py deleted file mode 100644 index 31e668c..0000000 --- a/lib/module/tfashion.py +++ /dev/null @@ -1,140 +0,0 @@ -from core import spiders, types -from utils import formats, messages - -class SpiderModule(spiders.Spiders): - project_name = "t-fashion" - - def __init__(self): - super().__init__( - self.project_name, - bitch = 300, - worker_num = 24, - switch_clash = False, - reflush_browser = False - ) - - def get_category_urls(self) -> list[str]: - category_urls = [] - - self.tab.get('https://www.t-fashion.jp/') - one_menu_eles = self.tab.ele('.m-aside-menu-body').eles('.js-collest-li') - for one_menu_ele in one_menu_eles: - one_link_ele = one_menu_ele.ele('@tag()=a') - # one_url = one_link_ele.attr('href') - one_name = one_link_ele.text.replace('/', '-') - # category_urls.append(f"{one_url}#{one_name}") - messages.sendInfo(one_name) - - two_menu_eles = one_menu_ele.eles('xpath=div/ul/li') - for two_menu_ele in two_menu_eles[1:]: - two_link_ele = two_menu_ele.ele('@tag()=a') - two_url = two_link_ele.attr('href') - two_name = two_link_ele.text.replace('/', '-') - category_urls.append(f"{two_url}#{one_name}/{two_name}") - - # three_menu_eles = two_menu_ele.eles('xpath=ul/li') - # for three_menu_ele in three_menu_eles: - # three_link_ele = three_menu_ele.ele('@tag()=a') - # three_url = three_link_ele.attr('href') - # three_name = three_link_ele.text.replace('/', '-') - # category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") - - return category_urls - - def get_goods_urls(self, category_url: str) -> list[str]: - page = 1 - self.tab.get(f"{category_url}?page={page}") - goods_urls = [] - - while True: - try: - goods_eles = self.tab.eles('xpath=/html/body/div[1]/div[3]/div/section[1]/div/div[1]/div[1]/div[3]/ol/li') - for goods_ele in goods_eles: - goods_link_ele = goods_ele.ele('xpath=a') - goods_urls.append(goods_link_ele.attr('href')) - if len(goods_eles) == 0: - break - except: - break - page += 1 - self.tab.get(f"{category_url}?page={page}") - - return goods_urls - - def get_goods_info(self, url) -> types.GoodsInfo: - def get_image_urls(): - image_urls = [] - - image_eles = tab.eles('xpath=//*[@id="item-gallery-parent"]/li') - for image_ele in image_eles: - image_link_ele = image_ele.ele('xpath=div/img') - image_urls.append(image_link_ele.attr('src')) - - return image_urls - - tab = self.browser.new_tab(url) - tab.set.window.max() - spu = formats.url_to_spu(url) - - try: - brand = '' - try: - brand = tab.ele('xpath=//*[@id="current-tgoods-section"]/div/div[2]/div/div[1]/a[2]', timeout=10).text - except: - pass - - title = tab.ele('@tag()=h1').text - - desc = "" - try: - desc = formats.clean_html(tab.ele('xpath=//*[@id="slide-down-contents-parent"]')) - except: - pass - - try: - price = float(tab.ele('.is-fz28-36f is-fw-b ').text.replace(',', '').replace('¥', '').replace('（税込）', '')) - except: - price = float(tab.ele('.is-fz28-36f is-fw-b is-color-notice').text.replace(',', '').replace('¥', '').replace('（税込）', '')) - old_price = price - - main_images = get_image_urls() - - goods_info = types.GoodsInfo( - title=title, - desc=desc, - brand=brand, - url=url, - spu=spu, - price=price, - old_price=old_price, - ) - - m_data = [] - attr_items = [] - goods_info.attr = 'M' - goods_info.attr_items = ['Color', 'Size'] - goods_info.images = main_images - color_eles = tab.eles('xpath=//*[@id="sku-list-box"]/div') - for color_ele in color_eles: - color_image = color_ele.ele('xpath=div/a').attr('href') - color_name = color_ele.ele('xpath=div/a/span').text - size_eles = color_ele.eles('xpath=ol/li') - for size_ele in size_eles: - size_name = size_ele.ele('xpath=div[1]/text()') - goods_info.p_lists.append(types.GoodsInfo.GoodsInfoP( - attr_items=[color_name, size_name], - price=price, - old_price=old_price, - images=[color_image] - )) - except Exception as e: - try: - tab.close() - except: - pass - raise Exception(e) - try: - tab.close() - except: - pass - return goods_info