32 lines
1.2 KiB
Python
32 lines
1.2 KiB
Python
import os
|
|
from utils import browsers
|
|
|
|
class Spiders:
|
|
def __init__(self, project_name: str, bitch: int = 5, worker_num: int = 1, switch_clash: bool = False, reflush_browser: bool = False):
|
|
self.bitch = bitch
|
|
self.worker_num = worker_num
|
|
self.switch_clash = switch_clash
|
|
self.reflush_browsers = reflush_browser
|
|
|
|
self.__data_path__ = f"data/{project_name}"
|
|
self.__database_path__ = f"database/{project_name}.db"
|
|
self.__urls_folder__ = f"{self.__data_path__}/urls"
|
|
self.__excels_folder__ = f"{self.__data_path__}/excels"
|
|
self.__errors_folder__ = f"{self.__data_path__}/errors"
|
|
|
|
if os.path.exists(self.__data_path__) == False:
|
|
os.makedirs(self.__data_path__)
|
|
os.makedirs(self.__urls_folder__)
|
|
os.makedirs(self.__excels_folder__)
|
|
os.makedirs(self.__errors_folder__)
|
|
|
|
self.browser = browsers.getBrowsers()
|
|
self.browser.set.timeouts(3)
|
|
self.tab = self.browser.get_tab(0)
|
|
self.tab.set.window.max()
|
|
|
|
def reflush_browser(self):
|
|
self.browser.quit()
|
|
self.browser = browsers.getBrowsers()
|
|
self.browser.set.timeouts(3)
|