From 5d8b746817f4581f81fb406c66865943a07d7eaa Mon Sep 17 00:00:00 2001 From: Cerys Date: Tue, 14 Apr 2026 15:00:43 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=A1=B9=E7=9B=AE=E7=BB=93?= =?UTF-8?q?=E6=9E=84,=E6=B7=BB=E5=8A=A0=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- {lib => core}/gether.py | 21 +++++------- {lib/model => core}/goods.py | 0 {lib => core}/shopyy.py | 0 lib/__init__.py | 2 +- lib/{module => }/example.py | 33 ++++++++++++++++-- lib/model/__pycache__/goods.cpython-312.pyc | Bin 17087 -> 0 bytes main.py | 36 ++++---------------- 7 files changed, 46 insertions(+), 46 deletions(-) rename {lib => core}/gether.py (90%) rename {lib/model => core}/goods.py (100%) rename {lib => core}/shopyy.py (100%) rename lib/{module => }/example.py (60%) delete mode 100644 lib/model/__pycache__/goods.cpython-312.pyc diff --git a/lib/gether.py b/core/gether.py similarity index 90% rename from lib/gether.py rename to core/gether.py index 92ae5ec..115a754 100644 --- a/lib/gether.py +++ b/core/gether.py @@ -3,24 +3,19 @@ import traceback import concurrent.futures from utils import files, messages, clash, formats import lib -from core import Config, types -from lib.model import goods +from core import Config, types, goods def get_category_urls(): category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt" - if os.path.exists(category_urls_path): - return - + # if os.path.exists(category_urls_path): + # return category_urls = lib.Server.get_category_urls() - for urls in category_urls: - if 'http' not in urls: - continue - files.save_line(category_urls_path, urls) + files.save_list(category_urls_path, category_urls) -def get_goods_urls(categorys_len: int = 3): +def get_goods_urls(categorys_len: int = 3, home_mode: bool = False): goods_urls_path = f"{lib.Server.__urls_folder__}/goods.txt" category_urls_path = f"{lib.Server.__urls_folder__}/categorys.txt" - if Config.home_mode: + if home_mode: goods_urls_path = goods_urls_path.replace('.txt', '(home).txt') category_urls_path = category_urls_path.replace('.txt', '(home).txt') @@ -42,7 +37,7 @@ def get_goods_urls(categorys_len: int = 3): index += 1 if 'https' not in category_url: continue - if Config.skip_category and Config.home_mode == False and len(category.split('/')) < categorys_len: + if Config.skip_category and home_mode == False and len(category.split('/')) < categorys_len: continue if category in category_index: continue @@ -58,7 +53,7 @@ def get_goods_urls(categorys_len: int = 3): files.add_text(f"{lib.Server.__errors_folder__}/category_errors.log", f"{category_url}:\n[{e}] {traceback.format_exc()}") messages.sendError(str(e)) messages.sendInfo(f"{index}/{max_len} {len(goods_urls)}条") - if Config.home_mode == False: + if home_mode == False: formats.de_repeat_urls(goods_urls_path) def get_goods_info(): diff --git a/lib/model/goods.py b/core/goods.py similarity index 100% rename from lib/model/goods.py rename to core/goods.py diff --git a/lib/shopyy.py b/core/shopyy.py similarity index 100% rename from lib/shopyy.py rename to core/shopyy.py diff --git a/lib/__init__.py b/lib/__init__.py index 52ab1aa..3d83528 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -1,3 +1,3 @@ -from lib.module import example +from lib import example Server: example.SpiderModule diff --git a/lib/module/example.py b/lib/example.py similarity index 60% rename from lib/module/example.py rename to lib/example.py index e8d6b45..90cfcf0 100644 --- a/lib/module/example.py +++ b/lib/example.py @@ -7,19 +7,45 @@ class SpiderModule(spiders.Spiders): def __init__(self): super().__init__( self.project_name, - bitch = 10, - worker_num = 1, + bitch = 10, # 运行批次大小 + worker_num = 1, # 线程数 switch_clash = False, reflush_browser = False ) + # 获取导航入口 def get_category_urls(self) -> list[str]: category_urls = [] - pass + self.tab.get('') + # 一级分类 + one_menu_eles = self.tab.eles('') + for one_menu_ele in one_menu_eles: + one_link_ele = one_menu_ele.ele('') + one_url = one_link_ele.attr('href') + one_name = one_link_ele.text.replace('/', '-').replace(',', ' ') + category_urls.append(f"{one_url}#{one_name}") + messages.sendInfo(one_name) + + # 二级分类 + two_menu_eles = one_menu_ele.eles('') + for two_menu_ele in two_menu_eles: + two_link_ele = two_menu_ele.ele('') + two_url = two_link_ele.attr('href') + two_name = two_link_ele.text.replace('/', '-').replace(',', ' ') + category_urls.append(f"{two_url}#{one_name}/{two_name}") + + # 三级分类 + three_menu_eles = two_menu_ele.eles('') + for three_menu_ele in three_menu_eles: + three_link_ele = three_menu_ele.ele('') + three_url = three_link_ele.attr('href') + three_name = three_link_ele.text.replace('/', '-').replace(',', ' ') + category_urls.append(f"{three_url}#{one_name}/{two_name}/{three_name}") return category_urls + # 获取商品链接入口 def get_goods_urls(self, category_url: str) -> list[str]: self.tab.get(f"{category_url}") goods_urls = [] @@ -28,6 +54,7 @@ class SpiderModule(spiders.Spiders): return goods_urls + # 获取商品详情入口 def get_goods_info(self, url) -> types.GoodsInfo: def get_image_urls(): image_urls = [] diff --git a/lib/model/__pycache__/goods.cpython-312.pyc b/lib/model/__pycache__/goods.cpython-312.pyc deleted file mode 100644 index db30e534311632694f64ada5f9d27570089bef3f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17087 zcmd^mdr(_fn&;K~fj~kC5KoIYvazti#x~ga1$J<3{7m9FI11*imzQHYJ z57Xc6^Y-w4@To?4SPa9>B;}%TSf}K)%%wOhr{$MmV6@^AbxL)PIxC}CIY4EmEo7{@aGyixk^6kOw+vhDMVx-W$3uG$ zd1zN3r=ia{2fKaG_Xx8{MxlTd{}$&Ufi;r(DJQAGgqDxUeG+xAxSmoyMfHjkm{R7K zU6=8XwETQ6MKN+(0sZJil3uc7#3w#awt@Gn#5%v6$@Wvdnc{OYzb@R$sA1)>(ykhM z2@3Q!;flZWdi?G0E`(;{fBmLjDs3L@X1e=r!Ynk~7T%d%ch`&u zzV0)9E~nG3;FMlh-&rODNero#UR)^n#6Pd2o4cO!Fy~#2w`;G98TNL02i@>0Z&#oD zOjkdycGo#vc*o!{r*%5r18$$wc_Amc8609^7If+DgybTX$fC;E1(aWEOYh_t2MqHW znOC-qZn@Gr+B#Je$tVxHZe^^C6|D&zylb{hR!vk*Y2J1Olw+E&GV)^vQ`At#8p>`P zDiR9HU|obBsUBe);62%7WTUmh18%et*lN9`YH8AoRLid9_J9Sa?DEd;LD-Da%Hpyg zIktQMq1`q(uO1^6;iexJAA};V)ved?Y546w)T@9$4?x$ zm6Q1@Z&}jk-ec{%PqYiP0)?%12}!wkaH!mNqW$CvTjyc;d*a~1x}^j@x3AB&I>{NP zdw}LimNoA7`Iyz)(k^dL+NN%wtAFE)H4Q7)G^VW?WZXUQ6`fBU+S7h)rB8+Z?)+AS z?tbLV<+d|@o--?THpKMtBUy>S<9i-X7v1ga7CLHc-?#gTgC}g|uZ)y0oyQ}`Iu7kV z_O$Ks_NQ&NWXbGH7RmnYw+I~Wfk$r>LiHDKmH%g!Z0YQmuewsJE`)FbJ38CiPuj}8 zu5PC1c_$vZC9Mv33g>s_Y2ST(?~*3VS3EuY@G(QcJ&t$EZ5U^}- z591#6L9qpkbvy_6zR>4I%(U5V1jgd$N>vGFpW_E^D2U(poLneGi#Dj%eAH zPHvzzVyj+BR!BC3@6d|!pN#qodeuq2BBCz4l5+sSqPJ%Z`GG)BRvVIqUw#^YBgjh= zNemI>2x%djZFlEB1(D$H=Q9i6y&*Lg$fQMTw#BbZ-TjBRd9rw55}HWF(ha9Xme0u! zF}R0@Q1lU+&HW_)!7pr_n&+KEI7JJS!T67V{V$`JITeAg6P%9Zcnmu2YEIGT8sK#8 zARQ0-+@1k`_o~P?g?edW)ue%w=ya|QiM9LXU&z}s%@D!7ftntIzlbs;w=&L*05pg zd`4Dac*@Ee%Oe>T!5402)FRE}5p!+WPqAibV*R$I7A@^Wf@J!nas?Ue*m=t8(?Imjl0h@5Uy zi-PKi+z(>#`8JqY8HlVZyL#ve=t5fec;JJDAN_1$>PCG0Z|=W$@!sEjH$EN!nNuQ= zZiIaW@CZAK2)g&*{`JE5f12E&Oa=7JDS+ZW9`F`JoqOf{CPUIN&3ajG8A(11@| z@^w(dI3QWJH8tb%4KV{t1V(AW{wwr9NhSGDuQ1(E zEM3V(`ippj#NX0!99lsurKL^7J3t>5_~%FpcT77vi66q~WXCDHW~dT|e((Fgj9>W? zzlJMqzMAelliIBQdB{Hp*>QF*msg6iIwr;PD=cXvKu14Ch zgtTEb(#9pEjVqB7ado9iNuFJafak?4)xrT>sVXTEtw4b++#$^l0g+WEN1`SZBvP+5 zKN2;WBawQgDI(Nlns7#udbK$t)Y80(Eu?}~zm_NkVv|*;mb?X~S0r9}rup$mnu6FM zIsZ#e2{~>2Ig9^%Y#}giS2JojZ=BZa8lXEs>Plgx%3hCqfK!9A?&^_*Ks2(TN|Gu9 za=APW51vk+$JvKQfD5+$l5xh7mf{?|+KVu8k-DojMYRR2wqVNfrPdxZ=S|j3)J(3Q zSRd2?Qbx_|S@ZhP`EbjV5%W_~^Ruk^*@QeV)3m6t=I8^OhYgf2JGH$rFObEvUIl2% zOMub{??X9o@dMzq1Slnm5lV$C32Sa@MB$gWQRA}HTPSKo=?BRXr9;2sMT+V1E5v6a zgOZxG3ciw-lU87E6LKZy=b$|=Z9-0rY(5EudvRr!T|lM5b|PnnXDvQIvXw5i6|H|S zg=3EmKrb0s54l0eGle`$$c;j767p;zH)Bq|B1b5(VE)MQ3bRp3IWMg-s_IIRgk0Z{liBKZ~f%%+$Zs$O~zlFTL^v@|LjUqWD*BHqjl`rta@QlE zI+6c_{hIGcCw0P2aGY7H421zG~pG}Z+>h8foV56Mc!}P)z zbE-a1H|@2nc&!7t2gPhgIvwrq@ewY{DajPzktae7uZUQdGfSkuux1r*iUhAiC*IG2 zml0VtCqHVgW6gCDvm>C1S@Ht9c}r2CEoRIa-#a=y)xjF8Vy1$r$|>)(oi$YkpJ7cK zqo%E_X=|*aCX~TeY?`I#TG*!jH!U~MvmK|x<5_F8MC zv>68geb1NIepokC7xK*>Wb1cF%J)3TH`JQPb<)=+3Mq?qN5|?KKexX$!_-Onef`4Sd~4rj;(5n+1H0Wti4tG z*f|5+*dD3ghtsE<**WjnbffQD-`u9~t|M&A(TL+1OhQ3nLZRE1H{LNt2OFo|tfe+o z6tQehY@u?C69;6JsUWPi@~R!IqjY7%9YL(SkUkQzWPn(pv=PMmAt08*uiz~temNl4 z>wdZTjJzhDG@nil^hsFMtChwj-Sno>9^frJpH&Em2igX{5wJobAf7@%Jjwsji`M#O z7f>%nJa9Z=KfXt`T7q~V>8}f4AYzV;8k^8jKFPn6?z_a9f}F=kg@BI=n#V_QXW>|& zp2tUUW(m&(AIY~8gaitLkB=NRK~Hgv(2C&XBU=&l6kC0ie_dFQ_^|GX0bYbTYIFJIhxet9oW$+An73B`FnPE}AolSotvU@)Tep*56Ty=XICcKMZJ zTN(8+|2TM`cm6cSMEz}5#ac^s3Kknx+4n(Sirm)fA) zPJ1Q~shUAvmtYRz%ao9CI_bj&?=Q~55e9vH%wbHbFu~IfVhX@{MiB6Y`{ zmkP|e`2oEI%RD5L)YQb9n&vG9lZPh`M=cK4;)qqR z4Yjb9UIp=4a58kB1$DM5FbK$CPksJn8#8z%j z4ruGOfk@d7p>JtCvwpUrEmGC~FyBz0H-2Es7i^z?g|(~$*tQl=?VIoiy|?n$#;oPR zO(E4x3u~>9TDP#)Em3PLYi*q`F1vbo`f#+ki7jr5IT~hLSjWyUEMJ^wcOMI%I2nGL zW>2`n>(9n&>q5`4wcBC<_G{R>-ShVP8^za(LG0PiHa!-xci;?foAJ!AZ@kfUt!u6_ zyz?;Id?d2|C`?ny7MP}z(uWFNLE-;MYy0jOA!@PTve^G;LIK5p_9CzOO--9(zgqrVwQhgC>bJ(+{cBaf zU8{n7E{kj18nN<7hQay0}2J=h0sZagk7-D9w{U!lFp2D?u(>98d%!u5J{yA zcla?VG4F##STrw=sQk({>hv!_98uHqbO8^3buTLB%Pv|nV4|_IP*NC2G=2>z;4(%w zw$6v*5xIKNd$sKHOHP*&?IYho1e6h-U)7tttQ&X)C5;Fog7Bi1o{8Gd3n|OWh@e92 zq*6UE90>goks=8?z3Bg1c1bOXS}uZzRLTn>x^sH5285cw2c=QfTePercog9aOth+A z^nNV6lJ&z1d)818s`W%Efn?#G{~$$^#BY2Szx)Ykns+~W_x`ny?q7TZOJ4gaxO+hB zOe&P3e@NV|;70oVeF#0_<4~lo1+t}^9D-Ng``h^UKfQnXtyC9)&{S$Wgi}`E-QRt| zGr-e9Ow7XjpF3Xe^S;cxQR#rpZe)%@ zD+Uh$r$=QDz-YkbW$>JFT8vQH_|& zC2Ot>9*>yqP?c|+{LaL8g5F5}y3igrzdrET9fKumsALU5UO3e_VGL?m6Ch}g_-J5F z;4(wKR;VWh{j6zw%#(v7eFH=v%F+pQM35>_CBlzkJ^8N_ez{E zEki#3yk~HD`2Peq=6#~Tg^9nG_tf$XRvJMndnY@)*>>;a*Y16OIsU=B@!5%cGh+*r zABzEoug6FKE{P0W25A7X9Uyow@rk0-A2p~&29kH=kMfF+%F_uiyznt9OXsep!Ab3> zfI<%Y5q2#`*X@p&_8=3^N>GYS-Zjz# z{YV#bAu{tg?ssI96j+fcIHxtM`=v*vQw`ycU~@W+X8=P1WP-h2M?)xrUygd{m=R6_ z1+y&gdX&@!5|2V0B9lJ9~@3-znd3I|kQ<(9wg4Cueg0sAdjWCPl} zrkwG^5mRMAHD9(iwDWgmEv%s=&_3Qgk8ocC55e4^Cip@qH>3%dZuru$5tcF7{9)@% z>s%pQ0cXH^$66k>Zf32UBi1c}j<1WW#NN%b9Le$-gT8q0zQ zB1Q)gV0gz5VN}-1tck2IO}3aRcPe{AKd$jh8&F` zdM#1QRzFQCYL^1epVF^@6V(YPN(CZ2+9Z};$*2(C)|!^%r9^pv93V><3G&F^RkS=s z`BM4SAUmkYnLdJNI-UJj#KsW~tsc>yqd=|Fz}Eo79MSvrX;nZalp`-)cA?TFqWMUM zU+2#tNUKXKetszij@Af>zWEWBI33vdB*ixQLR#zB0y=0RGEhJVT6Z1S4=x!+zCugu z$@{fq=fGx&Jd<2$?S22Iw?Obz#?ZUasf4v@(H=mAhvdD-pIkxr;yDBAAsC#WqEzt_ zRlb*f>6J0b!mvzdlLo{*2g75!G5HQ8oUFBzXi;uc4FrQK9X;L`hfoC-emKx2jaAUk zpq_9}uoB`T5ENj};@!)bG-86BlrsXWgvLoei42tRS6&IrCPO7&OVqWXU%O$M2F3Ja zN138N#2flNOUnkjfm2_?u3y6n-o!%q@?|UxK;lJdHH9e(Tv0C!5Z-(tcPSR(xS@C- za>EixE>aIcgDP60GsUchfy}Q9DnKVHu0pd_Xm6x=!>pMt-V`m~&K7T7JRs7X3cC)q-^tS&ny!zYmQi3pmoirz@aJoEkh+>6euIwrA+n?GFjpU z{JPK{H~^-zsI`H$Hq7q$qVTt+|5(a49*J0wLN|uosRGte9_W}in#Z@j`6{quV-;va z0^2N@?TBt_XE(KfaW;JP$?%bk8sm}3God!k53^S9(JgUYTK1lPMCFqVM$lX7ve(0- z_%2>6Bf6m)jdY+}0pIrne)!e2;sQ`&)HOj4(8_?HYN0>dH~WKrega6L1I-C_;$;_< zrklC6I^OEv2|cYgbGm=m%qeO7r+QBZdUSIBbpM9=)2E+5zqYqx+0?+JKh+%g0m0|0 zG*l{oO1wFEpw1`L0aDUiQODt3ZSm^{Rw6B4W7GME7? zS3A5VCEgoDagbt znr3(Zpa+zR6uJCDNeV!Hv$`8F%KI8508|Dlza*Mh&*s(7>LYnAfs9yXeYA2jIB%kr zTUdiFuy=gTo&4%({#q71F$J~J0tZ{*AU92HK~v0H6SZz+tsBWr9c!&i(v;LiOPau< zHeYDFS~p!6{7$rH2V1ivQrH?3I~3zO8X~2R2e}yqS%Hp+R;tPo@k(^_+{b7arK>| zva8M0&9Rb-t6kGwv9g+I*+#Z(qxiz68#}H+B+IrKPbLPEFp=^G2tDEdKgiKr&Ev|4 zd6d0jwmRDI82HY9SKGl_D#zQWwsHBk*v955B|5D_Wi$JO+rlO5Z&@8MAo~FsgqCC% zMol&teb92-RGnBu6*VNvsm$yvoui#o+k)=j88$rFFQcmX2!99_IwSnda!Cv$8kiOE zxA+At*I$H()M|LhM>?JcHl6DB!C%V2?`D;$Hawz{pl-A5x-LAAsQqf-;jPdUpg>xr z#P~_sm{|wEe+CUA9cpS;vjKjef?rfaEJ^y_3Bv4JBA=(>^Qr#$F+cD>#WIh+V?_H1 z2_LluIgB9+?xxd7Z=_eO)_8dtKM%gUuna9;|IXd%W?;bAidkr-;Rnur$JGgb~C)=16>Ea zMT4+mWMLc%yh03c($mg7g_MM+qnQI3G~shI*gy-^d{`mg@m2_oB1UEgk~TO@dR)*W zm%%eWLdp=aoBa9$H<`rT2y0`mLD7=mYe=2Eg)A%v)xg5CC`;aC^F;HMJCaiy(8A9# zqB(VJPF*C&5zu~}zb3GM9z4kI;4`zb*_zpca79bRx*eHCMYM1STL=tkXW;R#OYP(O zSWd}g>qKktB%8B7W+_g{71_IFF-!S)+hpfNXYfF@avQkWBb64qpf65wemOBy1gZu-_%#`wQByORR#B*b{WA-yWE}VkWem*6*nMpb1{ zu}DGohw`%nuJfW7={DH!h^zpv|Endov diff --git a/main.py b/main.py index 41eb1cd..61e5386 100644 --- a/main.py +++ b/main.py @@ -1,45 +1,23 @@ import lib -from lib import gether, shopyy -from lib.model import goods +from core import shopyy, gether, goods from utils import formats, files, wpdata if __name__ == '__main__': - from lib.module import example + from lib import example Server:example.SpiderModule Server = example.SpiderModule() lib.Server = Server - # category_urls = Server.get_category_urls() - # files.save_list(f"{Server.__urls_folder__}/categorys.txt", category_urls) - # Server.get_home() - # gether.get_goods_urls(2) - # formats.de_repeat_urls(r'data\bricomarche\urls\goods(home).txt') - # gether.get_goods_info() + # gether.get_category_urls() # 网站导航 + # gether.get_goods_urls(3, False) # 商品链接 + # gether.get_goods_info() # 商品详情 GoodsModel = goods.GoodsModel(Server.__database_path__) - # GoodsModel.to_shopyy_xlsx(f"{Server.__excels_folder__}/goods.xlsx") - # GoodsModel.to_woo_csv(r'data\fs\urls\goods_2026-01-23.txt_old.txt', f"data/fs/goods.csv") - # shopyy.Shopyy(Server.__data_path__).generate_album( - # f"{Server.__urls_folder__}/goods.txt_old.txt" - # ) - # shopyy.Shopyy(Server.__data_path__).generate_navigation( - # f"{Server.__urls_folder__}/categorys.txt" - # ) - # shopyy.Shopyy(Server.__data_path__).generate_album( - # f"{Server.__urls_folder__}/goods(home).txt" - # ).generate_navigation( - # f"{Server.__urls_folder__}/categorys(home).txt" - # ) + # GoodsModel.to_shopyy_xlsx(f"{Server.__excels_folder__}/goods.xlsx") # 导出shopyy表格 - # wpdata.Wpdata( - # 'data/condom69', - # r'data\condom69\excels\goods_1.xlsx', - # r'data\condom69\urls\goods.txt_old.txt', - # ).run() - - # Verify = shopyy.ShopyyVerify('https://ssfshop.zenshop.cn') + # Verify = shopyy.ShopyyVerify('https://takeokikuchi001.zenshop.cn') # Verify.get_zero_collections(True) # Verify.get_error_images(True)