From a9c7b7ead558c7f9a4b27cfe53fb00277fca08ad Mon Sep 17 00:00:00 2001 From: BXY0414 <1079823339@qq.com> Date: Tue, 17 Mar 2026 10:49:42 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=A4=84=E7=90=86=E9=80=BB?= =?UTF-8?q?=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 8196 -> 8196 bytes .gitignore | 1 + .../project_bitable_common.cpython-312.pyc | Bin 67978 -> 75862 bytes logging_config.py | 65 +++ prefect_gmv_scheduler.py | 118 ++--- project_bitable_common.py | 453 +++++++++++++----- run_project_bitable_pipeline.py | 82 +--- 7 files changed, 442 insertions(+), 277 deletions(-) create mode 100644 logging_config.py diff --git a/.DS_Store b/.DS_Store index 45bae9d4666962b9749505901bcd0e5a149fc99d..129834e9559748a98872eb3a9d023a52c652685e 100644 GIT binary patch delta 163 zcmZp1XmOa}&nU4mU^hRb#AF@;OMZ5SWQHP!RE9)`5+Io}`J9OSW>&Y;Ioz)WFwnzCae=CqE)a|pjQo6 ztwF8gCzjCIO24|q)+((_XGk>Th*kR))UUN21*^6FzW&d-Z#EG7`~EMP^X_u)e(t&F zoO|!Pi%8z~Np!DiwJHXl@11(Kv-!V{>ykttD~=c3W9Jx7)otmqHd%X;nvz&X#5lE1 z+fyv!GE7$`x%IweZ`Vx08xNaOovEC9pPFHs(!iymu5?b@l)>qm?ClJve}rwygtr0S zS&*%aOE{ev$ymfJWjN#Q3}<>uB;@*3IP<1#z|H2&oQ1P;NvCa3NfeG$Jt%6*xncI{kbTBCvd_ICt{reQzk!=4;CipCcNQRLe*-!Hh8gDoZtgd53vP&; z2e|p)z;!siGTL1Q5bgLzbRk#hEaHkdwoSqn-!5q?j`Np*zl1x=l{!nEdK>f?SLQ6^ zikwAlGOqk~X;Zl~t&Qa>oZ4{uX%JRH<3-$bNIMPO+BT7wn*pBb;fP9b&j`C`g1a*8 zo(1liVfSos&kDQefO~e>Jr~?_xOq^Wxy~Y|zDMX+S0IL>47jSjhHdWmUW`A3J zvKKO30~s#k);b+f-9qS@b>Lr4`$5F52X{TCyA42hH%I8MX$7~D_B2*F)~A~QdnMNd zF`MDH1%8pvK+4o`TOocGcZ}NxG0pH}_j%<&j>3p<2eNBWGD?O!&b2_~^;yIc(KL{u zf!hITRzr$oTr2cz9;G290S*Y<3BNYlJK+jKaYAW%P?dH_*~nFUooXolCTMjB)NKvd z867yB9bFK%7HCe7I6`v3+zFxUVl9J$6?BMYTsOq5kBhms+aq|C+#&ZK!1ivSwZG9^ z=!JIlId2w7Ia>E`%o!NKerU@^?iQ%X_47#>{cs*p(&wSzO#;66x(SE`8<{&UKx^XM z&K2B%GmpDbb#^&7Q=XyWb_4E~Xv=Zr5q}T-5NB@~&%o^iysdPQzt8P=7II$fa1TTe zUf)s}Uln%%TDFZ2%C}{y{$iX(ALxI4h#8t1e0h|t+7W2iaDC52d zVV&ImWEg{Jh7h6P?uLl2y{v`JmS2bPx1XUuSPSDOGG> z4P)O?9AQ<-zzKN-sU$^}CCj7UvBj!is0}ITPDPN8AZ<)%S)o!sDdG#z>mdG28`*Df zoo`2PCW0&kG6XpYvJof|cpe z34SgOfbTFL=n`&Q6;Nq!UoI3-0r?!9ZHKs6or03?PM2G#53SjC`OZLW1fP-bW@QNV znSu3T)5)JQGrW664sp=9%IW4iTi5RGZLQ(mooy|xZdXv($~#-!POiDF)7j0r8XPkI zx}wF<_1|9d94r~Zm+*_U$YebsKE7;Hk}9-l4y}osc{C*o3VnOk4oy&B+tS&+7n`!UueYtUJ*e2x;&L_*@ZCX8PfLG4 z5DB!X+Zi;t+ci@PJJZRh)dE_XY3boBM@Y;dTq>%F*+L-{u- z+0obM?dI#i%`ZW)6v1_)S%qN;{s}+VVgL~;NwiEzECGvcLShS;tP>JzK#Rm44=e&%rzm&2-s4FRvnu@DM=BA0&^vknHFyoIEEP~)5LO2lW0A| zu}(23!v2LZ5aY4}ehKwUVU%QHsX!baw`;@b2s&Z>I5{05g+m!M#E+2MxyKzeE^&6c zItCVXLeF};or}9WoxSd$td+j$tO&}&oh}cLVNfrORo^Zr&v$Z87ogMeZg42B?`b+O zWliGP?GeHG#zh;-)+0SnTZz9$<5gbFzMTzdV zl~WS4ABMU8ElE0`M~?p0X8kvXZ@ac|em&$9DPGDscMP=iH$zZ-=}IhJh}sNMmTweq z@k=bncE2Ubfa0%apQ^|f^4tP1ek*2%x|KcL7EM3z+|}va-Ap;zc8E!4$s0MC5BL z0WoO8La%3R{Ou4m2iu2z2sEPB9-VV|j?Y{=S-xySz1$~(1ty$%HF>wlSRbuaFtN67 zNzM9|>zdcDt68(Ixwd9q-E~YUXu&c$XIslaHEW997k987 ze#Ety%dgv};{6`9)okO;_i1amz}o~kHj^GTp0UHT*hY$gAwIK=KEC+kh*#I(IR zv((r)v9DUbba_J!Z^2+ISzWoffHE0609cpO@}PXDbMNjxp5swM3XkXzjOe+_>Fi$e z=gI;$k(g)Zuw!KI%)%myBn34g0Kuilp7`j&=RSJqfj|CyRQRX$c8F;k_NlN@_EiI2P9EGftDqI@>+9Cg zX7SHLdfFVowt_pTT(oBOin@lT)vE_kDE#>N9Unb@^pi&p@(JJr_Wc;VfTzC5kc8Pr z7|h(+8PZoEY=|tMy-bDDV^D%p1KB>^GIn;hM#OF;FU(ENYK9m=Q0T!N6#T1*D%!mR zelETNJl~&J3i*}IOOj53?+95rZ%SxxDRIo-kWR5qv=2UQnD8ox40zbWqaO*V#)KEdCafwlJF=AT!k5Id4$SkrWNs~ANn7+=%#JOG?k~X@+OjfmIkf!U7@$CaE2ALf ze@Z^AHP1~0e^Alpbi?fK?&fLVQgI#omJ0I@CI1(6{}n+9!4(8oNqL>Qd;**<99<+j z`Ur%VKyz&Fp(1oM-vDw|HdqGA!9rUzALfpzJur+@toVR!$mb3TeIo=FnI|i)3jt zq2!+fUQpK4*X!Y62s`uIEyD%epP}=Es_PB&DlmHSh1rSVJyS}k;^_q2cwatxd zR@c`pUbmL#u{5LuC_Xl*I!yB-ga%c#=FLC zDiitoBfVA|3H2~tIxuP*iEmON6??>83E)fdO5Dg!>EjW5Bp%iy@-U~xPlbzF%Qz$r z1E}R*x@+X)=b!)h;ioa=RRF*QlZDcf}d*uvVv7NO=lmmOl8jSXU>?=RSqq^Xek?& zO_kUA%j+gAOUUbg&YWX;BI9JngtlU6NkD6y(q{Oz;L99Za#5WU(3|n!U=1YYgX5}N zrjdmhnN&t6%h;_q{Z(Y>$2wCX3>g(jjYI45HswGSC}jsVYX*AVojqYuPr}pn$w=&!3knu0USPFJ1l&X#Zv-4uJO(G?#9%Ek1BP+0*kR$L zklN|Pp${h$ecVQ(+L%iE zf2PnuusRY9Q{bj$CQDaRBp~+gN+HL~jbz@&C&W6wnq1nLC8Ok2O0=77nL%AM?`(zD z-__jO(%Rt+YQrQYc=-ZSu&GQ-{tL<@RU58FB~O1J2*k$MLU@8MaJ`*n+>HeN^gY_&KHsQw%e90K<5$!uVRX&m_iK33%(x1iV>6`XasPB)remQ z;rmjE-D@I^Epm1S+1TPJH~W%`Pi;80Y_w*S8{Kej_V|+V**?q4w=}E%sJS-IXPG~# zsTw=m@&?Ot(26}jaFvr+yK@V?WEr^1B}N<0$s*^6ef ztV=(?ovCHE<}l2l)Wde^BejaX-6(F+$0OxbIAh?g#`kFr*}~b#=P#NPw45%oXW(Lc zZ8ZCI`ySRMVi~NbM+Cv_5fayIJy3jrOJhysZ`^d;9}phS5bX|_K#>+dz;Fo^HU=g6g}Pt3K06T+$n z@)6~^p-WGE_v5>tCA*y&Kv~1is#Ho71|)}RfI($vSn35g2|1g+YJM)pQsSemo6b6< z5q>5DB+Z}6Jf%gNJ!&ul1+GQcB(W8t3XFKpA2JYW%?MvFDO`GGlVkeB2o1< zxvl3z_EF;OEnPXAW`?EG;hTr9A_xnLTR4tRb~=z~?*NkoCkSHO0SlxaI((EEgfS%V z_omIHBtr*rD`um=u5e*!k)_j-6#(dA)A}k}+i?iT0pJiCkGZF;4nO>d2eERrW3=g< z=k>+D#&uJTJN%71e62hEjolL~dweT;eU`pSO@DMmM>_n*GM}dGq9Va(nsrVwy3d!m zXi~A5{JQU89#F)*bIZ=n9l!Z})A^OYjZR-%|An?epL@{fJm9l}z`OG~x`&{Ap^4GgRW0+deauLYxTiAM0*yr2t^=;WcwT1I<;e5_pe6E4F zw(JTCn1F(qkYF{tLIi;g>`HtYSBao7WrHYI&+*jPTYYP8ey!{L7GGl1q++u`Ot#OZ zGz&^iuh?x%vl977p%+`l)X>ww4{N1y>`_ME=I%gD9b@w=GzdoBS75?xD4hTl(_fyJ3+tA zAsqv`H?0E&m9m0^{0M-s{IoqRKN$(T%Siq-;K(|T zr1v^tdW=; zaK_MXl)ECc5vJm@{dMd~a`*lU-F0*4jr~;?=z-etjpOy_yL_9r`!=@tEITGOt;Fuh zwn9|>`Pt_Sv1^*!eB0W6mX1kHCu#D`sgLAzW9t6us*N!uf6j=MNt9o&@vYtLTeHQt z1!iW~RNF3p+b*ATx6iU?QnQ!5=gDL}WB>AWv!y9eacXZ1Dq1^0*5^S2M*BDAjsivP zL$3$HE^_~Y8FMJTJcj8gzl*L^*U$^auZ2W$@-~rqYuKNWW!_g3kb8;Yccl%e21Q$P zQ-_6AblkI&lMYIhj7P%B_JP_pDB&w-mL+QqB@B*$O2q;zSEE6;*y5mq8Lav$GHyoK!s@O81CBKeROr zpy@-9F+TqB=%wfHjR+Wmn%Buy4WN8nY!_lt<{%qyb$DwbFt!l#EjUgM2v8mv$c9*u zM=l+`Pe^v@&@gJBhwk{~5lBdY6q{%OwjNu3ha~{f(_YuG`<(GqJMQx1!Hy>7UfxLRY&yzcFuQ zkKb7C)0D?Jm$2Fe@(K<~17_) zpsz=fBxVyfvscApt?n}gV@eMg z%%NOHZ@ikvC_&1iEAlsaK|#qO$&@0+uShwu%&*9$D<5U0K_%~0q5=ndq5KG0b(@V0 z-((=WZu=qG=qe`1cx3{J@wYDUNI-s{L*;iVP6neYLdzb}&I0If_DB***kctdIvMim zZ&JyEPK^e4#3b@^{iX;^p1i$s_J>F`zl5T~ z%T&>+WvAwjZW(usr<`9txuMCoxn*i|n}2iL#O98P4V}LAUB1MflZtL)`OXZGYa~il zNW~XBuqc1)NmrcAT_rrGB#lct65m~5ZP8C!lx%xLoHNI6Wr9jeD?BFI93vzT89 z4fO^!uzBO|Yz^zPmDqUXrGmAMPA%+3K~F$wpQr2mG7Q^;-a!OI2o54Rgdi7VQBS5O z5y2#}0=y~MQ>f?O&V)odscOWi3o-CMWx9f|kez(_ofRuAQuB#*BaZuYl^00O zBa1@}yiV1QnnqjCO&{$zo9-`Ocm=~k>sX!CbP5pQb#5(vjjzM}OpxF4J8o)BgY*y% zuT$$rb4Fd~7LV>ZTk0>Vxq@M#YK=5`q&39AYgBlR&p0nWzxKT1rD_`L+t73sV}%y2 zV8*81xsy$7#pZDUU{ObVEZ;@W9I~#)r62cKz%In^#9+#Ou)o1db{BRIUH7RvkCrZB zH|ts}8J~=J2tuUj@WV1RPP9sif5J5O$HN8@dz5_f-7dp@h_nH~lVWOqBm9vS>MRV* zMvzC&AK9>-F0FTCIRzN#Kv0N)iYRmzpk@JWxEn){BA~XAnDz8A42y~=*9#^GzLz5X z5ny~xe)zr5%+xlbiXDjHXF(LoTIzv9~u%O?BF2lVPUaV z#Y?tNLDlg=YXS#$P{fDD8!QTFLm-AcrK%FwbX;&%m=>)A4NC z46J_zsXAWCt|6}D8`D=p0EiY<7>)uLY#_tY5Eq||0eR$;;|1(mlKB07-YzJZKZ2oX z!V60JKx3x*wjkmD`yN&Ck7G*g0Aa|nykNL*us|N$O)C)VUH&wLxsaS-T`G>QP2dk2 z2l}azHoRF$`q!%qcat_Y$z%Yo( z+1ykaNKSIFv4ArPoC+%6C~>qw%~5jd2SuEQY`!;BtM$lHnxY{Fa`@itiUckZLX0%T z|Q{T>o=nOxR`qCu4#`cQbdY|yrjA1zVlcvK!GmwOua$M0KgE!+tk8hnb| z8pSK}sJoI-{9}vBhxg@Yltf6FE8Pj)lGnxImU%Q}{&#I`IjI?5F3pdmloo(184aXj zI7dxh9Ja9)Uq1P_sl6Pe{dMCZK-9tIE;clNaN_ke2v zjboSZ`Vq)upFDE=$4?x(^uPy&Jzem$p4(kHb08aVFVQjrXiS4D(}A|kxzf`>nF{s$Mnq>T7 zrffg@^T_5OG(vUq;Kk9tTFyGg>X6*C>GLfy+P-kI&ZK)A0NfHa&FcK_R z%8tGH&;^#&l9Lbjv3im)Qo<&n!a&xHxR z5z7FjwO6ngaMs)HL&B0Rq<@+WJXQ+FHy(TJm+T)%`;XRSV-7%}sN_ZV7g_hUTZ^>h zb42#|@BHHDrk!Ipq=DYgv{?w}d#72ZRpwnBO&k$)HR z94B8q1t(+1?5A&Ly+KTkCYT^wOm%BGP^apIV~HTU^|m^Bl*WJ$^t$=CA!$$whY-Om zn2o7$;{$ge`ny{I5eETkgU3gx=ny}0K}~-P4~Hhh4f_M+^fD6vF=p(`r#G>R4ooZI zw~>u!vMZzFK~O53%rOIcoMw)ts$koerw3A8{6`S|HW@v$5jIIyKV>6XqYu|nT9OId z<5b2(q`2Wmnt%eMMe%Zs$DWKUh{(>`rJq2=`=sdX3MCaf7Xr%Ik+Xki1zq-0tPHI2 zaG(|1j_fZeg5#Uw_MTnz;KtHb8$NpOzK?!!JH%2_^#aO(5mJ3}&(NiNe*DQJ50!l~ zG<5m&3*Z7Nj`k}Rov?HHvj}KiRs0CL7b5s?EQb7JELr_2xcEOKc$CSfaY%$v&n=Tv^n;z>mB6d6J|5b_C5XX3^zCtxS>{-FYL`RQmF4$xq+aPF+D6I#j{XEX4 zA#^M2_Ov?t>CqJIUWYNRkqaS!4m52p9sHmUPNjBgc+eWd=EPc(`EoB*`Pj?VY(Dwj z%h@sw^q*NW_SMTamQ5H-eC3oVi?6|~76Z6OKtn-L48JSgBJ$<=jwLZ;vPTpFCo_$O8DXjBcxmkJdHC!A>ypTdYOwv&iL z7hqa>rHHT>EI`4FU(Zw-p`f)AU?AzF_w{K-y1a`DYrvd#(dL-26$R2t0x6DwDJ5W^ zepM@1s;-EYdfCv@t0pEbo4o&enK%DT>64}B@K(2=&4 zy=rHqTAw!mElB~Iu1)EmGZL*OU`!saK3+X#th`{Xyl6`GWz0E8w!V?NL6*IEp5#~a zl(p04FH3T2OXV+@un0?~5W*)y1A~e5Tyb;TKzBEofGEQbrotgII6E1>A3>LES99bz zdaDHYeRThX#9)CIx%9AM4GKlMiTw5rtzHsmumw$)wEQHMgnlQTg`4S;80iANfm!-i zP260M?kf*UdpAM8H%IcdQEg*cj1@#Tq)GFe<~+9+#*jYIaHc~%D2=O#OsI$f6*hNt zbKN5)FYUBsT*C-FzzAg5FanaW(Jb*`4!{DSMW7PFbQd$A*1(A-YxvLr@E&BGLN%DK z9Lf^pU_Zw#@avu~IKi>VxLo=zu?5$mgBK2khsyz$Z~)i=vP=4KV8)O1@z(45n6ihs z)dkoCZpYoIM)q)aaWCK&u~HNs1#*j2#R0_;KjjwSisKgQ2EK!A@@*o|`SJ~2IDPd% z9=rj-Kql#VEhTNP{qpdSE}i!KDnzOLQffT*C2p0|q=IZ4R{)yfRO{UHt=ir_=cRRg2X|39*GY0;@!qs!lt zR120-D_r21NK!0QfQDtiGjr_Ii7!|+%4YQ8Uz;dHpc{}4t z>=L5hTLuTorxvrBWYg4it=)|xIDHOflKZE!)iJ-FygHQ)(%PS zfk3;?ykiq5AI>BRZ@j=N22)t3mz6M{lv~yA%4qXaBDi2tWCqiLvtCoJ!3=0} z9yIy+U{O9anM?Ad-zxH?12!7;qC^BM!6Tjw9*lWiv%;N_lEK-qwjP@&y_1b>(*okF z?i4cji(I(|vP<>Yi1%G9+en7rot^?aR>I>kdn~7;UvBVN+eGBUcQ+V#E{@S82yP2N}z>#)C z<>*O_KZuQb3|+rR7qWzy89^}IP)p5bjM$D~9s;nB0yn}GcOanaRWpX6C4>JLg1LJT z?nHn(7PXrQn?*AK(UEn%T4hBf%F#k({)dayk7ITt2*$|a55HT89m_}C#Lq=9Wr~l2 zE2tE1L&44;Uq<=>K!iBh#f;lxSqW7 zhkxdSb!gFe<*U^%Rr?mL@vYfBxuwI`IXJQ9fN!%m#4!3AJS8MuL@xi=#ayk=Fm1H* z*|}%ujuxNmd|flSezR{&-^BX>gxiA^eID@-;gyjchXS$=!r|L7tU~$kD8xNK8vT?mXCLhZ}Hs( zn~F|fQyWyIoo&EVOV-t*FW`t&qv-Q&Cb=S{XDlUGr=diwhppfz_MO}}((Tj2#m^ZR z)#i~tU;UOz^;TbVk5Ap>ll1)kb2!{$YGnT&w5+9L{aF<4*|lW7>)3-INJu}L|$PbjLbZcLBnDjp4P-O^Ohb-G*T8P(Bk?j zi;Gc5%82qyIh-|B#EH65H4u2xr%(xHe4Iy#6@$}MVeK~)&KIHPi@!nDpn6aPB5p#Y z4B#M%$U$UE*bq>AP~a_($eV-O7>Y9wjgku}8&89)Nuij~X}<95nH0yV7Bsz?2-% zoZOjl*=J$)HZ2qs)m9_1P)$Kt)_hBB9`1Kp;M8nbPN8kk4;nzxbzru7Lxi6K+g5ZH zVVD_O5?@C+cZ+p|m==9ooECEAO;E4@3i5Px!Q$esUwRbKw=ns(05O2YKs}f+7`L>6 z{l$@kw&y)R(8-yncb>A)^405G#7Pjc9Tx>a@S)+}G#ym)oP zlI2U+(tASmu8VMAA_Z^ig1W5-xDac&2m$58luO6hBQ9oO zxS2B^fY-q(rOVOzn=d3Sg{5<1)$%JL&(CUkpiqXPz9q09eFCz;XZD8Rd zxcYw4Xgg6xG@q7Pzs2%EmVP>IMRxVKSsv_V*H|6`=|y1I$kF*V_CqU9SVIOzn=+-z z^lLIl@+UQg0ZkgY|5Jx1mnplM1>1zF=xPCZ^V5x9aF;D)uc{bL#;M*(b(K$2_0`pq zu%*HlE_}*5=Yn-kz+}5n0A6g2X~6Dqb_Q4S^&ue0?nyG=OP3#MFndZQ8B4mdHbKuOP51- zg-8jrAr>gX5>-nAw%mVXVv=T#SWgv=?i-)|mSiPtINDF!NZMb^SD@|SNs%Cs)Ii=a zKT%wW+uh-95KO@$EZ0QKy{NX2*iJW`o8>QFJfW`jNor$xE<(YdUxBGs0*KEl83Pe4 z`DRwL{l&Et>N=mK4zfD>*ERMa&bRAxNWvTt)Jq8)N`Y;Yv{0p}TuRjcSuo}anOQZ+ z+rq<1(XdW_`Wdg)aGF7lN8(Bac4C|!BK|TBo z9xc^5Kr0&r8$G9^D|iA}>sy9R22S4@RZv0mO2oh|urNc_@fkG677CBi9W|sx%H@?F zBb>H0a$3w41a=9ez#SI)c#PpQmeB=>YI-S3Bcw|3=s=@`8e5}U7iccI(X_NGCD zttuldW@fs)Sv)3CK1|pa2(x<3;4_DPNgfL*B$`O?fU=VT{Z1EP9FQDO0$9#1@rsB? zAFiAp7O5;)cdg{i3t2iY*^N|6A6&J%LJ{8Wz~Dv;&@8{R|F0f7}?6TNey zf;%Va9xhARZRG!nlv#+BSVV^Z)&TnR<-gT? zt#SGTwO#ik8r@-`N(7xpbOul&&c-qyL$DbE)dQ%W@q0{0Rg4~tpi36+I`SVNpjyKr zbZv)Nhc&teCxr!lyj&u%k`YY4kRiYPa&|J+6P|#8p!S+i+3+Pq`qgYRHaDmh4xfg< zaRUm)ny(63D=bpUBG}YA@l~n{MX3gyx>v{xU)>A4^y~iaw(Cv6mWNioqfQJYrB5YI z^CzLuUwyWEym}&O%^~gBhkxJCW`N#MdTQXzz9;vc>OSXqd8O~B=84+vzQrv*-Hu5~ zE7|{#%=Agg9=LJslO+B9^R+BvTrS*4d1+>Dy-@;}2jI$elL~4OZ)S>$yKx7HDN_bK zd_W+BJ_d<9%%o|^{SYDrfmSZyfUJ$jmSHYQr5*`AFI6CcAbJBeYOj^KagNdl;*t?P zNd{{L$ik8E=$qX7JB#;55jQlZ!jU29Q+&YOs(^E!Luk_xLgHbZ!8pnhVsq)p10b3{ z4pSt|1G1o5V%YP7k;R7!j|Ra_4||ji%E>=(E=;6yVT@_AuyqA;p;vxDe&jun3gI0m z6-pZT1<zU?*g);5tBjCDxpA3#~K%&M4(hMF;=I<@R= zbLAoB`&P%Hr6G|_YrB}_I2940=1|pm?Ut#vJN#>R__&=@+h6F9@zX6%P7=i= zp~xhJn~W?1xt8Us2H|eD5C)!8$je`6%pW1a>!QOS= zWH|^q#`h5P04tp3tcQgs?s(oeyK7LyiGV*9!Mqp8^CD2~A(Mz$370-asMlM<=HP?1E5R|5203}u7qUfc^ZvW`m z9l~}Jd;y>kBIvR{fK2w|Cmz3a^f;+`&D0d$aH6MQitU%~e)!S@BUJ;aabG@j^zws; zDlQ#;7QZ8K?=!-|m;vnm%lD05dgO4Jh}APlb44_;`5<(7tn>eN4{PFHyJ61r1AyTj#0l5bX&>*ajNiE1k0cl!Q_|< zUYI`KSj{LZsw6=re4!}(jXN#!!k9P7)zLQq-z2zxcf|G09TY20vbb*-zy>ir-^4qi zh-Y!SEr3Du!aV?q#;4C7Sw59B%bzoAB4>^-d#+DAZ&EV) z-wWo&3DcHC%7DT$rAYHD(oS?vrO))I&wNWUD?>wcxBhP?iiveBP zsB)@&vA=xrM0uUBYzf>IS;|@h)8;)}a<*i=;=;6Azoqp2s>!D9zLvg;rheb1TYUAr z&+LMWBiaGHIHFWtRWQmjWE+-@K%_hJP|*|f81%<+6#&df$u{z7?Imgsw@&&Tr|YDBq!ikmw$k{P>Mz^2ALhF%Mq_!W-xp zT1{%cHOfgMVIq!R`Vt8uju>Vgy#w|{l?g9h!ZBfTW~s>vJUc87Aw!PZ`Y;m_So?6d z9&V0>Gx6fojed56a&Y+Y2JVx?p)uj+_N8M-E*^j9CVwT@JE0Aa#E4@1AZRxmHkzpA5whI0|11q+i(M^~C(I>0Vr>`8yHZ zhTumCs1TQoF0>Nx#ppsKCH(4d)7=!F@@%x^2)31&PKf#yKb!J!lvF7Hq;yI5 zz=ZAOd~B^^KAcS$=yq1~djSuMWn4>D_rAtfh(BUx-1|@VU*6ADj7WVfXOgvkDPq|) zU}|jI6)_`93ZWM+x#3yuQp^BW9C7V-SYAWW%Q=>?ZGLYgjW&uROtPR6NR^RfQOEj<;UH zS7<(CwFQz=KhrN~*~);{5E7%~1GO$BMMp@+=o3S7>QpcZ=8%#)RdCZUq^3>{lVA#I zsZ+;j^v6tx=Z0uzV9J!K6ju^x)Q)HSq->D&=CF={!4wk1`vYBKNDA+85$IAd*5r^9 zT`I<$6jGy0!vLNZT{^he7Sf{&bH0**u5~P9&kf9&{h2b8-O2{EX(2H>K7i^;(E*v; z(n50TR501zyph(E1*g`XEDO;DP!8mzrqRpTM0NpO?GB0I4Ij=8N#PCYp-OUeDX>aP zbg8gPYIJF^N?LU3Fe5#>3Rkf3^^briBP6D-57efRl)6H2_&#+#>rg(*5AwR*b zVNCXrR&Ya}>O*snxlT+!zQ?D|@JsDi^g_%QQ65`OTMO?GU;w1>hPuK4$5!0r;0IJLu%^OV70Z>sl#5-Q>OvD zA%Qv*u^)`oX~K>$Q)l_`a|(8WcP4A2y#a697x0E6p?EpE6ijALNQo|?8_=b}JhbTI z*`#no;f+;>k85E=vBKz5VDpsdnwiK}QPP4pB`tU(V&X0wzAC^(+Co=k_^P7AR}~$; xs!*Yot@7noh8TE{!!(4~`59O66`GO6%4ny;Nt)Q delta 18724 zcmaJp31C#!weQZpXOdZx$v)Z0WDlFHfe`jB>`O2VN!|c~Y`&QQiEo&wR6(g=uNqsR zM$r;0t>MRh@YlA%l~!#hCYCVeSNQ$7)*2RD^?%!c&V4hJ0r5Z0dFS1G?tad>=bU?o zKL!;KQibU)qfy7f)AZ!&##hfhVoH)7R6mmcs9DB&j2`>*1oJp z#^G`Bx=uaE)g%L^XIKhvs7d9GHEG*8-t;u7Nr!hFyfdI$6>mNnAE{WzE#UZseH?Fj zUMAK0e7N$OOrXu=t-Ou5^NA;uo>y=Imc^1Cd@}r(!jf4kpL$eQs z-KygA_bF>!p5(2BcX^Cq`T|HRVEHn>5Xu$;Zrm#4ivTa;16WYT7sE$!n5YEsk}zHh zcxf0f1H3GZmjhlN#>WFbo}U2Snc&Iw#BG&fr&z}(Vy9WhCIQVvelq*kz##5`?@9O_ zU<3+&3M5SCr-lnC_-XKc2S1&CkMrbWbx>>uq)y>y;@DVY(DiUm1>{WSX94}}*qm_l zm5@6P@Pr+^*vgIpYW64sd&a&a7|h8WC^VC=@@V+E@HY<_CkMuu#TwM`^C7o_Ul2*T zrCEs@=VO9rmM7nnFLe~YG^~62aK~BqX{RF2wGir?9c^t9qq=Qb+N4NM4Uhl32+{(J zp}I;odKte2@HtE>mI8Mzi*Q#>6~CO#FLrZmCaZyZF24eDR>I#Z_=`*d@}GfU4f*r< zHLPSVtjSvVp0DM2Kfi9QOdwi_5kf%|zaAKI0TAxxH^8iAGY-K8-U#el1Am*?n8O|6 zNv9fEJI9j^P2LG57Xr8F;d>D@Sqn`r<~K)I1zFz>X-j}64r%xRD-NVvAbDxDGq`{n zwtyof4ex=mtl+nP zI}hk_A;OhJ8G9Qvyz)Ef$3+EWJ5a6S8$DJ04o^1U;0fH)AL!0b&uXR-415z1uZfNv zR~Y->41Y+|5~efrtw6VyP4jornD{oJSr^lAE`JvcLlaG{!7i;C6tco1$&2k|POWsmj38o?Vn3iVSq#=~B zat+yyxAZQgXlnHOq@J-p-cnaN)`j46`19HUK+kNBC$b;eIHX7)NlJZU^$)ef3a8Y& za%#+X(jyzSq>bvw7t%*xwbI#>bpb!|fy_$}FB8cTj+cwv5$>3RSDaKv)E417IG1X9 zXMFC);@sxqTs|{*;k4Xk({fjb$fizNZRQ#m2`RTWwR^X_6arQllDD+(4ryQjTYZhq zo{+LlXl(JpicEq`VHyB08VV~jGX`#cRlrM_ zg>_+Dk){l7f@fEwXSbWp?;ObanxoSTDo76PEXXE5>i==U5u$(rYv|gd27^l%QhD~k zQh0@#^!G)}6fUJuMRSS^Qd!6KO|5mjn{V}bT6PH=pfV0@EpIQ~QT(jts-CmWC(tKn z$+J}}AuFWX=0U=cZe3kdyJwytvY=Qa+RO4`60Vjz=!UY5Dye3Ax-4BE!+P{mS+a5~kSEc9m8D0?D`-=qH?85#pYc3DHS$$wRt^x)#0> z>Ak`_C?Ko{;FV+12uIkk_H9CIdz*W=r+#~D>rPw@K?Ir{nm2JuxcWU4=a28e8aff& z(!&I-W2>jpyS?4ru)WUbZsY|tbSKi$iD}UaCezwU$+p{8LJv;L%Dxv$Ou<>Q0unPz zrF{uQ`iVisL?Neta1tS;gAPrePnparJ94_?APy6}#o9LAYRg2&t1w95lx z)hc*kojpywyQQw#BOCO&$z3uiOrZa&4+<}K7Dt}IEnWN zJ-pFMKmMngE-tatsZ*EC!12IL%LGqI?rpCRDVth%dj#PSB*${JsijqDhMGG)Zl7n5 zPk0RS7t?{MMKC>|O?`bDn-)~DP?us^U2|PX(cIj;L%<{<9)ShG*ZO&@=2T`bTeM)& z@=5{cNwCqsPOHj$jx1-3f;}lI3#oT{I(D}Tyx;(m@OpO%xpe*X31kC(XnH;|(-)`b zkjLqT=>>(1NC}j{6q&Onon&rks`Gjqw|2POgw`FN2A}XO5GuG-a-E~ zV_m=u8SU85E6<$x@aW4Q9y{{iFZD_Pv!1);Y!sb399HJC_8lGp@XBQ{j8(}qDw%siI@l3PSyBk_tT40GATU&&8 zfqGxRY5rNcYB9FaOy6511X%xnjunwISfB#TVdE$Q%w-hXFhPR=cQDBs!J0y zkb-XnCXZ2Uw5xjI_*0-6SoK}pGfQ4n4ChReObR&^i6zF*4*+zIqlc=;C9`5W`0nB! z)eojl8a7Xs%7sAZSZVc&8O5w^!X^N^a13Zes^-=f-}alBSa=dE&4)@ewc9)`o;__s z#jok*6V%Mz)ym{Wvhr}vSi7jOnwg>C(hHX`71@%_ zc}UH+1P{+9IAp1B2krs+3hUFBIt8fGJM9`Zt4!D7- z$FYo#4aCjX>2r>DudRl*KV0in{u6Rv?KiG7k^YA_el9cLED>P2eE-h8nJQRGHVGla zs`eJN%Ond#A;>UC4ggrn<82G6+v++{u)R33$WovCA-@+UWZYX4zMS zUTxTD%3;-k5N>bfh3ES>@aN@3P0O~o6Ak^z_AJ=Rf4hBR09Co9IMQ%{YQb|?y9cai zwq&T(1OucCM$FWEeRaNe7n$x9Ico_Ofw;fLr@Oj+sZ=D z36p4`F_RG5)95O*1``W==l4$OEALy=H~&ojz~q7apl!)}hNb^)xH&Iqn?7uq(f?KB zJ7m&-L)G1!51{n$BoZDZ+&0cnc$r_O2U_*|R2de9O1e0zYdc*NAGAz( zap~ZO+M&wL0D~*m4CK7&des%&P&-(;d1#_LIAKdLzHV4ke^Fx&TFPFu^sVhr?lYX3 z8;q|U*321+v+g&1bxq05A#TF@^vVQFRk2AZ?a#hzu_Chr2L)|%0!nB|+g7&?tXQxc zKjtxU8vTCzLefe9*gkoBr@AN;vuAdyXKZa0yuJ!yI*xV*fUh;BY$V>&uhpf@`~?>d zGuiZ1&;`3z*g6x6GGR*rBMIAJYGr08J-O>Lxt~6~yJTq!(lf&y{`gTYQK|kMfZ#L95?aY)j1p^DtEj~9RxSkJs zTHp1EL9ZC({XyH_VMBm+ciaUES=;$W0M>Krnfx=!15@9=^D4(x$rj3B-`hkM$--a3 z&b`4+?(>@(1~)YX`L>|odv8IGw8i{Efz%^I}J(nEJU=*9=F z)bK#A?tqNf@%p2(lLq?fp~*=+7tsYIwu}+V*wOzt(Z7g~mtyV91!lO)nT^6Ei8UHl z|KL>sjbx@w1NA?S{=sh{we&B3H?7)RnuEM3%tJ6A0osrt=U6Xj?~XPPSn?|R_}<+4 zuop2_TaIK;VVzM16Oa)@^0wAC2(XMX7FNO>zwk!A1vhGSpfalldFQu~`z3ghD1>a) z2Q=rB&l!U8Ylk)K=#D_91J-`_K+=HjTz+tMZE)4*pv^sO*fL^BpvMC~<$IbHLQa21 zS3s5=9mCf~Nt7gi*((&#Gxru5namYK7hOb9)2sI$R+ZrxH1vV{W~-U{`8!8{f8SCM zQ)6t9xX(%B8{6Y1V;b8EE1~P)e2k5h?P};6$LwX{ieaQ^GDQRkW+1R{9%-bn?<+S& zMjqyO#rI~~Va!zn3kRm1s|jv!2iI>2+UkZ4^>qFBvJ)X|(YgF{$v7FCJ;B{z{H~yF_po6P zwchU}HT{$CZz3hA^aa#+Ax*>f#wK3yv9*QFAeN}gT8!3OQz8k_uJ?@kLohm_AG<#Sq{;5Ij?9Wu;J4r!cYsPQ->$lD6 z*n}u&yKHmJE^A=58EMaYJALpWlX5l4-g5f4hg=icPEdtoFGaw%w5Wb(GS#Um6+8{# z=SOureeYnw=5{pdiZVa?dEb>^J@&z4eKTFETQ}3iSYvb^A%Be7ZnJ;hv~|+;-=C2G zAd1*4&`hj^g*Ujo{Qb%iTjGA*h$(@_cR1*xhihkDu%wNcZKDn@&T=D})50z)3FU1y z9rKk!T4`(;OuCPj9CFA`a&+bWNwn_Jqo|>D^wi;^zycUZNWNn2nlYy7)JA5dREi6& z0UC%AG2WSo?52S5i1#~~rw(W4KA?(W9xP*QZ^Jm6Tk2G&Q?eK6!ADZ2{1_)=``Ba@ z^iJt5=vzAA7|@@aI=pscux88o8qZ*jXQ*b|(Aw?6HI2de9mATP^!-OB2EO1FS{)m( z%Oczoqjy4`f)ht*K(n{8A#6#bu`S@fDY;clQR2WPvuHNvGBvsmu}ugzBiMq#gCHA; zb65$cz7sGJ!6)$NMdsMYjmiv4U5{mygIBNhI=)6zX$Dgn9NCmi@u zigF)NoTAVD=xPEkd`Q>Wj8hulEYaD{c}xHGVVM0_+_XW zMHw88$Ur~JAzRXFI@)|x4Sv%~(Mk^-%>bWd(lH&kFgQ7Zh4Kn&OwdsBWImZl3!dx@;3^7@n996fMJw3-lCjTb>;Io9 zo$x&@iTxLn5ySQ(LuNt$2Qd{pFg9tzL-6TE7J~KGvNdPEe^HM^_b`@6F`q@Nd&*T8 z4XHzh^!|fA@uV=WVs3`~wG8Fle04}2j(#_&pq9v^2L#=@>jm_+r=0Y&rxLUEK4dHQ zh|1@oZhTtMNmS!ZgF`5%E5@g)&>;jr4bUV&)oiQK;5zWyV_IH!6gQfCo^~1yqMA4I zCYBOMUwAruoSBb@lmwPy5oNsfsA3!8ZAWF1)?jbqcsu>->5K)5d{QIf9lTK_j6V6O zOw{lxN9BC#$+U=>$fqBb`?Y+A2%S16<1=~ZQJG(d94$kAOO~m zA>2_zz)tIba(^P@e7VH=Os^_Vl(>dsk`JJZyK3eP7PP?Q ztA{l!Ml>c#Zo1}Om@sqLoZYi{*jzd=zW={^RkGB4^E@;6y17y|KLM1bA?SEkvd5Nm}yvC!@ceVk-k{_e@_vNVA7WgFX>vNIKG}M z{i=T)xtls)tRN@omKPmqIN(tH)^wkM?2*BoB_yyE4v;XlISCLd>Fs`8Yj$T*a zwBd^N;42JOYzfY+3mWT(6%GB0e;+T)KLnMaaB<*vvHEyV1MDeH5J+&hv^UqoA=pdU zC%XAewsaO!Kp!}>ZI&<{sM%cwn2Q3$-vbak8zFWk84lgY9NqmAs@p}!zXV5~{k1Qx zCxPE!X>=(=q=@MOmll*fM5@7BYH5J@V0~+A6UcpwPdE)_LrOU2ggAv0i($AELyc`s zbqyXC>V)p#j?Xsio4R9Ys}taiIy|UXp{7pI=ibBp<9}I0G>f4OxF56Vqc3Mqif+gt zmF(2@e;^lFQW6(wqI3`u7ue~dS2zci9i)GKc>_dd=l$wTfN_e7ok$4GqCx#R+!z~>J{0=I43`Z~r%7Fo2KI7#zL*;Y27F?v_ zuP57?d0@Wv;I#jjzy6ApFrS2Z@33#fnJUZ~s>zV7kq^nYHSY?^7c5%`iHvz`>8Atf zc~0yhvte+`g+~#vVdy05{4FRiyiH5brtANJ=k-}^6c-6 z+xrRz?K6ip6&KVN&?bkB2aLgti33H0cBWtS@m^<`sY2QwJf8Q%zB)~eudR~il z2LU`Mm$)F>FE9aXAU&jup40L|E>g3??*Zn;Et_dOX&LQN%JA_r3zU0>KLPDaIJ^Bo z57Dc<{uf^BCv(R-aRCTmim*A>#7(@$SYP6K=};cLJCLOR1mZX4^e1n$+QE`+YV-h0 zzW`|=H7|6yh4z;IoHr|FdPtETUP~|k#X)r4f0hwD?fLDZ?#nXJhdsZYM=O7qN)qYj z-%TeDx-#EF2R^aWSASP;!7D`U$>9zYA>VKIrO_#GU5TMu@b)t#x&NjZCSBq*PDaH|)x!uI2S>B#Z-}!S|p>Q6` ze}OD`<-wkhe)TIb&8{5&{s&L=Tv=K*nfChrtoi!ru(vuq**};-ohO36G|>* z28Rw)L@HKtPs({d7#lGEeI*B)^J) z@!vJTLU9r!weA7|Z3JG#9yvK_C;kL0{S?7V2#zCQjQknwz=@jho|!^tU$Xxoq|$u} z^=V-lyfSJIjaS=C`%FXjS#;t?XJXQjv7l@Ih|zN1m@#O~7&1D$=3h+BqRpYwg?WAB z&P*Jzor@bv+HhET!@wB}MiSFT>?tG8@{y#BziX97)75w`sbEyjrPY!f>73FSG>&^u zk^kk5M9!LijZ+xy7c6$kCQ_NN6>Q~DM!g3{~G^o%=R6PT% z*Uiz{K6@HZcILyNAan+fi`=jBp@^^tTtzzgdRhROUdh~rsIWk+IVxCU^rR>(LSyAs zJ`_3j+$2Kk*wdw$bFk=H8v#2&EZQ~X9z2_idIS|=KBBeKT5 z#)G?=$O0B4vc~1?V2Z^UQ8FKU@xzxN|L~Xl+70yOKUe958z>_vsq|le9+$NMzrFwf z`|B<8`Zi{P(2jrKl@%Y7^K~8npXAb(_vZ%wjzz}Abxg*Xnal*&B$<@oDYF`A2h1SY zQ~!aLu|6@=nRVwl^gAr3kn1(UC)@EQKrdZTFlQm;0J(K!3I7730+iYYK)Se5GnZ9( zBJHWPKIgm62?zF`Nv989%F$emk>2U4!h8?iV>5)wz!ibpahj961>$K|Q=3vpTWH0;9#q4cAR@lJG#r1DUkq&29ZJ z|K(wHxt3hXB`Nf-E6&O^AYqTFiAab)&KFI^SPMRAH%J)AMbqL%`Hl?u`Ar%fy`rJ- zUa`o;gvXU|swVAh8jO^i>EIQsir4PrqIt>G|3MYWq-Q^vV9fBLm%*O>nUs8(tsnc& zpr!B)Yal(m0gWG{#)p@U*yGEk_kCFH7#H~#Es-IB=*IcR(N8~I;D}O-wwT-mUkY9F zkrV8Rdp=6c24lsa=yOHuNrdnXs6>4HNuokX^%Zspl_b{&@tR{XYOf^~q`Wv{ZyA#! zg+L+Z(q$hzl5$-@nBq0eB;22$W_zj@Ul3~+ML_NZO(3G}1 zF>DZ|_*2K?Ct^xu2%-bkw3i&TiK%|@TU!WR2uQw1?<;4$1|TL!XhD&4{xne&X;|Y= zhcV^(3hAa#3iTL*N)prVk%?*akxy*ubRioCH6b#nlyu&K-4cPOkwnf#Kv%u#gK*&| zg(QileVRyS(Mg|9NSqXH+$P#jW<;ig-uLM`^9oo<$u+>46*}n0pZp8f^X*AC8Jb4GD=Z!dp7V*=n`2_M6SNe8|3S#iQq ztS1{l#I1MGrJp57!U0el1TlL)YrY*_#^2In={o7Izh%MM&kz50gFPu5w@^ZKB$9q* zE}&NO5#caJD59TSE!V4I!2|^Wud_Q-N0RC6YZ}Gu*aT^!ov57z7r8Ur^tPFYEhT)&uW2*#E7<(@7$RnV7u zX5(w(@S2+7rk0^KtwRgig7fbR8iiqnmma;bQ)W-4<}b=EhJ^Emyg@@=Py30zPwnmN z95&23UjGG|1}B`Qeah3iQ@UR9%+!IJfu+F}a5%#UH+tY4ek)mq$MN>%vM=DedA00X zHkVvJYUJ$2H_B+&H7o7=yJ3c+_N!}42xnbEz6#k_dr@`1k!7i#sd!_CWra);(uboK zcr*|)#9lLR&{FgB)<6jVdJMTn za||)I6P7W=bzzF_J#AxDp5@R1hcx}DIF=iW$nQSJ}w8Mg~%Y-<% z-eiukehiVC!R(=+7UN^{#CVijxZM^}{rdGnj8l`~xA?6$nMrYCyqF+bL}SGG@!Mkh zsu1Iuk(t1&jwm2LZs9fX)`~W;sLmd;(!F0AY12pAWbkxsqEfVqYEcVkw&Cc!KT+Cr z?AS(Z4R8kwYXI-qowzkdiys8-a_})Qy~bus9~mv~U@&4(Y*#!K#vZCNhDeX(zOTK2v6 zgBL-I!#l=ki(ejNmp%7&vAze@!BYsHrkB6AB;R72#i|6;|11bRiIhJ@pg|GW0&rPE zre*WitXfp%uBu)>f6;=~?CP^{8P-?0jsUM^3OljTOvEY?MEx*HX)*ou3naM{bCEX$ zR1;EoGpcg}u4Ew%lZTLC7(p$9bWD2-i{I>-3GZRAYy-A20QoAH^+lg8Y}%u?_iN74}Y7J{Y${8HKx=5{KjR-=1SKSa3rmT zH}Yu0KQ~yf%DK|nNOXBA%)Nd%1 zfYU86*&lnBl$GQG(Q`6s&w^h)V~m{zGGSW6F=gR@ajg&p6_2%&R0xvRzU08dxoFA> zc*-ES0K~+hg>SG7f?>sDU$Zj5$#-;@5i&KCMV@b^{~x4BFrw%_Ku8M7>3)`wNu=QP zp9rY{8@M}JP99UkElMvKaj3ewe=jGQaP$J!4-27n|3OY{)ZC=W@f$!H8hL}?1b=a& zVJ?KyYBN9`=tLcFME3TZF(j85XKIXI!1Bk7@!eaL#I|WnDTCDs#Ox6he5nyUVkZe& zF#)cqB=82T6U;V+Kfz~-P>BhW!N^Bd7Z~55s?`ZlDqb|Ap`9cqjA zGi~&T^M1>%`QuhwMN5}Pw05b)I6kfu)S*A_VE>qfE?UApFpJg*)~SPNQha0tej8Yp zWxl9I&L^PFKFcB|fHt&>T7Wju0=A|FM+aYa(F$MI@K>T}1LYUpYQU_`fYorJ%Er~O z7n>pYWs5aKMN_z=ro-f5)=eR{vS6kD%&1zDpLaRvV=&n?ge=m{laMmGnF#DfdrECOaBGK+8y zly%u-EJBb#3nvs|ry^cIa4BcPqh|i%hUC<{$^!~giYfzKbGwDRdQx$y`a5MJw zK|Q&fWPn{*-0OK^$8$S+?>yspZANg#=AkNgaL$&Xv2IvV-<@Y9&c*L3_I!1%nsAmy z(jD5flXDlF6*sxH(Wq@63_12{c1!?PC8}tp+uA*0e|bZ5%ZBEh@{6p> zBE{Jv?HsfE?4ea!Bzx25Zr}8L=Zr5FM=xw4&pyYA{<7PxtY%*_EQ8eBNzY>(y`0kUifX87k*B9 z^AE+Cu?Ig$X>Ws{(T>@CX6zEP^qBRB7muWi3r7%R=H6+7=}SMmqVIZB!>CMTg=38N8n0ujA{guKP>|$+qaov?p>#Ie7I#2E0a- zPya0g+(w6)6g-iAorBju@;YTica=jp+Ntb@AGvG{jIJU!V(Wq1aq#M`x{j~W8C;^{ zs&OHh8Y4#Ys2q_?def*9kx>;FmoTbksD_KTjcOUHgWHp%dWIUfcQOo1r)#sx>m-m)Tq9=ds2tvxObMe(c!%3TOv5E6k7^Oq zakivUJz@q9=!}S&xa9QFIK<4@T0CNNNFJ#rBhV!|B9~0@qe?`eQm9JJPz~0lW#|GO z)~08;0jo1IRO&rLSCK`8Y$hXyj8QqmF6phKN`{5&M7-OTM^XY(s#K+f8#rs`s8Pb9 zQuFmV3Evow2p5UVHMS>wp8z(mp^jQ|lg07RfYXc=yU1FGSiIip&JO8sZ3 u0c$cc6pk>7i7gep*;2t9dkidrn6y-gN#zlfX4FB@8sdCf+ diff --git a/logging_config.py b/logging_config.py new file mode 100644 index 0000000..1114458 --- /dev/null +++ b/logging_config.py @@ -0,0 +1,65 @@ +import logging +import os +import sys +from logging.handlers import RotatingFileHandler +from pathlib import Path + + +BASE_DIR = Path(__file__).resolve().parent +DEFAULT_LOG_DIR = BASE_DIR / "logs" +DEFAULT_LOG_FILE_NAME = "gmv_sync.log" +LOGGER_NAME = "gmv_analytics" +LOG_FORMAT = "[%(asctime)s] %(message)s" +DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + +_IS_CONFIGURED = False + + +def _build_handlers() -> list[logging.Handler]: + log_dir = Path(os.getenv("LOG_DIR", str(DEFAULT_LOG_DIR))) + log_file_name = os.getenv("LOG_FILE_NAME", DEFAULT_LOG_FILE_NAME) + log_file_path = log_dir / log_file_name + log_dir.mkdir(parents=True, exist_ok=True) + + formatter = logging.Formatter(LOG_FORMAT, datefmt=DATE_FORMAT) + + console_handler = logging.StreamHandler(sys.stdout) + console_handler.setFormatter(formatter) + + file_handler = RotatingFileHandler( + log_file_path, + maxBytes=int(os.getenv("LOG_MAX_BYTES", str(10 * 1024 * 1024))), + backupCount=int(os.getenv("LOG_BACKUP_COUNT", "7")), + encoding="utf-8", + ) + file_handler.setFormatter(formatter) + return [console_handler, file_handler] + + +def setup_logging(force: bool = False) -> logging.Logger: + global _IS_CONFIGURED + + logger = logging.getLogger(LOGGER_NAME) + if _IS_CONFIGURED and not force: + return logger + + level_name = os.getenv("LOG_LEVEL", "INFO").strip().upper() + level = getattr(logging, level_name, logging.INFO) + handlers = _build_handlers() + + if force: + for handler in list(logger.handlers): + logger.removeHandler(handler) + handler.close() + + logger.setLevel(level) + logger.propagate = False + for handler in handlers: + logger.addHandler(handler) + + _IS_CONFIGURED = True + return logger + + +def get_logger() -> logging.Logger: + return setup_logging() diff --git a/prefect_gmv_scheduler.py b/prefect_gmv_scheduler.py index b4d3265..ac8505e 100644 --- a/prefect_gmv_scheduler.py +++ b/prefect_gmv_scheduler.py @@ -22,9 +22,10 @@ def bootstrap_prefect_env() -> None: bootstrap_prefect_env() from prefect import flow, task -from prefect.logging import get_run_logger from project_bitable_common import ( + DailySyncConfig, + build_daily_sync_context, get_base_url, get_default_end_date, get_default_start_date, @@ -36,10 +37,8 @@ from project_bitable_common import ( iter_dates, load_dotenv, log_step, - run_gmv_sync, - run_metric_sync, - send_feishu_app_card, - send_feishu_sync_card, + run_daily_sync_for_date, + send_sync_notification, validate_date, ) @@ -82,75 +81,22 @@ def load_sync_config( ) -@task(name="sync-sales") -def sync_sales_for_day(stat_date_text: str, config: SyncConfig) -> None: - stat_date = validate_date(stat_date_text) - run_metric_sync( - metric_name="sales", - start_date=stat_date, - end_date=stat_date, - base_url=config.base_url, - mapping_file=Path(config.mapping_file), - store_field_name=config.store_field_name, - table_map_file=config.table_map_file, - dry_run=config.dry_run, - skip_missing_tables=config.skip_missing_tables, - ) - - -@task(name="sync-refund") -def sync_refund_for_day(stat_date_text: str, config: SyncConfig) -> None: - stat_date = validate_date(stat_date_text) - run_metric_sync( - metric_name="refund", - start_date=stat_date, - end_date=stat_date, - base_url=config.base_url, - mapping_file=Path(config.mapping_file), - store_field_name=config.store_field_name, - table_map_file=config.table_map_file, - dry_run=config.dry_run, - skip_missing_tables=config.skip_missing_tables, - ) - - -@task(name="sync-gmv") -def sync_gmv_for_day(stat_date_text: str, config: SyncConfig) -> None: - stat_date = validate_date(stat_date_text) - run_gmv_sync( - start_date=stat_date, - end_date=stat_date, - base_url=config.base_url, - store_field_name=config.store_field_name, - dry_run=config.dry_run, - skip_missing_tables=config.skip_missing_tables, - ) - - @task(name="notify-feishu") -def notify_feishu(title: str, config: SyncConfig) -> None: - if config.dry_run: - log_step("dry-run 模式下跳过飞书群推送") - return - start_date = validate_date(config.start_date) - end_date = validate_date(config.end_date) - if config.chat_id: - send_feishu_app_card( - chat_id=config.chat_id, - title=title, - base_url=config.base_url, - start_date=start_date, - end_date=end_date, - image_path=config.report_image_path, +def notify_feishu(context: SyncConfig, start_date_text: str, end_date_text: str) -> None: + context_obj = build_daily_sync_context( + DailySyncConfig( + base_url=context.base_url, + mapping_file=Path(context.mapping_file), + store_field_name=context.store_field_name, + table_map_file=context.table_map_file, + dry_run=context.dry_run, + skip_missing_tables=context.skip_missing_tables, + chat_id=context.chat_id, + report_image_path=context.report_image_path, + group_webhook=context.group_webhook, ) - return - send_feishu_sync_card( - webhook_url=config.group_webhook, - title=title, - base_url=config.base_url, - start_date=start_date, - end_date=end_date, ) + send_sync_notification(context_obj, validate_date(start_date_text), validate_date(end_date_text)) @flow(name="gmv-daily-sync") @@ -161,7 +107,6 @@ def gmv_daily_sync_flow( skip_missing_tables: bool = False, table_map_file: str = "", ) -> None: - logger = get_run_logger() config = load_sync_config( start_date=start_date, end_date=end_date, @@ -174,22 +119,31 @@ def gmv_daily_sync_flow( end_date_value = validate_date(config.end_date) if end_date_value < start_date_value: raise ValueError("end-date 不能早于 start-date") + context = build_daily_sync_context( + DailySyncConfig( + base_url=config.base_url, + mapping_file=Path(config.mapping_file), + store_field_name=config.store_field_name, + table_map_file=config.table_map_file, + dry_run=config.dry_run, + skip_missing_tables=config.skip_missing_tables, + chat_id=config.chat_id, + report_image_path=config.report_image_path, + group_webhook=config.group_webhook, + ) + ) - logger.info("Prefect 调度开始: %s -> %s", config.start_date, config.end_date) + log_step(f"Prefect 调度开始: {config.start_date} -> {config.end_date}") try: for stat_date in iter_dates(start_date_value, end_date_value): - stat_date_text = stat_date.isoformat() - logger.info("开始处理日期: %s", stat_date_text) - sync_sales_for_day(stat_date_text, config) - sync_refund_for_day(stat_date_text, config) - sync_gmv_for_day(stat_date_text, config) - logger.info("日期处理完成: %s", stat_date_text) + log_step(f"Prefect 开始处理日期: {stat_date.isoformat()}") + run_daily_sync_for_date(context, stat_date) except Exception: - notify_feishu.with_options(name="notify-feishu-failed")("GMV数据同步失败", config) + notify_feishu.with_options(name="notify-feishu-failed")(config, config.start_date, config.end_date) raise - notify_feishu("GMV数据同步", config) - logger.info("Prefect 调度完成") + notify_feishu(config, config.start_date, config.end_date) + log_step("Prefect 调度完成") def parse_args() -> argparse.Namespace: diff --git a/project_bitable_common.py b/project_bitable_common.py index 9a5c503..51a52b1 100644 --- a/project_bitable_common.py +++ b/project_bitable_common.py @@ -13,6 +13,8 @@ import psycopg2 import requests from openpyxl import load_workbook +from logging_config import get_logger, setup_logging + BASE_DIR = Path(__file__).resolve().parent ENV_FILE = BASE_DIR / ".env" @@ -242,9 +244,39 @@ class SyncResult: feishu_only_stores: list[str] +@dataclass(frozen=True) +class MetricSyncArtifacts: + created_fields: set[str] + + +@dataclass(frozen=True) +class DailySyncConfig: + base_url: str + mapping_file: Path + store_field_name: str + table_map_file: str + dry_run: bool + skip_missing_tables: bool + chat_id: str = "" + report_image_path: str = "" + group_webhook: str = "" + + +@dataclass +class DailySyncContext: + client: "FeishuBitableClient" + tables: list["TableInfo"] + mapping: dict[str, str] + table_overrides: dict[str, str] + config: DailySyncConfig + + def log_step(message: str) -> None: - now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print(f"[{now}] {message}", flush=True) + get_logger().info(message) + + +def log_debug(message: str) -> None: + get_logger().debug(message) def preview_list(values: list[Any], limit: int = 20) -> str: @@ -264,6 +296,7 @@ def load_dotenv(env_file: Path = ENV_FILE) -> None: continue key, value = line.split("=", 1) os.environ.setdefault(key.strip(), value.strip()) + setup_logging(force=True) def get_base_url() -> str: @@ -468,6 +501,22 @@ def build_target_project_months( return result +def build_daily_sync_context(config: DailySyncConfig) -> DailySyncContext: + mapping = load_store_mapping(config.mapping_file) + table_overrides = load_table_name_overrides(config.table_map_file) + if table_overrides: + log_step(f"加载表名映射完成: {len(table_overrides)} 条") + client = FeishuBitableClient(config.base_url) + tables = client.list_tables() + return DailySyncContext( + client=client, + tables=tables, + mapping=mapping, + table_overrides=table_overrides, + config=config, + ) + + def get_feishu_token() -> str: app_id = os.getenv("FEISHU_APP_ID") app_secret = os.getenv("FEISHU_APP_SECRET") @@ -494,6 +543,8 @@ class FeishuBitableClient: log_step(f"解析飞书 Base URL 成功: app_token={self.app_token}") self.token = get_feishu_token() self._fields_cache: dict[str, list[dict[str, Any]]] = {} + self._records_cache: dict[str, list[dict[str, Any]]] = {} + self._tables_cache: list[TableInfo] | None = None self.session = requests.Session() self.session.headers.update( { @@ -514,9 +565,9 @@ class FeishuBitableClient: url = f"https://open.feishu.cn{path}" params = kwargs.get("params") if params: - log_step(f"请求飞书接口: {method} {path} params={params}") + log_debug(f"请求飞书接口: {method} {path} params={params}") else: - log_step(f"请求飞书接口: {method} {path}") + log_debug(f"请求飞书接口: {method} {path}") resp = self.session.request(method, url, timeout=60, **kwargs) try: payload = resp.json() @@ -538,6 +589,9 @@ class FeishuBitableClient: return payload.get("data") or {} def list_tables(self) -> list[TableInfo]: + if self._tables_cache is not None: + log_debug(f"命中数据表缓存: 表数={len(self._tables_cache)}") + return self._tables_cache log_step("开始获取多维表格数据表列表") items: list[TableInfo] = [] page_token = "" @@ -553,15 +607,16 @@ class FeishuBitableClient: page_token = data.get("page_token") or "" if not page_token: break - log_step(f"飞书返回的数据表名称: {preview_list([item.name for item in items])}") + log_debug(f"飞书返回的数据表名称: {preview_list([item.name for item in items])}") log_step(f"数据表列表获取完成: {len(items)} 个") + self._tables_cache = items return items def list_fields(self, table_id: str) -> list[dict[str, Any]]: if table_id in self._fields_cache: - log_step(f"命中字段缓存: table_id={table_id}, 字段数={len(self._fields_cache[table_id])}") + log_debug(f"命中字段缓存: table_id={table_id}, 字段数={len(self._fields_cache[table_id])}") return self._fields_cache[table_id] - log_step(f"开始获取字段列表: table_id={table_id}") + log_debug(f"开始获取字段列表: table_id={table_id}") items: list[dict[str, Any]] = [] page_token = "" seen_tokens: set[str] = set() @@ -574,7 +629,7 @@ class FeishuBitableClient: if page_token in seen_tokens: raise RuntimeError(f"字段列表分页出现重复 page_token，已中断: table_id={table_id}, page_token={page_token}") seen_tokens.add(page_token) - log_step(f"获取字段列表分页: table_id={table_id}, page={page_index}, page_token={page_token or ''}") + log_debug(f"获取字段列表分页: table_id={table_id}, page={page_index}, page_token={page_token or ''}") data = self.request( "GET", f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/fields", @@ -583,7 +638,7 @@ class FeishuBitableClient: items.extend(data.get("items") or []) has_more = data.get("has_more", False) next_page_token = data.get("page_token") or "" - log_step( + log_debug( f"字段列表分页完成: table_id={table_id}, page={page_index}, 累计字段数={len(items)}, " f"has_more={has_more}, next_page_token={next_page_token or ''}" ) @@ -595,16 +650,16 @@ class FeishuBitableClient: if page_index >= 100: raise RuntimeError(f"字段列表分页超过 100 页，已中断: table_id={table_id}") self._fields_cache[table_id] = items - log_step(f"飞书返回的字段名称: table_id={table_id}, fields={preview_list([item.get('field_name', '') for item in items])}") - log_step(f"字段列表获取完成: table_id={table_id}, 字段数={len(items)}") + log_debug(f"飞书返回的字段名称: table_id={table_id}, fields={preview_list([item.get('field_name', '') for item in items])}") + log_debug(f"字段列表获取完成: table_id={table_id}, 字段数={len(items)}") return items - def ensure_number_field(self, table_id: str, field_name: str) -> None: - log_step(f"检查字段是否存在: table_id={table_id}, field={field_name}") + def ensure_number_field(self, table_id: str, field_name: str) -> bool: + log_debug(f"检查字段是否存在: table_id={table_id}, field={field_name}") fields = self.list_fields(table_id) if any(item.get("field_name") == field_name for item in fields): - log_step(f"字段已存在，跳过创建: table_id={table_id}, field={field_name}") - return + log_debug(f"字段已存在，跳过创建: table_id={table_id}, field={field_name}") + return False log_step(f"开始创建数字字段: table_id={table_id}, field={field_name}") self.request( "POST", @@ -613,26 +668,27 @@ class FeishuBitableClient: ) self._fields_cache.pop(table_id, None) log_step(f"字段创建完成: table_id={table_id}, field={field_name}") + return True def try_list_field_groups(self, table_id: str) -> list[dict[str, Any]] | None: path = f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/field_groups" url = f"https://open.feishu.cn{path}" - log_step(f"尝试读取字段编组: table_id={table_id}") + log_debug(f"尝试读取字段编组: table_id={table_id}") resp = self.session.request("GET", url, timeout=60) try: payload = resp.json() except ValueError: - log_step(f"字段编组接口返回非 JSON，跳过读取: table_id={table_id}, status={resp.status_code}") + log_debug(f"字段编组接口返回非 JSON，跳过读取: table_id={table_id}, status={resp.status_code}") return None if resp.status_code == 404: - log_step(f"字段编组读取接口不可用: table_id={table_id}, payload={payload}") + log_debug(f"字段编组读取接口不可用: table_id={table_id}, payload={payload}") return None if resp.status_code >= 400 or payload.get("code") != 0: - log_step(f"字段编组读取失败，跳过比对: table_id={table_id}, payload={payload}") + log_debug(f"字段编组读取失败，跳过比对: table_id={table_id}, payload={payload}") return None data = payload.get("data") or {} items = data.get("items") or data.get("field_groups") or [] - log_step(f"字段编组读取成功: table_id={table_id}, 编组数={len(items)}") + log_debug(f"字段编组读取成功: table_id={table_id}, 编组数={len(items)}") return items def create_field_groups(self, table_id: str, field_groups: list[dict[str, Any]]) -> None: @@ -648,13 +704,19 @@ class FeishuBitableClient: log_step(f"字段编组创建请求完成: table_id={table_id}") def list_records(self, table_id: str) -> list[dict[str, Any]]: - log_step(f"开始获取记录列表: table_id={table_id}") + if table_id in self._records_cache: + log_debug(f"命中记录缓存: table_id={table_id}, 记录数={len(self._records_cache[table_id])}") + return self._records_cache[table_id] + log_debug(f"开始获取记录列表: table_id={table_id}") items: list[dict[str, Any]] = [] page_token = "" + page_index = 0 while True: + page_index += 1 params = {"page_size": 500} if page_token: params["page_token"] = page_token + log_debug(f"获取记录列表分页: table_id={table_id}, page={page_index}, page_token={page_token or ''}") data = self.request( "GET", f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/records", @@ -662,18 +724,29 @@ class FeishuBitableClient: ) items.extend(data.get("items") or []) page_token = data.get("page_token") or "" + log_debug( + f"记录列表分页完成: table_id={table_id}, page={page_index}, " + f"累计记录数={len(items)}, next_page_token={page_token or ''}" + ) if not page_token: break - log_step(f"记录列表获取完成: table_id={table_id}, 记录数={len(items)}") + log_debug(f"记录列表获取完成: table_id={table_id}, 记录数={len(items)}") + self._records_cache[table_id] = items return items def update_record(self, table_id: str, record_id: str, fields: dict[str, Any]) -> None: - log_step(f"更新记录: table_id={table_id}, record_id={record_id}, fields={list(fields.keys())}") + log_debug(f"更新记录: table_id={table_id}, record_id={record_id}, fields={list(fields.keys())}") self.request( "PUT", f"/open-apis/bitable/v1/apps/{self.app_token}/tables/{table_id}/records/{record_id}", json={"fields": fields}, ) + cached_records = self._records_cache.get(table_id) + if cached_records is not None: + for item in cached_records: + if item.get("record_id") == record_id: + item.setdefault("fields", {}).update(fields) + break def load_table_name_overrides(path_text: str) -> dict[str, str]: @@ -727,13 +800,13 @@ def send_feishu_sync_card( log_step("飞书群卡片消息推送完成") -def upload_feishu_image(image_path: str) -> str: +def upload_feishu_image(image_path: str, token: str | None = None) -> str: if not image_path: return "" path = Path(image_path) if not path.exists(): raise FileNotFoundError(f"未找到预览图文件: {path}") - token = get_feishu_token() + token = token or get_feishu_token() headers = {"Authorization": f"Bearer {token}"} with path.open("rb") as image_file: files = {"image": (path.name, image_file, "application/octet-stream")} @@ -758,12 +831,13 @@ def send_feishu_app_card( start_date: date, end_date: date, image_path: str = "", + token: str | None = None, ) -> None: if not chat_id: log_step("未配置 FEISHU_CHAT_ID，跳过应用机器人卡片推送") return - token = get_feishu_token() - image_key = upload_feishu_image(image_path) if image_path else "" + token = token or get_feishu_token() + image_key = upload_feishu_image(image_path, token=token) if image_path else "" elements: list[dict[str, Any]] = [ { "tag": "div", @@ -854,7 +928,7 @@ def find_target_table( f"{year:04d}_{month:02d}", f"{year:04d}.{month:02d}", } - log_step(f"项目表名别名: 项目={project}, aliases={get_project_table_aliases(project)}") + log_debug(f"项目表名别名: 项目={project}, aliases={get_project_table_aliases(project)}") candidates = [] project_related_tables = [] for table in tables: @@ -944,7 +1018,7 @@ def build_field_group_payload( other_fields.append(child) groups: list[dict[str, Any]] = [] if gmv_fields: - groups.append({"name": "GMV", "children": gmv_fields}) + groups.append({"name": "GMV编组", "children": gmv_fields}) if other_fields: groups.append({"name": "销售额/退款额", "children": other_fields}) return groups @@ -992,7 +1066,7 @@ def ensure_field_groups_for_table( if existing_normalized == desired_normalized and existing_order == desired_order: log_step(f"字段编组已存在且一致，跳过创建: 表={table.name}") return - log_step( + log_debug( f"字段编组存在差异: 表={table.name}, " f"existing={ {name: len(ids) for name, ids in existing_normalized.items()} }, " f"desired={ {name: len(ids) for name, ids in desired_normalized.items()} }, " @@ -1040,29 +1114,71 @@ def sync_field_groups_for_range( skip_missing_tables: bool, ) -> None: log_step(f"开始同步字段编组: {start_date.isoformat()} -> {end_date.isoformat()}") - table_overrides = load_table_name_overrides(table_map_file) - if table_overrides: - log_step(f"加载表名映射完成: {len(table_overrides)} 条") - client = FeishuBitableClient(base_url) - tables = client.list_tables() + context = build_daily_sync_context( + DailySyncConfig( + base_url=base_url, + mapping_file=get_mapping_file(), + store_field_name=store_field_name, + table_map_file=table_map_file, + dry_run=dry_run, + skip_missing_tables=skip_missing_tables, + ) + ) for (year, month), _ in sorted(build_group_dates_by_month(start_date, end_date).items()): for project in SUPPORTED_PROJECTS: try: - table = find_target_table(tables, project, year, month, table_overrides) + table = find_target_table(context.tables, project, year, month, context.table_overrides) except ValueError as exc: - if skip_missing_tables: + if context.config.skip_missing_tables: log_step(f"跳过字段编组同步: {exc}") continue raise ensure_field_groups_for_table( - client=client, + client=context.client, table=table, - store_field_name=store_field_name, - dry_run=dry_run, + store_field_name=context.config.store_field_name, + allowed_field_names=None, + dry_run=context.config.dry_run, ) log_step("字段编组同步完成") +def sync_field_groups_for_day_with_context( + context: DailySyncContext, + stat_date: date, + force_full_rebuild: bool = False, + allowed_field_names_override: set[str] | None = None, +) -> None: + day_mode = "完整编组" if force_full_rebuild or stat_date.day == 1 else "增量编组" + log_step(f"开始同步字段编组: 日期={stat_date.isoformat()}, 模式={day_mode}") + allowed_field_names: set[str] | None = None + if allowed_field_names_override is not None: + allowed_field_names = set(allowed_field_names_override) + elif not force_full_rebuild and stat_date.day != 1: + allowed_field_names = { + get_gmv_field_name(stat_date), + get_sales_field_name(stat_date), + get_refund_field_name(stat_date), + } + log_step(f"增量编组目标字段: 日期={stat_date.isoformat()}, fields={sorted(allowed_field_names)}") + for project in SUPPORTED_PROJECTS: + try: + table = find_target_table(context.tables, project, stat_date.year, stat_date.month, context.table_overrides) + except ValueError as exc: + if context.config.skip_missing_tables: + log_step(f"跳过字段编组同步: {exc}") + continue + raise + ensure_field_groups_for_table( + client=context.client, + table=table, + store_field_name=context.config.store_field_name, + allowed_field_names=allowed_field_names, + dry_run=context.config.dry_run, + ) + log_step(f"字段编组同步完成: 日期={stat_date.isoformat()}, 模式={day_mode}") + + def sync_field_groups_for_day( stat_date: date, base_url: str, @@ -1071,37 +1187,17 @@ def sync_field_groups_for_day( dry_run: bool, skip_missing_tables: bool, ) -> None: - day_mode = "完整编组" if stat_date.day == 1 else "增量编组" - log_step(f"开始同步字段编组: 日期={stat_date.isoformat()}, 模式={day_mode}") - table_overrides = load_table_name_overrides(table_map_file) - if table_overrides: - log_step(f"加载表名映射完成: {len(table_overrides)} 条") - client = FeishuBitableClient(base_url) - tables = client.list_tables() - allowed_field_names: set[str] | None = None - if stat_date.day != 1: - allowed_field_names = { - get_gmv_field_name(stat_date), - get_sales_field_name(stat_date), - get_refund_field_name(stat_date), - } - log_step(f"增量编组目标字段: 日期={stat_date.isoformat()}, fields={sorted(allowed_field_names)}") - for project in SUPPORTED_PROJECTS: - try: - table = find_target_table(tables, project, stat_date.year, stat_date.month, table_overrides) - except ValueError as exc: - if skip_missing_tables: - log_step(f"跳过字段编组同步: {exc}") - continue - raise - ensure_field_groups_for_table( - client=client, - table=table, + context = build_daily_sync_context( + DailySyncConfig( + base_url=base_url, + mapping_file=get_mapping_file(), store_field_name=store_field_name, - allowed_field_names=allowed_field_names, + table_map_file=table_map_file, dry_run=dry_run, + skip_missing_tables=skip_missing_tables, ) - log_step(f"字段编组同步完成: 日期={stat_date.isoformat()}, 模式={day_mode}") + ) + sync_field_groups_for_day_with_context(context, stat_date) def sync_metric_table_for_month( @@ -1111,7 +1207,7 @@ def sync_metric_table_for_month( metrics_by_date: dict[date, dict[str, Decimal]], dry_run: bool, metric_config: MetricConfig, -) -> SyncResult: +) -> tuple[SyncResult, set[str]]: log_step(f"开始同步数据表: {table.name}") table_fields = client.list_fields(table.table_id) store_field_name = resolve_store_field_name(table_fields, store_field_name) @@ -1130,7 +1226,7 @@ def sync_metric_table_for_month( normalized_store_key = normalize_text(normalized_store_name) preview_store_fields.append(normalized_store_name) record_by_store[normalized_store_key] = item - log_step(f"飞书返回的记录店铺名称预览: table_id={table.table_id}, stores={preview_list(preview_store_fields)}") + log_debug(f"飞书返回的记录店铺名称预览: table_id={table.table_id}, stores={preview_list(preview_store_fields)}") feishu_store_names = set(record_by_store) sql_store_names: set[str] = set() @@ -1145,19 +1241,21 @@ def sync_metric_table_for_month( sql_only_stores = sorted(sql_name_map[name] for name in (sql_store_names - feishu_store_names)) feishu_only_stores = sorted(feishu_name_map[name] for name in (feishu_store_names - sql_store_names)) - log_step(f"SQL 聚合出的店铺名称: 表={table.name}, stores={preview_list(sorted(sql_store_names_raw))}") + log_debug(f"SQL 聚合出的店铺名称: 表={table.name}, stores={preview_list(sorted(sql_store_names_raw))}") log_step(f"待处理店铺数: 表={table.name}, 店铺数={len(all_store_names)}, 日期数={len(metrics_by_date)}") log_step( f"店铺匹配情况: 表={table.name}, SQL有但飞书无={preview_list(sql_only_stores)}, " f"飞书有但SQL无={preview_list(feishu_only_stores)}" ) + created_fields: set[str] = set() for stat_date in sorted(metrics_by_date): field_name = get_metric_field_name(stat_date, metric_config) if dry_run: print(f"[DRY-RUN] 表 {table.name} 将新增字段: {field_name}") else: - client.ensure_number_field(table.table_id, field_name) + if client.ensure_number_field(table.table_id, field_name): + created_fields.add(field_name) for store_key in sorted(all_store_names): store_name = sql_name_map.get(store_key) or feishu_name_map.get(store_key) or store_key amount = None @@ -1175,9 +1273,50 @@ def sync_metric_table_for_month( if existing: client.update_record(table.table_id, existing["record_id"], {field_name: field_value}) continue - log_step(f"跳过新增记录: 表={table.name}, 店铺={store_name}, 字段={field_name}, 原因=飞书中不存在该店铺行") + log_debug(f"跳过新增记录: 表={table.name}, 店铺={store_name}, 字段={field_name}, 原因=飞书中不存在该店铺行") log_step(f"数据表同步完成: {table.name}") - return SyncResult(sql_only_stores=sql_only_stores, feishu_only_stores=feishu_only_stores) + return SyncResult(sql_only_stores=sql_only_stores, feishu_only_stores=feishu_only_stores), created_fields + + +def run_metric_sync_with_context( + context: DailySyncContext, + metric_name: str, + start_date: date, + end_date: date, +) -> MetricSyncArtifacts: + metric_config = METRIC_CONFIGS[metric_name] + metric_rows = fetch_metric_rows(start_date, end_date, metric_config) + grouped_metrics = build_project_metrics(metric_rows, context.mapping, metric_config) + grouped_metrics = build_target_project_months(start_date, end_date, grouped_metrics) + created_fields: set[str] = set() + unmatched_summary: list[tuple[str, str, str, SyncResult]] = [] + for project in sorted(grouped_metrics): + for (year, month), metrics_by_date in sorted(grouped_metrics[project].items()): + try: + table = find_target_table(context.tables, project, year, month, context.table_overrides) + except ValueError as exc: + if not context.config.skip_missing_tables: + raise + log_step(f"跳过: {exc}") + continue + log_step(f"同步项目={project} 月份={year:04d}-{month:02d} -> 表={table.name}") + sync_result, created = sync_metric_table_for_month( + client=context.client, + table=table, + store_field_name=context.config.store_field_name, + metrics_by_date=metrics_by_date, + dry_run=context.config.dry_run, + metric_config=metric_config, + ) + created_fields.update(created) + unmatched_summary.append((project, f"{year:04d}-{month:02d}", table.name, sync_result)) + log_step("未匹配店铺汇总开始") + for project, month, table_name, result in unmatched_summary: + log_step( + f"未匹配店铺汇总: 项目={project}, 月份={month}, 表={table_name}, " + f"SQL有但飞书无={result.sql_only_stores}, 飞书有但SQL无={result.feishu_only_stores}" + ) + return MetricSyncArtifacts(created_fields=created_fields) def run_metric_sync( @@ -1191,42 +1330,17 @@ def run_metric_sync( dry_run: bool, skip_missing_tables: bool, ) -> None: - metric_config = METRIC_CONFIGS[metric_name] - mapping = load_store_mapping(mapping_file) - metric_rows = fetch_metric_rows(start_date, end_date, metric_config) - grouped_metrics = build_project_metrics(metric_rows, mapping, metric_config) - grouped_metrics = build_target_project_months(start_date, end_date, grouped_metrics) - table_overrides = load_table_name_overrides(table_map_file) - if table_overrides: - log_step(f"加载表名映射完成: {len(table_overrides)} 条") - client = FeishuBitableClient(base_url) - tables = client.list_tables() - unmatched_summary: list[tuple[str, str, str, SyncResult]] = [] - for project in sorted(grouped_metrics): - for (year, month), metrics_by_date in sorted(grouped_metrics[project].items()): - try: - table = find_target_table(tables, project, year, month, table_overrides) - except ValueError as exc: - if not skip_missing_tables: - raise - log_step(f"跳过: {exc}") - continue - log_step(f"同步项目={project} 月份={year:04d}-{month:02d} -> 表={table.name}") - sync_result = sync_metric_table_for_month( - client=client, - table=table, - store_field_name=store_field_name, - metrics_by_date=metrics_by_date, - dry_run=dry_run, - metric_config=metric_config, - ) - unmatched_summary.append((project, f"{year:04d}-{month:02d}", table.name, sync_result)) - log_step("未匹配店铺汇总开始") - for project, month, table_name, result in unmatched_summary: - log_step( - f"未匹配店铺汇总: 项目={project}, 月份={month}, 表={table_name}, " - f"SQL有但飞书无={result.sql_only_stores}, 飞书有但SQL无={result.feishu_only_stores}" + context = build_daily_sync_context( + DailySyncConfig( + base_url=base_url, + mapping_file=mapping_file, + store_field_name=store_field_name, + table_map_file=table_map_file, + dry_run=dry_run, + skip_missing_tables=skip_missing_tables, ) + ) + run_metric_sync_with_context(context, metric_name, start_date, end_date) def get_sales_field_name(stat_date: date) -> str: @@ -1247,7 +1361,7 @@ def sync_gmv_table_for_day( stat_date: date, preferred_store_field_name: str, dry_run: bool, -) -> None: +) -> bool: log_step(f"开始计算 GMV: 表={table.name}, 日期={stat_date.isoformat()}") fields = client.list_fields(table.table_id) store_field_name = resolve_store_field_name(fields, preferred_store_field_name) @@ -1255,15 +1369,16 @@ def sync_gmv_table_for_day( refund_field_name = get_refund_field_name(stat_date) gmv_field_name = get_gmv_field_name(stat_date) field_names = [str(item.get("field_name") or "") for item in fields] - log_step( + log_debug( f"GMV 字段检查: 表={table.name}, 店铺字段={store_field_name}, 销售字段={sales_field_name in field_names}, " f"退款字段={refund_field_name in field_names}, GMV字段={gmv_field_name in field_names}" ) if sales_field_name not in field_names or refund_field_name not in field_names: log_step(f"跳过 GMV 计算: 表={table.name}, 原因=缺少销售额或退款额字段") - return + return False + created_field = False if not dry_run: - client.ensure_number_field(table.table_id, gmv_field_name) + created_field = client.ensure_number_field(table.table_id, gmv_field_name) records = client.list_records(table.table_id) preview_stores = [] updated_count = 0 @@ -1292,8 +1407,36 @@ def sync_gmv_table_for_day( else: client.update_record(table.table_id, record_id, {gmv_field_name: gmv_value}) updated_count += 1 - log_step(f"飞书记录店铺预览: 表={table.name}, stores={preview_list(preview_stores)}") + log_debug(f"飞书记录店铺预览: 表={table.name}, stores={preview_list(preview_stores)}") log_step(f"GMV 计算完成: 表={table.name}, 更新记录数={updated_count}, 双空跳过={skipped_empty_count}") + return created_field + + +def run_gmv_sync_with_context( + context: DailySyncContext, + start_date: date, + end_date: date, +) -> MetricSyncArtifacts: + created_fields: set[str] = set() + for stat_date in iter_dates(start_date, end_date): + for project in SUPPORTED_PROJECTS: + try: + table = find_target_table(context.tables, project, stat_date.year, stat_date.month, context.table_overrides) + except ValueError as exc: + if not context.config.skip_missing_tables: + raise + log_step(f"跳过: {exc}") + continue + created = sync_gmv_table_for_day( + client=context.client, + table=table, + stat_date=stat_date, + preferred_store_field_name=context.config.store_field_name, + dry_run=context.config.dry_run, + ) + if created: + created_fields.add(get_gmv_field_name(stat_date)) + return MetricSyncArtifacts(created_fields=created_fields) def run_gmv_sync( @@ -1304,21 +1447,67 @@ def run_gmv_sync( dry_run: bool, skip_missing_tables: bool, ) -> None: - client = FeishuBitableClient(base_url) - tables = client.list_tables() - for stat_date in iter_dates(start_date, end_date): - for project in SUPPORTED_PROJECTS: - try: - table = find_target_table(tables, project, stat_date.year, stat_date.month) - except ValueError as exc: - if not skip_missing_tables: - raise - log_step(f"跳过: {exc}") - continue - sync_gmv_table_for_day( - client=client, - table=table, - stat_date=stat_date, - preferred_store_field_name=store_field_name, - dry_run=dry_run, - ) + context = build_daily_sync_context( + DailySyncConfig( + base_url=base_url, + mapping_file=get_mapping_file(), + store_field_name=store_field_name, + table_map_file="", + dry_run=dry_run, + skip_missing_tables=skip_missing_tables, + ) + ) + run_gmv_sync_with_context(context, start_date, end_date) + + +def run_daily_sync_for_date(context: DailySyncContext, stat_date: date) -> None: + log_step(f"开始处理日期: {stat_date.isoformat()}") + created_fields: set[str] = set() + for metric_name in ("sales", "refund"): + artifacts = run_metric_sync_with_context(context, metric_name, stat_date, stat_date) + created_fields.update(artifacts.created_fields) + gmv_artifacts = run_gmv_sync_with_context(context, stat_date, stat_date) + created_fields.update(gmv_artifacts.created_fields) + + should_sync_field_groups = stat_date.day == 1 or bool(created_fields) + if should_sync_field_groups: + log_step( + f"触发字段编组同步: 日期={stat_date.isoformat()}, " + f"原因={'每月1号完整编组' if stat_date.day == 1 else f'新增字段={sorted(created_fields)}'}" + ) + sync_field_groups_for_day_with_context( + context=context, + stat_date=stat_date, + force_full_rebuild=stat_date.day == 1, + allowed_field_names_override=None if stat_date.day == 1 else created_fields, + ) + else: + log_step(f"跳过字段编组同步: 日期={stat_date.isoformat()}, 原因=当天无新增字段") + log_step(f"日期处理完成: {stat_date.isoformat()}") + + +def send_sync_notification( + context: DailySyncContext, + start_date: date, + end_date: date, +) -> None: + if context.config.dry_run: + return + if context.config.chat_id: + send_feishu_app_card( + chat_id=context.config.chat_id, + title="GMV数据同步", + base_url=context.config.base_url, + start_date=start_date, + end_date=end_date, + image_path=context.config.report_image_path, + token=context.client.token, + ) + return + send_feishu_sync_card( + webhook_url=context.config.group_webhook, + title="GMV数据同步", + base_url=context.config.base_url, + start_date=start_date, + end_date=end_date, + ) diff --git a/run_project_bitable_pipeline.py b/run_project_bitable_pipeline.py index c85b3d1..6c8bb55 100644 --- a/run_project_bitable_pipeline.py +++ b/run_project_bitable_pipeline.py @@ -2,6 +2,8 @@ import argparse from pathlib import Path from project_bitable_common import ( + DailySyncConfig, + build_daily_sync_context, get_default_end_date, get_default_start_date, get_base_url, @@ -13,11 +15,8 @@ from project_bitable_common import ( iter_dates, load_dotenv, log_step, - run_gmv_sync, - run_metric_sync, - send_feishu_app_card, - send_feishu_sync_card, - sync_field_groups_for_day, + run_daily_sync_for_date, + send_sync_notification, validate_date, ) @@ -47,65 +46,22 @@ def main() -> None: raise ValueError("end-date 不能早于 start-date") log_step("流水线开始: 按天执行 销售 -> 退款 -> GMV") + context = build_daily_sync_context( + DailySyncConfig( + base_url=args.base_url, + mapping_file=Path(args.mapping_file), + store_field_name=args.store_field_name, + table_map_file=args.table_map_file, + dry_run=args.dry_run, + skip_missing_tables=args.skip_missing_tables, + chat_id=args.chat_id, + report_image_path=args.report_image_path, + group_webhook=args.group_webhook, + ) + ) for stat_date in iter_dates(start_date, end_date): - log_step(f"开始处理日期: {stat_date.isoformat()}") - run_metric_sync( - metric_name="sales", - start_date=stat_date, - end_date=stat_date, - base_url=args.base_url, - mapping_file=Path(args.mapping_file), - store_field_name=args.store_field_name, - table_map_file=args.table_map_file, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) - run_metric_sync( - metric_name="refund", - start_date=stat_date, - end_date=stat_date, - base_url=args.base_url, - mapping_file=Path(args.mapping_file), - store_field_name=args.store_field_name, - table_map_file=args.table_map_file, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) - run_gmv_sync( - start_date=stat_date, - end_date=stat_date, - base_url=args.base_url, - store_field_name=args.store_field_name, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) - sync_field_groups_for_day( - stat_date=stat_date, - base_url=args.base_url, - store_field_name=args.store_field_name, - table_map_file=args.table_map_file, - dry_run=args.dry_run, - skip_missing_tables=args.skip_missing_tables, - ) - log_step(f"日期处理完成: {stat_date.isoformat()}") - if not args.dry_run: - if args.chat_id: - send_feishu_app_card( - chat_id=args.chat_id, - title="GMV数据同步", - base_url=args.base_url, - start_date=start_date, - end_date=end_date, - image_path=args.report_image_path, - ) - else: - send_feishu_sync_card( - webhook_url=args.group_webhook, - title="GMV数据同步", - base_url=args.base_url, - start_date=start_date, - end_date=end_date, - ) + run_daily_sync_for_date(context, stat_date) + send_sync_notification(context, start_date, end_date) log_step("流水线完成")