From 7958556613be46a1fa49a15c49458ae20deeafa7 Mon Sep 17 00:00:00 2001 From: HotSwapp <47397945+HotSwapp@users.noreply.github.com> Date: Sun, 12 Oct 2025 19:19:25 -0500 Subject: [PATCH] Fix: Improved CSV encoding detection for legacy data with non-standard characters - Changed encoding fallback order to prioritize iso-8859-1/latin-1 over cp1252 - Increased encoding test from 1KB to 10KB to catch issues deeper in files - Added proper file handle cleanup on encoding failures - Resolves 'charmap codec can't decode byte 0x9d' error in rolodex import - Tested with rolodex file containing 52,100 rows successfully --- app/__init__.py | 2 + app/__pycache__/__init__.cpython-313.pyc | Bin 0 -> 165 bytes app/__pycache__/import_legacy.cpython-313.pyc | Bin 0 -> 76888 bytes app/__pycache__/models.cpython-313.pyc | Bin 30299 -> 33329 bytes app/import_legacy.py | 11 +- app/main.py | 143 +++++++++++++++++- app/sync_legacy_to_modern.py | 2 + app/templates/admin.html | 82 ++++++++++ delphi.db | Bin 475136 -> 487424 bytes docs/IMPORT_GUIDE.md | 7 +- docs/IMPORT_SYSTEM_SUMMARY.md | 2 + requirements.txt | 1 + scripts/smoke.sh | 0 tests/conftest.py | 10 ++ tests/test_auto_import.py | 120 +++++++++++++++ tests/test_import_detection.py | 66 ++++++++ 16 files changed, 438 insertions(+), 8 deletions(-) create mode 100644 app/__init__.py create mode 100644 app/__pycache__/__init__.cpython-313.pyc create mode 100644 app/__pycache__/import_legacy.cpython-313.pyc mode change 100644 => 100755 scripts/smoke.sh create mode 100644 tests/conftest.py create mode 100644 tests/test_auto_import.py create mode 100644 tests/test_import_detection.py diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..5a3dc60 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,2 @@ +# Make app a package for reliable imports in tests and runtime + diff --git a/app/__pycache__/__init__.cpython-313.pyc b/app/__pycache__/__init__.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b35c77a38353cdaff56ec4d7b7fa64cd9c50ce0c GIT binary patch literal 165 zcmey&%ge<81d)z!G8};PV-N=h7@>^MEI`IohI9r^M!%H|MNB~6XOPq_7yZ!U)S_bj zjQo<~^27ptm;B_?+|<01V*P@m{H)YuAR{F;rywI!HzlzoF)6V)RkzGYKe3=dKR!M) oFS8^*Uaz3?7Kcr4eoARhs$CH)&`^-2#URE}Qx*g*gU(F7%%iaJf80m1^h;c9@J z6I;nl5;EgNv>lI?nanf&m=lAOc@cEv^Gr`X8A+B%O15M`4Tp4@cKqh)fWkC-S1!SfA2e;b_RZZ-T(iYJO3ZU{J)r? zKdY3u|8GqU^L56@@QlykGafbY1`->2BgE`cc7WwsHng%$Y8!+=`lIM4H}4*(T^Bgz@DKm>oYr^v9%gWx!IRT%6BI%&nM+=Ny`gJ`JSZZg`|9M(()ow-k!9)n3Q)U zEiWPE`;wNIlJfmY%eRp714+xvNcq8}<>jROP}1@WQhqpTxto-Gla^PK@*_#ht4Mig z((-Cj-j%exhLm?FEw3fzJxR;AlJef9<#nXIFKPKUQr@4m+(XI-l9tz#@}o)1x0CY0 zq~#5yd?;yoBPkzFTHZv;k4+gp%@<#@4+f_K;}_hWzLW0h*|{)3@19%;jn7YqLlO66 zn0I#tXXegKySoDOfzyFVu*q&eab`N=o()ed%mm$YeE972L@=V?HZU{eo)P<>klMSU zo0(vw!OjN*6Vst7_aq;lb%$pr8pk7N-ILQ$<_?4=+;ic%g_*!S)|fvNq`ml#4|-?D z&je>L-~fU%ktX{;;FoCFV-T!xiQxS7Y|vvAEM39z>Dj=Hkli0T8G64i^@c78)?;%>49q}#i!T_7U=4$i*&mt@PVv+87rJ>q%>M&^?L4J|Wi&hk zS3NIeh&OHIOgI!2q&wIHLl?{#Mnu1O~W^gfm0}K;l%b8Phn=!`6jYvLem)Cbl|mfU zz{Mir=d;opZswGk7LM9zN{M~8r&*u99oFurJ6nwrqnoK=GMKnlPNg00&8IGCh7=4a*6VCD}(`XJYRpQ-J(#;&RRKk6$V2-f|*a~LSP{F`+HhaxO|AmoK zlZHt~EK3;SHXLD!(dCaZx}z(=XF#uY%qm_}xsvAT6mII%DT1d{EI*xMou}iVPbc{t zd?jN{CgUsJ#fB_nMz~qz7l5ZRX7O!N-4!E!W|_<`V=40r!0HJgfxp|LdOTFW^^r7XIg_=O7;pjfe5e9dVC`XJ^BqnG5bn5ZX+K=vNxn z68JFNM}zYV@J++c7O-J>(!Gs*5yGcG7aX5&kbCkkM1m8ZCcDS-5w|cu*|_(jjYN%+ z>8XqM=}5S7@7~?*jV&Kp$coU|BACbLT6XT@8bYT2B|6~A4Cp;|Q3Khg!9J!KEAPz(6Q>=QzV|WX*9iDhez75A%kC)JyGM&=JEa_G^ zN?l&e9}D5i?Vb<2@h*w)M%~5oIyVjk)@*nU7UmE@1SjMt8@ZU-~v#g>^Y|NNt%fFYMcjYrz*~Rf2 z+io9O$v$?;8#6OGx!*8<#e8MqYtBpFyAJ0IJ(qi~w0yb$o+I~#qnD2^7q#AKzjSoP z;k)a~yVCZDLm%WY*~JgGGF-+V4FAsXvTOU2XZiTp^TR8qlQE-}8#KJ{$i7niV&`Jr z4bSb-701XW1MRi-pAACBYJ0(a*?j5qU$)(|X1(P&6ES7#0vs$>qd(spGvHn_kxwdUN zf6t0_&n4SEd%=|pQG4aGsWNgJ9`7G{xg)!cuQhrtU1sL>-S#e)d;O5R^C0&FcXnrs z=?6Oum~Jt4?za437l-MC9MpNkZ1393zL8ztwTpdY7l-NnUQ3UOxoNWZl(085jXnA7 z&3q2hBGC2Ye}tJQpV&Bbg}4XN(ft?%UuVYf3rBu|d{`NnQF!|je{{8`H7B6Xlws|< zA;fJ)#BBt#mB6ZEgjE$krQ|mXU#-Mn74x)lw^T!gsqlZ1{6;fqN{IpaF+Ni}Ah0O| z-~$=80fK=m;2J-4m|-Y_@NgG9M*@5#=tgG1jUQiFx?p8G8{ntI3xuPY4a`Rx+~S>c z4+DaCPfxl}173cvd0J!^nuehtYwc`v?L>3!XXMlcNPVWUcDAv00%;I6ilroVLCC>t z0^Ehm(@(BJ$RI>8VQP-Y2P9Y_4d1z0U?ENhW){Q`8V|E!_)Z95IT^uf7U05g z%mO<(J0HQ8@ZI8vvB)l73@^cd76>S*Nc0U zg)e;Z@)x6770>s5Xf|h=pYOe&&14timxqNfkE`^KtL8md&DE!)u3hgrc74D>;RA>K z8QHqjhu@F`cQdkn^}xntm%x9GEnjft!T+T#|Dy=Pz#IAAI^%0~cJFTPwN?X!0tf8Z zLNM+P#b=1qxc>$O>v==reZzjpZ|%30VjO~kL-vc@H#7b;aGnu%&dszin7QXR?uYBQa zza7o0UAET#>YiiEpM7pQ!*ic8LNWaiH!h7`%3QJT`zf^Jpx~njas=Nj+RNOi@m8>} zl^Y<8d&c-*PM+~IkXX+%Ui-P$FdpMYPv=Z95TgI=^YDLd5N~SYy>pCX7x+zu z?*$)#F@F*MBIOWhZ{6LD>=*W3-uKTAESrk%Lk&hSPMjW>zkEQz?--f=NqBl+2mTg$ zT%WOtCErMVX5=?!pzq}Zw4Q5C_4OQMY5fq_jJ%OW?uqjOf81^a?iu7Eq_^{9Mjv;; zcz{SveP+b{5jJ8(v;_Y*GS53^)&1(qT!DN=+n7yWA7HdY03%Py{Nk7`WsfrNO?yn) z!bWS-IAsevtx02FM!P9wA2W~294x(KuolX-;lEzyVHFPdR4%RC@qLB~>?5lutf7fN zrG*>USNMt~{^ZgQHcnF^6(0{}i3fh|K{SWNf?6Hioqh*P+Vi;v! zr|>_8{6^hULKRhkEMf=Dz01&MzW}Paf zt;RCwGX}2KMAL-l&kkjHGkOg}e*h8&l2-VBKoIKd!g>kUWk#!MuS%@UZOw7zBpH#+ zbHlr(+?!gZf12{;>h<7riEE5Fe!OFeGx6jv;J#lYi3b6=`jiBdH*~ST#B)OMlhd|>P!*=CO?V&;Wa-pjfT&;KQeZ> zhd*-mfJ}IXa)v^I#7xbD%mBNSFcd6@$gzS{wkWXz$R9wiFdy`|2zMp0Fnhs{(%n(< z^1u}WBZtCE@=X`mso*@1|H6V9xVKfnQd_t>7<1HT;YG9jq?TGrue@ zxqAABVa3_;d{4|&&mFi|(6o3VTF`zu%T+Oa|-vyZDYaM@H5_N6An^$lrZ`(@Vw#EF_yk^LKqyH=Ye{%G0 zUdfB?(Y)G!<60d1%aWD6U68)JrTnYiAJ~|@h7T&4vOWK`m2u?Ub!6UgxL05z`KyjP z$i2{ix&O;YaeD1hN7 z{;51#lYpN61Ze0$ngq~OOj{{&`~oV29-V6@f*x49;SGS$k_7l{;&=e~R48+zDP?Y_ z+>|z2MN>))z$ae=d_Ytp{#3vxlK>x`=jF>Hz$g2Z48PU@J~sk%XiG(Sfw z2BeCh{iYQaoWn=3$9V{b`30=5MmIAsJ_+6M=b$hF+mK=z+vE{!!(YJuE@JQn3<4PZ z76t_v5N7|kF@=09|0N8b$KZD`NCsT^-^Egbu2h(X|0hV50So_6G5dQM{5}SMfWZqG zT*4qtkb-mK{{$KUE5$@&>j5c0Nd{8#(*P+NOrk|C+ZJ6f6<#k~DvEkKqNV%pxf>RJ zFFkersiohDHg-ncT_0qa32MpQ7}WB$?hgrSseTlwrScP_mV%8&EfEp57<;Rj*D@-5 zwi;h+AnB$~J7nJ2>urFNn_KO@72M4o)x9O$Etdr-w@Nsm+^S$PU1je#v9}rkwH#t^ zAL1Yrhgwj*O;F2+5N@*A<-41NS>V5QeVD~(Yd1b-A`!cc4&)V3!54|>UlTURYLJ0iG(klo83Y-~s8N}m&q0uZQ%`)L5s73<$Y2bR z0ZGRp1NC(&GN@LrL{o9dAo;kG32D^GU~Q2|w%(k4ITA7;<0r@rv^L60AC0hl)VB`JqdFi#Q)AXq2;iBb*CYM(Fh=VHZ3!eU>TP%5wW zq8uanT93e&-^6A9MZ8|BWj-^0HvFGKqyG$lk$xm*h;8?O` zTQ_FOrh+Ee| zZ?$lciCeOtXiDHeCH|*aDN=?$r3m1OrsRpHM0$yyXiC=Bl=SoucKc3vPbjzmO!v^T zI!$1JRTsd>tSdM<9hyezfMD{WCLu!v3IsQxP6QMwq~e6CVuVx_L4hbwkQE~u2tfq~ zXn2TyC141u43Ue|1`&w%HWNf#!|SC25mcc;r^8=|RuupsBEQf<5uQy^XjE)p?0l*B zdhgO;w80y#IHD*(Ca;B~#cf*2z$4UW>|8I*NYH1jA7*TrK7;>5c-C>KKxX7?HeLPD zxBO{gAQzts#&fN%$qO+o==-88YjihGJj{p3TsENm&0)9Lu_u z0xk{fuNKZz*55%`e_14=+-TTmg4Ph3dIEF}>z^gD{vlxfNji@8S6`R1{>Vfp{Nh;u zCs;dwF-EkU*C8+oRj|MN9Wz zvHqg4Lb6vVtG%~{3i^wqo_*0R`|njXFR|aXyllCV72VMnt?K{4Vj@xl`^HKQDo|>$ z+#F{(Ag@WvYX69=B^7+2b4`tE zU@d&Ge$_zQV_COqK#LEMX(KpUwNAEaX~xi&sfUzg&8RH}jGShTS9is=`HrDW-^x?5bPVchSPY`q%!_PKAJTRa!7+Y>F>`()|=M3#yCci_?L zuhjp?CUNqVQ-M9sHW&=~jmf(sz_`#T@%NX(yeawq2-Hi6ZEd6SE(b~jPD;B!!aiol zPYnJh;2J}FU^~{gflX!tm^(-e>2$u;r?N>C2Dqy@90yt z*WrZeU7;f+V}gHN)gA=9&)RMPYk2dh4$rO_oq-bFGAget>M_vq!#7+pR#OZMWtFeW zu&`2TPg4q_AnN*5YmH5oALv{&5g&j_Av_BRfw9)zQL%~R0gT|Gb5>&doZueTQ|f2~ zO(`)L7UpPV2sN}^!Uzs{!|guy`UEvH1ZT+Mb-;c57QFc+ z9VbIjUl%cg1I#QyhLQMF=Cq zfO>+s9ThzwhsT`|=|Kb=Rwp|qz_?HJgMxb+(f|c00atB*NHh?Xi)BzyoM0dbo8@7b z!x!MjC{}_3_xXh&e}Tt;Trgyqo`D?^{J+NxH#yD2wl_2?5ZJ)P#3wZ%bSj%3k{S>S zPIMIEpAPxS-UpREAr3_aB=CzX?|8sO{H_Oj5BS?q?-c+Mk&hrq-1X3$Qi||?QR%2d~(EQRoIl0XxO>%A&4DDOvE1ySKC(_2cxc`J1*af%Xh~$w&EHSMG=erE5&VB%y+Z%=Iof|(!Kyn7(?CIdEy^~}vCdv6_gb8mHT6?d!D0+d@-98hl6v6!y6_dD2ItzKup zg?-DyK}H|oJW*A^LmiuNL0KGRfj$4bL>=;OIEq*+A45-%P z(ZscwdNWOHE#}{bw}by71mN3`EIqj5rm@cQbRiX8112*3a&g-0>{h()I(7;0E6_m- z#{+a!{NF;|Eg)4(V(q_P#{A%qtDKarWJkoRR<4w-@@Q=u!?p5Lwkn{tX$-9jQ?@FiwP_5kic_{Kp|xoYtx8k2 z+Cpp77+RI3Y*kKc(->M+q-^D;wP_5kDpR(qqP1xZt*TSDs-d-M46SNY8kKIPwP_5k z>QY{78?8-aXyr-Ss-D)SF|^vAvQ-1EO=D=)n6gz9txaP;IIG$qW7+*g`KP0nOHg*{ zsA=MmX0xp9l2wFC4;p2caR|Pky~d$$CsB57;&|3=FuPZy>{1)9g0cw|mqs5sKt#_S zxvzy7%g!U2m{bO{$;Z{2WJGFZmm8L4^3ye# z-L2PyuT4^Rk?|8{SHes@?drXp1XGh8e%K%ZHo7BNhaX*ar!w+2CwgFbzSt z2CS!Bf59gh>>xg`HtxNSo<8X2Us^81`c&4v=#z+7aV``oPST)~O7CR!jEAEJ(L9pzc$AmtlPy((Mcw3W+|@U<=S{aa$|{$qsFG zC(<*u-De>^n<=W1%#*sV4XqaLjO8FXS3`CoIZs2*M{r+diXjAuUWlyXY>07nE*CBnIW=b9)gM+w+jj_O{`cSU{ z6{6q?dz~)g9^akY)Y;0g5uL3JnbFw_rkncfA@jo`Zv*7sG24d@aqnbT4|Q5@7hAUEFJ4HwOy9VmP@g^dNe+4(^fwcAGYwchJKa= zgyzE=l<-p^EwKR8fvItzIPI})f>kzJTqRdI(p_7!_1qd3S6Pz9)j70h!gL%6Euyy! zEWN13RW+q4aol}MJ}$LYwk(|Z#8%mI4=Sr{7{9b@D{dz7yS7I9jtwhYwxq+c@Pw_j zU?_hV<}GmIkzCfn@VIo^7Y}bA(C5feUDg?iz6O4CN$yNY3pIKJ>_)MjSSBO4REkeD zXId9%OUJlMWli-z@OmadUnMHeW@=whD(QnDJJm~pSGJ-@mv+UYOUFm|N8!}~*Refqus6$V?CvnW?l3~;^~@s-WZuYx%~@^C zn;rI^UEEDmbx#v_v(5sPn@t>2cCnalv-cITw;0%*D>Egd=qj z4v6a6cQKXR{|a5}L7D+loU;i?KqbW?rDx-=4aoRFwv9qGP(si^1)>2kkdo_s8qmPwD<$A<@Py;mZdv2|9oRgzxgJz|*}iW) zDz;u>(dpJKmBzRKQ>F2hg#bymf$M90Q?=Sijjys%TVMbEY1R0qnwP%DH&rWrjc=+} z`WoLME zNZINjtxaQSbtt7c^)RhXV_B~+>(Bv_NxwKJ7qVzl;b2Y`(3+}Eti_b3#7YYy!juJm zN{u+toLAQC>s&LDYwBf(V4Z~UUnCK!ZsK@Aq*|%WiKdjfovNa=QN+CoKiI@Npwa7Z zp_NpQE{>2*tUAxj*GcsHU8gE&D@7o#;hegOUf+ZV*ZD{~PNb^7E{as~STM8=P53Eu z15t1CarGn_ky@{>+#8*Zz`kC+9(;Y0UZ0Ggh*T41;yJp2=AU$T2|Bx!`6f}M+5`Jb z)8d^33s&_Hhv5WLOoYNB^Mry{vUimK8%PN@9yI1W9tbYjM&+a#&p{4?QNS@NCTBN8 zz7x-Ip5TessK3OXmN59gFt~=nH!=7N4E|3D1Zx)`LmXCPSv;b!0l!cTP-4PQ#FcE$6fs)Qx0YnTdHF$ z#P7>)RV2Fw4v1hy0iIHKy|m~0o~3=!?T1#&563J>w`%D&q}!P-)QSR6l3Oj^8_PgC zm{BCsok-7Q%Ij4}%dM6jjAbD`o2lNR^4PXo)gQ}2dal+S!-e!brm|6Wl*_8SE0&M+ z0&V|=NC#62Y6W76Zri(B(jF^DdI?iT$K#3C?_b?=AXbX>Elfp&nnBD)o4Qur-LVR!yS4O6q*rO@RgLr-ZU41M->U7W4(Z#p{dkaG&(!W*s{U@> z%XP~gN25CjS8Il1+p(a5anl?0OtkUHYDH(P5$R1#Rr6xxrE}NMEqyWC+_PHQ8*4`T z4($b6kiJvr0=uxFRol^Sq_;6OElaNN7QS4#+}fd$kSPk=~)P6xj!4 zuoOACv6dn}VkrXWp|8_YH1u^o|>4B%}>VU`~++Yxwv zL+>YHbYx};mQm@Kz<7O{Y)70AX*=Si46TOI$qG5)xDCV5^Tc)p(s7JV5xrNE?T8Z? zosF>_QIAN?=*XU*Ms*COVLOV<9*@aAsBA}I{F3cR+)TiB1X)ToqvIcyS)Cq)jH79> zIeZo%EWt4v1OyM>{3B<;XxKg~8SnN_fXWC_o8aJ86?kG{ksP^F-H`}P`6Rj{S}FtA zRCXjN#a$O%%}m2dkIIgOz*R3|Rx)2@D?K7;3R=ZjgsxPBB+ZQPg0`JtBpA5|i9~yn z?de(oCq0$8Kp(r_lO6XDM;kh$6xipEH#RVx3BnZK!*^h#T*?^ZT~YUvT#mDHla zY1?{H#N(?;icq_fLugl0-NPDRZ#4ECVqS0dZiCDlgi;EX>D zQuc62+0SD7kiD;xy=4Qtk}d4*EgWR@VZ;-IQ@ZRwF*wD=EKx%G#I6MI3M~Fl3{DeF zXZ1jXVpl>yLz;Fa2pZz-O2}GC>@41eTfc@>D;^spvnvr-(6ewz5F8Q^oDX_ZVFPp! zx-M++kT#~ud=*%b!mdP`6W<5-?i`|mpXi{1`gNlMVppP-XeEIFpHl$96T^~+H8V|i zaG=((M3w>kS}orA!joSTV+t!S%Q?XTf|MrtfxW+j$da^Dvm#` z-6H6pclg)|MZ1Zs_i%VV@(@hF4Y%_8yWrR!K}aGNJG{5IU}>z^QnBpHIjHP&_Y zd3{hdPGJSQD)o7Nq*z|rQ|G$~0)t;(cMlXvilQo7e)tiw`x~>3{vt|6?)BpD4B;{Xog@^BNxys#;VsI47P_4+eQ?hJ339^tSrBn($mfG=< z#)VKyeMA=RoyI9!Xv|Mjw#cD<)A*FU$|sq-##l9F3m5I1#wlB9oEfE@M?USF#wlB9 z3}JnR?G9L-&3Z0Ol+_=wn~b;=^m9sH@;Xb`V?cDWeurSU3Z;am;&4EbY?`KX z%|z@5ItTbqgb=*o>7K8!iQ@tMwkF0owUy3ybQ?`6F_@;6YfRHn>P>#2IdOvkoB@rE zjhLqC;UqgTP0JXw<0%WfIQmw&nF@5u^9;jGG6&cSX3|i}oPyrOpJeogx}a1&7oV!8 ztrWe9Mi*23rQC=*xl9Jt*C+qgA-Fh{vlAIrGGv+3ubS83-n2jWxnX@ zaQBDK24v z1uV^*o}CNx^M1G-%sxQ@MWL!xC?nCrwGF2Pb~-fAKLy>3NECewsO+e|h}kb;@H_^; zgTWCDx-jU*;4lVW41N~_u*76MpFk%iWJq@ly!PYuU48e$To` z-jqFNV2Y~lGsc{<(1uM44`?un_Y0_{7Sa_PIlc&cHLsl;Hv)gX^+wV1wj+14 zJEPg1AIDg%{V|xCIr8r_EVu7rrEJ^3vuo+|ue9N2?ha)$_r8xJXW{?nlGk|D1*SV* z<3OSDbth(C&(7$tH@;rlT?wRH`@Ic7d#k>7Adh=H%K)Uevss|L?J^)Gk3~wM{b)A# zc4hU^4DN>}3zYsag9FMBNzEU+?1Q`6AC`HYgFDzC?cgAzubBJa$ceThrPJRgqLKW` zPPBEfg2DxCs$5QST%C?w?z>=#^SvavPDbqTLQ1~`oGGJ6S`O!xNlAjE(jLpY<#M?i zTqh%E34Ng}LD5;Y&a?742(ELgWLUCPRfE1VrL(uk0DX~k9QsmU7v*xe&oTSl0A7&z8ndCk?|Asl`s>JOMPxuSNF&S=ifmh(e`G8PS~#SNOsvx(Ry#R;>Z=p-xpR(0=LS>dnI*}6FoGa zj+V4PBFEgn?gwh~{|er^xD^-mPJmsR`~Mmq@a9_7g0u~A{{n#Kx%2QnN`8VSY#qUChx_(2 z{6CO%98yqU7miOI+1N-`ofQmCSLU+m$%0`8AXPhbKa!Ea;m2?Zw^4`Q;6s2W=KdJ*dW0 zta-csawuBSq0!-KB^8N+38#e!CbBnHF!AdSJ0d8u1Mx!d4iGPN?(1nazG;R7Gj}jI zTkXA#+|7g4z1z53l@_4f+QtFpRwIk)9rpfQ_ErZRnCWESa&nM~!!G#t5$tjr>|Zt+ zyBPaI4~JcnXrMHR1=ehNpN@`;nO&bM7c(+)OEPBtJcD4^D<;?YUyd>Dwt`ZZa2wxk*ZyiD$Q3IyW-t9qu3QIVNwE8<`1& z`a_f9wBWX2!RmfE@53^O2{`n_MiQaGESS)d!eNpmR6NCAqMpE{JqWh!&=V;x;h@_I|LF^Yy-P|Txgc03_`u{m=vysaQo?o`oRUm}3ypEXc0@{c3g*vG@L?dC zk9Un8^LGj?94K1~jRx@skvcM;p)aT;j0Isa@Lk#L!$No-I0cm^c5Oi=ofH2x zX!U!@XdD4pn8;}C;nFl0Ov-3f>2DIG4lXJ`WL%Pi3y9yB^|REI1qW1J4;M(cGH&Ye zVKCa@U9C70vmxEifWB4Ko+1~sx@CVX1L+RUX&X)?XKKhd;}xfesx8v(cdgrkna& zA@g>Xw*hiL%C-;LxF3~N51F_h9k2l99TNwXcWf-C9rocS_MHN+bJ)Ya>){}y&%*sD z>!!1CY+vi6V&PK4RE^jtf{C;Gy2wuBCbU#4tmNZ5m}Eq1c6x0U)*-zfe1|1=nv9>Y(+M;2bS_lF`z)!j zkYQAIpdAM`@?o~0SxyG|2)LXP?SL(3r5xd+J`YThVDz_KG(HRGU;1}Kyi+jE1^D@k z+_NJP38uiz{2XMwGxH+?I~SY>mUScuM_=;62y(CB?a?+Khy?v28{UaY!iI~~aX6e( zun}q}CM}WRIQqX4EIuibL0X3vW={usA%oN$7BkMzElfPN2Ym5N*@aeS#v>t96Jr11An5xwr7n<1g}J}o(+cl|Ur07~li zkR;aSaK=G_!#d2C7B!lH_+xyWbpq{hrvI*eixS8^*MYUua;=MPA<~Pqb1Fu9iPjxvDblwv)zk~5Kib&6 zTGbOPLwY$=Be}6$ZtICQ^{!U;#VWACt!3#ekzS?a2C^CpYP1eFYmvTHORq!vHmyX$ zgYlQEMfd~?Lk$ECEA~eC14SQpHH*r6PV-t zl#HEb7bmpZT(Lwdc5n-2_Gt_wEK51BC&!tHuZ8$-=!+#%2{S&G$C*5K?0d^YvF}E* z2icf(5Ayoi_q4~eZuY$+E%tq9QucipVc%QzSX~YKzFT77r{GIS((7j5laH${$%xWt z-}mVC;M*&)?_~UheNUK)XJ4E)ynEQ!e{5Lc;75WXWXKz( zn8)CI@Co5z`xL{!iNQS#{tkm*LLhLbr}_Ca0(XQ&CO$A8#680hvXS6j_AaEzrq%Wmy$2-TpOI;j~WL95cEf-WZOV$E`GxBs{Nd zKr0Q9|2b67gn!M*S9RFl{p;3i*L?!67nI!v64whc;d&v~<$95>cYp)Z?OJ{*1L+Pe z-HCJ+-#d9Nyjt8A%R+KCQ(CJ!`FXXZEtZ4yT!l@Rs=JV!r?A6g*PdN1YK`S1xqvCD zQFchL7PrO;|N7(H?zJhV*jnnF|$2cQYk*lDi?;FO3|2C6cR{a;o*l zebuXFhho)8uhDj2i}bC`7RtfmHtW^WeX%;EZ_^$z;X!)6w*T!w2adF9V>!}Z!jXa% z{W=dmc#QSr5=T0`m2jj(uw&Y3gv{$%{f&_MA?%xm+&lI5;cD)k*6QJM?%hHQP~I)) zfbwoNi|MWQ5i9%d4zF{BWtUkFGI1PfHhg-Bt@9c94s5cmv!M^{0Uu_cRuYArj;-^f zRg{mR-@{Zqjl36%@-!)1$a`<(7OBwt+WWcm)T^>`M7~CuAr zyg2!?+ij1TiZ17&Funn8U17HrI6QeoyUfxa&$=;waatHZHz~$<5sa_iktM%{8jPP$ zFurjJzDqumUN^>1KCS{igbpK0ALAG5_24U#Fg_VS!T1R?@suop>XxeLD5P&5@{W%7 zdwaVT(^nCw5e*aoT`2`16UG?zcSw)d`A}ly*$!CHAIH1nq3-V{D$Af;c zBF$k%nkE%5+MwO^JSd-PJvN1<5FE3Tj%<=#i00;@JxyN_FBFk3Bvzz3tVq*h#SCai zE;KhjzlI8{T=JL$C-@;(b!N#J>W;DJ$ylznGy|MK@t5%tvZ1E!FCm=ClCZ6Jj7U-9{B9LTu&0XEz z6McyYu`3vuKl8BgkQ-c>oof+Dv{SJ3^5JvyXIf}tr@+DKzT;4i(N2$5h4;7_u_Z>( za*DPT6VMV43dWWg?OX%Cl_ub_37kd(b}Q4ZBLlZoyNvMBtlN>nno?Ej$C6?;G70{_ z17Q0y0=7mSz}CziSubEK`2>KiRsw8^PLDKRknVxVull9B>vcj&c0_oxPkEO$V0=kouV5?glAp_o*hx(SzjB0XT2Q=&mi-9 zR_6i8+;n&wAa|?7-nWapZL02T;%?VjfO5NuL&`1|({1*FBK9o?;F*hk+r>d94xY&# z4$gql+vZCB`ab;BfU~F6n`1tq%uinGlE^MK;LMlNZrUKpmv}nL`nAap8Pw#{g)TX$ z8G`3K^dJeK*~GD|Tk4gk0W@j@aSCXh1klJ}kKP{uCYc1#Wa&wWG*Yi@3DAJ!A0J7_ z0UGsnDWFMZn4WxGYHv?!h@}QJYa6EL>dndLk^l`EKLIoeGx6l>O1*rYecfGSgBq!q zZ~RPfVqxZ?rCwlU9f7k=z+78OdFMSQ6?g%2Y%B-UZYdRb7ztBE8dU;HNt7_j)nuTO z%4)h2fy!qeIa*1SFwueXX22;YMj!w8wzGmjKQ z=8ZaUJ>+h>?7a@|W?6Nwg}cRAfO5;i0p*s1#dMavuZ6t@vY@^O_I3jYnK-cV>v^zR zs$FLfyAyW)=9rv}#phH=X;`X)4%ayW>KRH+pK7DM&i0kUQWg$-vR)f2!D+~GL z2dnLYE)!)TzU~q4sP}{xIs{=G8klxHtVATsCpJV?5*ma_2d(jFHz#C8ltS!$4Y@|5 zT1sk}N-ZTe6lw8~1^O5a^#su5}qIfHAtQeJB#x3HudVG+G|G=>otryOAk ztxaQSRhqKZMs8tAH7_?E8I56tp9aqwsph4>g(X!h{Vgo1S~bvnM`O5$jVYgU6Rk~S zX!U9EtdZ(&=zG>k)k?>+hOa&4J?xd2g_oZyLpVp=^v^tQo)j?XD#(smed1(DS z(njfw56R+H9llKCN|POeH4uD%zz%m^!@@#j0h0uf_H!8tZdoYdLm4e_+^Q>STIVsp?;b=lQO%Ysm*oNdJ zH+R+_m=(B@lYZ}zU?w^6;X$$|z|rVgKiPJ48eEE=^pl-OM+7_83WXN1D?8Sd(%dFt6rAGi7!nOFX{ls1NlSgAbtUeU;(4;e3U*3C z3inruw7~2KZ6>xF-q=UW#3t9}H9)AL8&hfJBY2MS`c z2MS``14aE(qFR1+%Ym2$>97Ziy7)XAZ8*GI?v2@y4#zXn>q;AdGLY=hZuD^?JyW~U zCkyG>+RNr3Jy*NY$Axs*1hrK8)D&Zlt$qZ`&TE@72=VfezcX_HT?X7#=|-1gx^cXkm~M>Jq3H%>Ue6k=hs?X}-Ui4m*V&J|x#gDX#%Q$aqEsi{C#H9{2nC3-GZNjNu56*0tb2IygQ( z8<^qY`^E5xlLRw3)0z%V@$gw=cnb#LYJ?FoBcGoM%z*N8_Cgalhvrdz5l0N3ALmh1 zgBx-Ah4J~B@RVSl1;1M}5gy$n^SG&$$D?a`WD9uQIL0H0=5sMX=*s6|kdHwD1_(v@ zA`B36@`$l`JUfX;g*H$8Ka-uW#JJv#v`P$!P?IbQfYlftVHm#^0>M5NgyVw4Q{dl9 zu=bl{&F@B@y<>3cNjZGTTsM2W2y>c$bK(x4vL`$tV*c-kZ)|U@H zKx*ta4b9?(Ai@hlq`nYjV-toZu?j?31tN77$i_~mQ)Rc<3q;rpMCx833->OK3&EiA zzTk|Y7l_oo;Ec`~4vCY72qz7ZdeV@Mbs6E-U3p5pyYNgPx0JUueq-0tnU~8|cNcFR zlwmJeMRn^S3lE^rAa;ZCBJ_i?x+BcTP8wUp@j!$qkAeXm--cksWEJo!5PPc6aazyJUM literal 0 HcmV?d00001 diff --git a/app/__pycache__/models.cpython-313.pyc b/app/__pycache__/models.cpython-313.pyc index 1809b45a777e4a45e4e688b7c1912b27578de781..50e05c22ab4f8e4ac885e1c3495f1b47fa93fb0d 100644 GIT binary patch delta 2076 zcma)-VN6q36vy9drMwo}QV~|`&^INPXB6##!E8%(0#aIA>_avamZqZ*e6#ei`zisl zNSDmT#4ju_CirRflb9?Sgk)oW*~}8>T;d+lOzHMzOSVrg(*WOh&V95rxGZbnzI)F* z_ndow|I^c7p0SG`n;dI)yN!X*vG!*dCWbzBxY;|4-ZSPIH?yA+dp(R88menEqX)x~ zc53YdYq-qXL9G$6o+-1wNv%<^#>%Y6skI-hXUnYq(YPQ^PuRq>k(24+3)0xs(iQU+ zMLXrMXOC$o+PyQP#lsM*SV?T7oM0_*J#tzam>QSl1rx(eAL$82x(Ckpck@IVk%*KT zmiTP)y|l!SD1;vY-=o7x6>P9WZ7y(V!TA9B5w11N(%TsWUKRh>_QP3yV`7cH=EJ)P)CJLtQ0wIpIkTN07b2$tqsE9 zro-#OnyunQ`DmwC0n8ibCq=9N$*2w z@&gP8+(NB9nLtb#JGjtUBBPLt+KavKw$)OSn9v}ZQG(uStM@RUN@hz)P2?g(D_AjU z0$ykz?Q@?-y11`I_a4IG!4c^;<_B&K-5knC7lh@mZ=zpEb$|GNi|&l+T&$e)PT<_J z=K&tux|IX(MeML9-)Tvq!E+DaF<>U>d~Hut7eAmFKOeT+BiX*8mf&YuB4^O zLHPkyCLW7)zmpi`Cz3?QLRa~5iSR^8D=BG`moiy0)m*5AwKSQSAm8I2Gd@pHKn4Rfi_MsJMH zozE)^sjrkpMQ`i*7An3^=lY0G^NqJ^fA>{*Hse81)PZy&@ht@|Jj68)czHFtr9cCG z1+30l%MDvTuq>>&yY%?riuLW>TdS^D<}CAFD=uHIZ`D~ho4WR4u9vEERhi2^;u_0D z32%z>kkwB5-E2QJoWt7v&OPtbrc(_~$G+Kg29qRm@zS_lpc?cHo{UGr=lW@bDJj9H z6$N~ZuMGAYkSGm?|Lir|=a$!mVt=U*^lT;q5w@|v&2D0EaTT3P;D3ixtrZ)LcYeng z&f8Ai+p%ocodKN-0PfyMYi$r&(rT~x;F3h905-On(~r@P171!&vc(wNOgYW!owKdi zBDwAoR%|>$qplwt2u~ow8EYFv!w<1hl2t&m+8`M(lKc}%dgDcsP>-~Lff+69?J&vz zfe9J0acfH!k9Uh{G3Col$rD=04X&a~iRfQ&IsdxOtt&i_WO3>I#S3V|iN8e)UA%hOIni=C=LgX1rP0k;`u1uihAq%NhUPVxtA zhd)O3n3krqPuRY7#>QIKJ(aBgYIdE0$NDZi+t1b8IIsIO<7((cJ_$yjjq1M4Ea@Jao|~UBPs(TZ(=^}QRp-e# i`Axkf_fbX$1|VW6zQwp%py4_r*H;EEMz$hDpb!8+4j4fI diff --git a/app/import_legacy.py b/app/import_legacy.py index 4292e92..66c0d3f 100644 --- a/app/import_legacy.py +++ b/app/import_legacy.py @@ -36,18 +36,25 @@ def open_text_with_fallbacks(file_path: str): Returns a tuple of (file_object, encoding_used). """ # First try strict mode with common encodings - encodings = ["utf-8", "utf-8-sig", "cp1252", "windows-1252", "cp1250", "iso-8859-1", "latin-1"] + # Try latin-1/iso-8859-1 earlier as they are more forgiving and commonly used in legacy data + encodings = ["utf-8", "utf-8-sig", "iso-8859-1", "latin-1", "cp1252", "windows-1252", "cp1250"] last_error = None for enc in encodings: try: f = open(file_path, 'r', encoding=enc, errors='strict', newline='') - _ = f.read(1024) + # Read more than 1KB to catch encoding issues deeper in the file + # Many legacy CSVs have issues beyond the first few rows + _ = f.read(10240) # Read 10KB to test f.seek(0) logger.info("csv_open_encoding_selected", file=file_path, encoding=enc) return f, enc except Exception as e: last_error = e logger.warning("encoding_fallback_failed", file=file_path, encoding=enc, error=str(e)) + try: + f.close() + except: + pass continue # If strict mode fails, try with error replacement for robustness diff --git a/app/main.py b/app/main.py index d292094..0777f7a 100644 --- a/app/main.py +++ b/app/main.py @@ -67,23 +67,29 @@ def open_text_with_fallbacks(file_path: str): """ Open a text file trying multiple encodings commonly seen in legacy CSVs. - Attempts in order: utf-8, utf-8-sig, cp1252, windows-1252, cp1250, iso-8859-1, latin-1. + Attempts in order: utf-8, utf-8-sig, iso-8859-1, latin-1, cp1252, windows-1252, cp1250. + Prioritizes latin-1/iso-8859-1 as they handle legacy data better than cp1252. Returns a tuple of (file_object, encoding_used). Caller is responsible to close file. """ - encodings = ["utf-8", "utf-8-sig", "cp1252", "windows-1252", "cp1250", "iso-8859-1", "latin-1"] + encodings = ["utf-8", "utf-8-sig", "iso-8859-1", "latin-1", "cp1252", "windows-1252", "cp1250"] last_error = None for enc in encodings: try: f = open(file_path, 'r', encoding=enc, errors='strict', newline='') - # Try reading a tiny chunk to force decoding errors early - _ = f.read(1024) + # Read more than 1KB to catch encoding issues deeper in the file + # Many legacy CSVs have issues beyond the first few rows + _ = f.read(10240) # Read 10KB to test f.seek(0) logger.info("csv_open_encoding_selected", file=file_path, encoding=enc) return f, enc except Exception as e: last_error = e logger.warning("encoding_fallback_failed", file=file_path, encoding=enc, error=str(e)) + try: + f.close() + except: + pass continue error_msg = f"Unable to open file '{file_path}' with any of the supported encodings: {', '.join(encodings)}" @@ -250,6 +256,19 @@ VALID_IMPORT_TYPES: List[str] = [ ] +# Centralized import order for auto-import after upload +# Reference tables first, then core tables, then specialized tables +IMPORT_ORDER: List[str] = [ + # Reference tables + 'trnstype', 'trnslkup', 'footers', 'filestat', 'employee', 'gruplkup', 'filetype', 'fvarlkup', 'rvarlkup', + # Core tables + 'rolodex', 'phone', 'rolex_v', 'files', 'files_r', 'files_v', 'filenots', 'ledger', 'deposits', 'payments', + # Specialized tables + 'planinfo', 'qdros', 'pensions', 'pension_marriage', 'pension_death', 'pension_schedule', 'pension_separate', 'pension_results', +] +ORDER_INDEX: Dict[str, int] = {t: i for i, t in enumerate(IMPORT_ORDER)} + + def get_import_type_from_filename(filename: str) -> str: """ Determine import type based on filename pattern for legacy CSV files. @@ -1066,6 +1085,105 @@ def process_csv_import(db: Session, import_type: str, file_path: str) -> Dict[st return import_func(db, file_path) +# --------------------------------- +# Auto-import helper after upload +# --------------------------------- +def run_auto_import_for_upload(db: Session, uploaded_items: List[Dict[str, Any]]) -> Dict[str, Any]: + """ + Run auto-import for the files just uploaded, following IMPORT_ORDER. + + Stops after the first file that reports any row errors. Unknown types are + skipped. Logs each file via ImportLog. + """ + # Filter out unknowns; keep metadata + known_items: List[Dict[str, Any]] = [ + item for item in uploaded_items if item.get("import_type") in ORDER_INDEX + ] + + # Sort by import order, then by filename for stability + known_items.sort(key=lambda x: (ORDER_INDEX.get(x.get("import_type"), 1_000_000), x.get("filename", ""))) + + files_summary: List[Dict[str, Any]] = [] + stopped = False + stopped_on: Optional[str] = None + + for item in known_items: + import_type = item["import_type"] + file_path = item["file_path"] + stored_filename = item["stored_filename"] + + # Create import log + import_log = ImportLog( + import_type=import_type, + file_name=stored_filename, + file_path=file_path, + status="running", + ) + db.add(import_log) + db.commit() + + try: + result = process_csv_import(db, import_type, file_path) + + import_log.status = "completed" if not result.get("errors") else "failed" + import_log.total_rows = result.get("total_rows", 0) + import_log.success_count = result.get("success", 0) + import_log.error_count = len(result.get("errors", [])) + import_log.error_details = json.dumps(result.get("errors", [])) + import_log.completed_at = datetime.now() + db.commit() + + files_summary.append({ + "filename": item.get("filename"), + "stored_filename": stored_filename, + "import_type": import_type, + "status": "success" if result.get("success", 0) > 0 and not result.get("errors") else "error", + "total_rows": result.get("total_rows", 0), + "success_count": result.get("success", 0), + "error_count": len(result.get("errors", [])), + "errors": (result.get("errors", [])[:10] if result.get("errors") else []), + }) + + if result.get("errors"): + stopped = True + stopped_on = stored_filename + break + + except Exception as e: + import_log.status = "failed" + import_log.error_details = json.dumps([str(e)]) + import_log.completed_at = datetime.now() + db.commit() + + files_summary.append({ + "filename": item.get("filename"), + "stored_filename": stored_filename, + "import_type": import_type, + "status": "error", + "total_rows": 0, + "success_count": 0, + "error_count": 1, + "errors": [str(e)][:10], + }) + + stopped = True + stopped_on = stored_filename + break + + # Build skipped notes for unknowns + skipped_unknowns = [ + {"filename": item.get("filename"), "stored_filename": item.get("stored_filename")} + for item in uploaded_items + if item.get("import_type") not in ORDER_INDEX + ] + + return { + "files": files_summary, + "stopped": stopped, + "stopped_on": stopped_on, + "skipped_unknowns": skipped_unknowns, + } + # ------------------------------ # Ledger CRUD and helpers # ------------------------------ @@ -1635,6 +1753,7 @@ async def dashboard( async def admin_upload_files( request: Request, files: List[UploadFile] = File(...), + auto_import: bool = Form(True), db: Session = Depends(get_db) ): """ @@ -1700,13 +1819,29 @@ async def admin_upload_files( uploaded_count=len(results), error_count=len(errors), username=user.username, + auto_import=auto_import, ) + auto_import_results: Dict[str, Any] | None = None + if auto_import and results: + try: + auto_import_results = run_auto_import_for_upload(db, results) + logger.info( + "admin_upload_auto_import", + processed_files=len(auto_import_results.get("files", [])), + stopped=auto_import_results.get("stopped", False), + stopped_on=auto_import_results.get("stopped_on"), + username=user.username, + ) + except Exception as e: + logger.error("admin_upload_auto_import_failed", error=str(e), username=user.username) + return templates.TemplateResponse("admin.html", { "request": request, "user": user, "upload_results": results, "upload_errors": errors, + "auto_import_results": auto_import_results, "show_upload_results": True }) diff --git a/app/sync_legacy_to_modern.py b/app/sync_legacy_to_modern.py index 8078a4a..4675b11 100644 --- a/app/sync_legacy_to_modern.py +++ b/app/sync_legacy_to_modern.py @@ -526,3 +526,5 @@ def sync_all(db: Session, clear_existing: bool = False) -> Dict[str, Any]: return results + + diff --git a/app/templates/admin.html b/app/templates/admin.html index 2e7dfbb..6316c9a 100644 --- a/app/templates/admin.html +++ b/app/templates/admin.html @@ -53,6 +53,14 @@ TRNSTYPE, TRNSLKUP, FOOTERS, FILESTAT, EMPLOYEE, GRUPLKUP, FILETYPE, and all related tables (*.csv) +
+ + +
@@ -117,6 +125,80 @@ {% endif %} + + {% if auto_import_results %} +
+
+
+ Auto Import Results +
+
+
+ {% if auto_import_results.stopped %} +
+ + Stopped after {{ auto_import_results.files|length }} file(s) due to errors in {{ auto_import_results.stopped_on }}. +
+ {% endif %} + +
+ + + + + + + + + + + + + + {% for item in auto_import_results.files %} + + + + + + + + + + {% endfor %} + +
FilenameTypeStatusTotalSuccessErrorsError Details
{{ item.filename }}{{ item.import_type }} + {% if item.status == 'success' %} + Completed + {% else %} + Failed + {% endif %} + {{ item.total_rows }}{{ item.success_count }}{{ item.error_count }} + {% if item.errors %} +
+ View Errors ({{ item.errors|length }}) +
    + {% for err in item.errors %} +
  • {{ err }}
    • + {% endfor %} +
+
+ {% else %} + None + {% endif %} +
+
+ + {% if auto_import_results.skipped_unknowns and auto_import_results.skipped_unknowns|length > 0 %} +
+ + {{ auto_import_results.skipped_unknowns|length }} unknown file(s) were skipped. Map them in the Data Import section. +
+ {% endif %} +
+
+ {% endif %} + {% if upload_errors %}
diff --git a/delphi.db b/delphi.db index 35ae41b36a3a2c1374d23ff29fd9abe7128ce4b2..d23b9a129c0e5d6f92b13c88037adce5c5bee0b9 100644 GIT binary patch delta 10160 zcmdTqTWlLwc0+Nf7a!NMEXy)Yb8JSDXp0huFNty-nj)!(Ej~3rsyM!7U=^CP$Wf= zkL@{kW=M)s9Dnqq1%%7F_nvd_dEax-)yoBwEg)@r!JFwD+#Tu684)}wDcs(aaP z>LH)O7_&pZAfNqk=)y^Uql>7q8k_1`);SkqD&KUGM zo!yS@$&=XHY}lSSfvwew?eXK-j*Y!3g>=t9YcUWbL;iw%2NnttBP{Z{MRpntg!rBQ zNJ`kPh}A}&6RJfnB8kOvqn@obge|QzKgj~&To9Gz7e!HBTW&+(;QUMvV4in*mX~WyX=`^s8jjbeIJ#@OdTrjP zL!#n*HY+vj4K7?L)SA_XLR789?HPsHM)^j$u{)2lHINS(^2R{V7vvfC6W#aNPkIhT zxq*)v@-O7ay>I$$)32JIFo~wLDP$Tj{>1p4 zF=HGZ{9y3a!LJXl4%!Aj2DyH3K%U|ZV=KeVdXmeemoh?yTdzsifnY9KV!sr^x{HP%>Qt4!J5ycE+Lq|G>KzcYd3`4R822#PX!yRD2DGm*F3<3Qx6g*_G=7vcsmWZw- zmclzKH^TO|RJ#Fzpd{5QTxv_Ga^Yg7TrJldlF+D0fa#48gY9uZ4QH}kbS;*Z(V7Is z89Dge7Qi`+O>vro1w;YNZ(G>q%9RSl8DM*B_Y|z+nYMsk4>74QrwSA>3!7qrI!=3I zb>BqtL9<~Pc1@(B*{=QMd)v@>1hTn3Y60l3jTyrbZ7WxLg%5ScqH+Rk0~@jdDfg z@*=1sk(@Ictd9(n<>V>~N{yVHBtG6OL*PWs$&#qBxdUzw3au5J1=OLBA2ir64U@&> z>S}VGTT5YB;Hn~s0rU)rd66sDsv=jba;5SX#s+?7z+jCJk!Wl+yb+6PZiRG7Vtu<- z1z}Rn6t>D>J&7bg+i#G8J|=QGqeY*HCdJFqG=wBDwPyVQU!3bh3!sRaE8YbY#MFvl z^SuTuxDsxOjwmZZ=`P57!iT|Ng%E(qEjPRiYUjSQEUJQ{sy0IrWfbRE@TxbA;iiin zdrG1}1IJ=bs-QO3K(

?%eaf9)s0COgsx&M9oTN@C}#;jWjk3HSmmU8zSiKn9eXf zj$n~&jav}8$X2Z`7Ez;k{BV{h8S**#Px8;?r*J6$A$gtr4*4zeO>&de$#poQGh~Sb z$ys6}gy<}vSw6CSV0qW_CzdxXzi0Wb<<~9WusjJzO~4`#>c@1-Im4-+pgX(z3CcLB zegdsF^%F#O!lbw9w191GbpZ8vs{_bmtq!0cYjt3OQCtfmm<%E8s9A52g;dT!eJlyWVibF=dRi{RuL%5X|P6%$LLIh3JIsnE@Q{$EFXPUTh55u-ND+P!Fpo1N_r&{nKvtX}95y zyO-teBf4{p<Z^mPTivx}RS9%?W40~JuTF*c9m|^-Gz~Zn@zLm5R##X!4 zH>T^?^$&T&-qH<0+PdD{)_!vWKj-!F0l&*Dxcx3)KHzc9`SW>Ku}}yW{IjKEFc6q7 z)Nd4phTtk!w!!~q?iE>XVV4Amv&D4!$|D}P$M53ZF5bg=yz?ISJU@HD&~#un;P&|s zn7Nfa-pvv$jpsUsa{SHty@i6`>vg;2f-mTE`HF>rYi_P2z@EOm&(Djqe2L%3(?0C4 zKvr1dxNesC;JiPm5H4$M(y?g|KkJ_L?&XQj+)9ph@zjv2^~UbDsQu==e10w;6umCL zF9>RK3k6qx)-Ae9p8Qz%sHT0EXiWOA8U`k`1fj2PO`;>v0)df||M97bYtZZVmr zXd!ik4rF5wXCFk80E7-W&X4G4bhz3|B$itbFU4|cpzHyP6ayhaCz?wp&S7-V#2_C` zCg$-wl}pD`IbX2-?VW3Xd*s=6hY#S6soc_9I33OT{GC0oue0azcJ_GBF}+`xO0Fcb zncVs^u$PO6)9IC1l&4^;;Y2PGj>p`ny%h8uGsF6`a8#xvF*%XVh1X%fSaceBNrg9n z;A{>B;G)>-fMZIV2D#Gdv)VKcIL@Gm5qmkgmZtMg)LLuAF^OuyBUzCiDso~(FV84r znq+bK{v&#qGP;~a^=4%}B20BGcNXE%~L)#{=S9WsY> zUtEhW#bY1}v^G1@J9I0VTq+$~N`w>HlZdgBxVV<48nz*Ii)&fvwmCU5s`qMxovc`o z>UnLvgPG$f3%Z6J)A$)f15;tBY7$wX@E9g6v6aLq5)p;8Kb+3wl8Z;tYNDWFU~mKx zqmkT0;nlS;+8yQM2+9jaQQuVi!xU_x3rs#t;TGC(L#Wn>W`u`2dtTDn^H@52yqWfv z!Fn<)(U=K6GO|o}=8Z@#bfU=xo@P|=4ZQIK{rda}PB{8e=8r?x)}B& z$!M$>Nnc2Z69AfAq&^K^Xh3sbh+WQQSnbQJ*St{A}>|2Wx|| z!Qp|Q4ZJ?^%)pg_WBu>-ztF$dKi2oJec$iiL_V=X)OM@#%i9`~b+784(dBgy>Ux-W ze{G06EAt6v?iEczJ6_XkHh^@r$g@r2s+bk6<0`Ah59&);`X(4h3IhQ>n6vVrXocwae`}uU9Iz z=axFrJRXn>EzSX&cagPkv{vMKC>3Z^+$3D2;Z6ozY?imDxKa~pNL8UCIwxZ6sSqG( zM@$rD2Asb;Pv{NlBJi+Y(4hN(;9L|tG z%sMlScBU!E9A%>3tW2vJ(3aryU(>2?ui{Nn7nmQ#Km}jkPR`tkFi4s%*unJcfp#TJB7_cc! z-9_3q8)7{)8QZcm47bM1NR3Ut0bJ=>n9?yz@y zHo#g%b#>|zd#AtDF=VI1&)S24sb(Q9%5dkUrXWGZtr~LpI)V0**9P)-0_`QT1$5rS z+AFQ?cp2TdMQaP{OX+nk@~pqi9F+W|YGwK$Q7AMkO^5=Z`}{7*mZ}_f+1Ug$=PH6E zmBnIc+RfUNOiI>{%SA0zpa!vmCy5Ox&nXpFC}M_2X4uhjhNe@rwpeXe@}d-4pJvaN zgX#gUrB=XD0gVEdDvBjA5FkaY!dC;$u`_7P5Vb>oG{tdla0E=OT5gtn5oM_b$C$pOpg#YDcwFo6eXUILp+A}R&$Co3@7+$D9xzDoWaf+fP zd~qn+R+<$K!b@WlURR(7OU0qk=?qPup&|qVX-4>^;MxvzPIlr7-Jz8;F3ZZp4RzN& z(S?SHHb0Z>smpCtN2izXMbkE$pJ1Jj07hs2*&ULh>MMg?2-id<7*G2AQ(mvf8M<&k zJ5x}QwE%O_Q9!H4YD7_2-~&^^IlnXX;5h3-<-ztHdEkBmWH8K<1hG{kL0cenA3J_s zSzynBf<;u&1fU&!ptN`hJ6L~?IVfR{qdw_6m9o55ZV1w@78kpm>qw zPK09XIBA@x*i(@fY$QX|28`}XI0ULStdN^SZPI0Bnw7`VDRDi_eM)7i-k8FGTB{bP zII&Tfc7|Mbb|S{?$=fwzE;ftT6uWF6;_j2IBgu3m_3P!5vii0pUaJaK*q~a2w-czm z)2=f#X=6`?p|-E({o3;7>SIk1&}2M56OGPnY&b)q6RfMibYBU&bz5q}GnUwCT>-Cm z%EP;zp{SMhUQ-r;`@r2SOkHHJjm>$c{C-F}7LK#LsL^p3D^nNQV`kbKa&(M!KCUgY z-_lgvFR`^2N1%`7UGt;N1=B`l(%;gty9ABvEF)u`u$?jL1`FEMcQiB-0@ zYP%5RAh)PC%JAf=9KmY5J$IBnb&YA?jHoeBDG3OAMpG(m3XsEIuWx z_+T&yIv!!ION@Gtq6(Jl&HQ6q7@rr~Xe5 zu-0>kfVbW-m^)bJ9#A+TeEsZ*$jAWXNYGRQf#`ntK2M+MK(O^z)Z```m>=_ftm87% zadnR1^Qht+oKUZquhDE{m*y~VE0Y=_PcLhak5Ux8T~mZY4I~8M@&WKZH}B?y26kqR z*>?$t>z{I9R76aFz)td#P=)_M0OHjs6&hpNQ5QpR>X~h^fbU%k@Yo3@?a+uGlPH{` zIpl(vQ^$L>ahf)2;BFl|dWC}1oK_v)YYbfHH7|r0f0j%!gFogRoiC1`flw@Q7kN_wv3KqiFe&BkYq=%F|_cSf(w4RP~L{~ z>8rP+tQtAS^0xU|)5As%SUqHvZ~N_s7;EL0!QRu~KXB^`yp%|_EztH8>QAoV=L@rW zcfmE6_XS+O!mI%AB&ED7pD%j+d0&wac!J#p_lNspxw|*!=zEaY zJ@4sy528cU-eABp*Y$QnXKwp#|6iE%_(6Fs{V4BWI5f?Bc}SoSn7Qpg@qeDVxzn2M9}d$VD*ylh delta 158 zcmZp8AluL&J3*RtDFXvT$wUQv#-)u3TN4o_pNk=thYQG; z;4hlaZop``Sx~`>e|wQV=2.7,<3 +pytest==8.3.3 diff --git a/scripts/smoke.sh b/scripts/smoke.sh old mode 100644 new mode 100755 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..d64adf6 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +import os +import sys + + +# Ensure project root is on sys.path so `import app.*` works in Docker +PROJECT_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) +if PROJECT_ROOT not in sys.path: + sys.path.insert(0, PROJECT_ROOT) + + diff --git a/tests/test_auto_import.py b/tests/test_auto_import.py new file mode 100644 index 0000000..147f917 --- /dev/null +++ b/tests/test_auto_import.py @@ -0,0 +1,120 @@ +import json +from datetime import datetime + +import pytest + +from app.main import run_auto_import_for_upload, ORDER_INDEX + + +class DummyImportLog: + # Minimal stand-in to satisfy attribute assignments; not used by SQLAlchemy + def __init__(self, **kwargs): + self.import_type = kwargs.get("import_type") + self.file_name = kwargs.get("file_name") + self.file_path = kwargs.get("file_path") + self.status = kwargs.get("status", "pending") + self.total_rows = 0 + self.success_count = 0 + self.error_count = 0 + self.error_details = "[]" + self.completed_at = None + + +class DummyDB: + """Very small in-memory stub for the DB session interactions used by helper.""" + + def __init__(self, import_results_by_type): + self.import_results_by_type = import_results_by_type + self.logs = [] + + # SQLAlchemy-like API surface used by helper + def add(self, obj): + # record created logs + self.logs.append(obj) + + def commit(self): + # no-op for tests + return None + + +# Monkeypatch target lookup inside the helper by patching process function in module namespace +@pytest.fixture(autouse=True) +def patch_process_csv_import(monkeypatch): + from app import main as main_mod + + def fake_process_csv_import(db, import_type, file_path): + # return test-configured results from DummyDB + return db.import_results_by_type.get(import_type, {"success": 0, "errors": []}) + + monkeypatch.setattr(main_mod, "process_csv_import", fake_process_csv_import) + # Replace ImportLog class with Dummy for isolation (no DB) + monkeypatch.setattr(main_mod, "ImportLog", DummyImportLog) + + +def sorted_by_order(items): + return sorted(items, key=lambda x: (ORDER_INDEX.get(x["import_type"], 1_000_000), x.get("filename", ""))) + + +def test_auto_import_sorts_and_runs_in_order(): + uploaded = [ + {"filename": "B.csv", "stored_filename": "files_b.csv", "file_path": "/tmp/b.csv", "import_type": "files"}, + {"filename": "A.csv", "stored_filename": "trnstype_a.csv", "file_path": "/tmp/a.csv", "import_type": "trnstype"}, + {"filename": "C.csv", "stored_filename": "payments_c.csv", "file_path": "/tmp/c.csv", "import_type": "payments"}, + ] + + # Make all succeed + db = DummyDB({ + "trnstype": {"success": 1, "errors": [], "total_rows": 10}, + "files": {"success": 1, "errors": [], "total_rows": 20}, + "payments": {"success": 1, "errors": [], "total_rows": 5}, + }) + + result = run_auto_import_for_upload(db, uploaded) + + # Should be ordered by IMPORT_ORDER + filenames_in_result = [f["stored_filename"] for f in result["files"]] + expected_order = [i["stored_filename"] for i in sorted_by_order(uploaded)] + assert filenames_in_result == expected_order + assert result["stopped"] is False + assert result["stopped_on"] is None + assert result["skipped_unknowns"] == [] + + +def test_auto_import_skips_unknown_and_reports(): + uploaded = [ + {"filename": "Unknown.csv", "stored_filename": "unknown_1.csv", "file_path": "/tmp/u1.csv", "import_type": "unknown"}, + {"filename": "Rolodex.csv", "stored_filename": "rolodex_2.csv", "file_path": "/tmp/r2.csv", "import_type": "rolodex"}, + ] + + db = DummyDB({"rolodex": {"success": 1, "errors": [], "total_rows": 1}}) + result = run_auto_import_for_upload(db, uploaded) + + # Only known type processed + assert [f["stored_filename"] for f in result["files"]] == ["rolodex_2.csv"] + # Unknown skipped and listed + assert result["skipped_unknowns"] == [{"filename": "Unknown.csv", "stored_filename": "unknown_1.csv"}] + + +def test_auto_import_stops_on_first_error_and_sets_stopped_on(): + uploaded = [ + {"filename": "A.csv", "stored_filename": "trnstype_a.csv", "file_path": "/tmp/a.csv", "import_type": "trnstype"}, + {"filename": "B.csv", "stored_filename": "files_b.csv", "file_path": "/tmp/b.csv", "import_type": "files"}, + {"filename": "C.csv", "stored_filename": "payments_c.csv", "file_path": "/tmp/c.csv", "import_type": "payments"}, + ] + + # First succeeds, second returns errors -> should stop before third + db = DummyDB({ + "trnstype": {"success": 1, "errors": [], "total_rows": 10}, + "files": {"success": 0, "errors": ["bad row"], "total_rows": 2}, + "payments": {"success": 1, "errors": [], "total_rows": 5}, + }) + + result = run_auto_import_for_upload(db, uploaded) + + processed = [f["stored_filename"] for f in result["files"]] + # The run order starts with trnstype then files; payments should not run + assert processed == ["trnstype_a.csv", "files_b.csv"] + assert result["stopped"] is True + assert result["stopped_on"] == "files_b.csv" + + diff --git a/tests/test_import_detection.py b/tests/test_import_detection.py new file mode 100644 index 0000000..65fbac4 --- /dev/null +++ b/tests/test_import_detection.py @@ -0,0 +1,66 @@ +import pytest + +from app.main import get_import_type_from_filename + + +@pytest.mark.parametrize( + "name,expected", + [ + ("TRNSTYPE.csv", "trnstype"), + ("TrnsLkup.csv", "trnslkup"), + ("FOOTERS.csv", "footers"), + ("FILESTAT.csv", "filestat"), + ("EMPLOYEE.csv", "employee"), + ("GRUPLKUP.csv", "gruplkup"), + ("GROUPLKUP.csv", "gruplkup"), + ("FILETYPE.csv", "filetype"), + ("FVARLKUP.csv", "fvarlkup"), + ("RVARLKUP.csv", "rvarlkup"), + ("ROLEX_V.csv", "rolex_v"), + ("ROLEXV.csv", "rolex_v"), + ("ROLODEX.csv", "rolodex"), + ("ROLEX.csv", "rolodex"), + ("FILES_R.csv", "files_r"), + ("FILESR.csv", "files_r"), + ("FILES_V.csv", "files_v"), + ("FILESV.csv", "files_v"), + ("FILENOTS.csv", "filenots"), + ("FILE_NOTS.csv", "filenots"), + ("FILES.csv", "files"), + ("FILE.csv", "files"), + ("PHONE.csv", "phone"), + ("LEDGER.csv", "ledger"), + ("DEPOSITS.csv", "deposits"), + ("DEPOSIT.csv", "deposits"), + ("PAYMENTS.csv", "payments"), + ("PAYMENT.csv", "payments"), + ("PLANINFO.csv", "planinfo"), + ("PLAN_INFO.csv", "planinfo"), + ("QDROS.csv", "qdros"), + ("QDRO.csv", "qdros"), + ("MARRIAGE.csv", "pension_marriage"), + ("DEATH.csv", "pension_death"), + ("SCHEDULE.csv", "pension_schedule"), + ("SEPARATE.csv", "pension_separate"), + ("RESULTS.csv", "pension_results"), + ("PENSIONS.csv", "pensions"), + ("PENSION.csv", "pensions"), + ], +) +def test_get_import_type_from_filename_known(name, expected): + assert get_import_type_from_filename(name) == expected + + +@pytest.mark.parametrize( + "name", + [ + "UNKNOWN.csv", + "gibberish.xyz", + "", # empty + ], +) +def test_get_import_type_from_filename_unknown(name): + with pytest.raises(ValueError): + get_import_type_from_filename(name) + +