From 5d9dee15e8533ac66d7522da2719aa67ee0dda32 Mon Sep 17 00:00:00 2001 From: phiresky Date: Fri, 7 Jun 2019 20:12:24 +0200 Subject: [PATCH] check for too deep archive recursion --- Cargo.toml | 2 +- exampledir/droste.zip | Bin 0 -> 28809 bytes src/adapters/mod.rs | 2 +- src/adapters/sqlite.rs | 6 +----- src/adapters/tar.rs | 2 +- src/adapters/zip.rs | 5 +++-- src/bin/rga-preproc.rs | 6 ++++-- src/preproc.rs | 34 ++++++++++++++++++++++++++-------- src/preproc_cache.rs | 5 +++-- 9 files changed, 40 insertions(+), 22 deletions(-) create mode 100644 exampledir/droste.zip diff --git a/Cargo.toml b/Cargo.toml index c28b6b6..f4c520b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "rga" -description = "ripgrep but for pdf, ebooks, Office documents, archives, etc" +description = "ripgrep but also search in PDFs, E-Books, Office documents, etc, and also in archives" license = "AGPL-3.0-or-later" version = "0.4.0" repository = "https://github.com/phiresky/rga" diff --git a/exampledir/droste.zip b/exampledir/droste.zip new file mode 100644 index 0000000000000000000000000000000000000000..8b53ca566d1430f71a3f7cb1a541d95cf2688882 GIT binary patch literal 28809 zcmeI*Wo+im+92pB=`b^=!_3UgNr#!a!_3Ug%*@Q(VP;N;nVA{pJ!ik&+1dSOq><*w zj1*gzU36EK>>rlpOC=}u0|W&C06+jr!j(CMl-imTSO9>Ye-0=BEWpU##=+5;*4)-a zSqT*I!|d)uT-g->NNQ2mLWxGc>~wUI97d9oU?YcxrmA(6l1y<7U>2v|Xk!*nacqH* zDhUkqWfsrg`s#F_K5+YenVoj@mNxFD?Y#kp;E$?|0wgXj4mkLNx^4%H{gI1L^i}hP zeCiVw92OiL9v&R!=NBCo5fu>;5ET#*BKs>eL^d!cKuBChTv$+2R5VzMUqC`eQsS4S z=xa9uapIS=+-&X4Os)8h?X~z<&ciA=ejms?aA7V!15j*OtXycUP6(j!pQ+-4LSBC% zGh%Uqks?u|jrlZF5&$4UF<~KLVL_39{J{JH;1~S70wx4-f-w-`kwPX!l2UZHX_BW| zGNLgpn$d(4jB7=^X(`V23`{ zAhTa`rR1flrKK2OId6F35WeA$c;ce-nSrA;C|P_jK7kGjE4t_y6Totexw%0eYI|pt zfTwOXqx`R-EuihKOWMe`HsxM=vB3>`>g<8HR#m_(zX`)27HDGd1Da{>E!49-St>lZ zKFpsh<*H}T`Tm_L4s<_hl^^Gcc^e!&=377MvDVORS{KguSo+m`K(8OY|n&c5l3Wgtm3l@{@dq!ksYcP4Bt^>*(tLwT>ZOJq;iv&>RX z>LbtrJ}pAW%lE-a_Hm#ZB3ijzZBVJGJr6Tg%T5=Y$46`@?BN>EBV(TMWI2G2J3sSL zLZZ}ffCP~8CfNw>Q)sq7o#_NAta$a)CrZkk8?66?v15i#5hXRIJB2bPPKs(MgkM|t z^LphF21hh{Ks*Vx9xwQai~iXKGHU2!kEyQ4KuZ(wwNa&0_o1t{QTL_mrX`v1DcZT zT*D19A+GI$S*mU0>l5zQrgN!UL%@g7LiYv(Xsx8lX$zx{mvI)^3wR+SBSA=ayJ6FO z>3uYi1>GSL=g`Ywuag2_WUKRH_f{jJ1&a82{qNkCIE;P4Y;2v)k77IXxoQMmhDJ15 zq=W74+*Yvd`o_i{A?6Xt`cKa)kWaZFrMCB}6XyEtMe(bYB`cp+Ow766eNW1%+|Plv z21&!GJg+ABa;nO&1IZ&){s1b*Uva*tY3yQ~*ZX6dPbqt6<0ZqTuO{FVg{#>SpRI;G z;i|rM0e6%9xQ>jH=TC|@!Isov-bD)HQYXgc$EMuVB4GH(AVekgEn~PFcUcWn*>0Ln!Cq>HQkeoy?vF7 zjBFmfO1%4UZsK#0B40laYckwf2WpH1TJNDi38n7)x?DMUy~Hif%85G|y_AvKQ~56! zqvDGJ){`2fQVS*U=%yEozmO!$-m#zb41U>bjJSehYx4(uY`um0Rx``%vo+rKfp>@sK>iow@XyHV@$+}pjKO^c=ooPlEV(5cCY@b z_W)fkUxbZ9+CjI1Jk!4xKqqo<4o`sdp!KJ8e1%6HlwnFA5AtyU7~=8A>CV%LuzE0Y z7`jCSBG0t!xSC`G?%T{UM0wW^XS)69%mkN<&22Zzle?`ZaMw>$iwn%1BQ$XJ&g`jW z=~0j(@CUwLB52SV*QHlEdpc2Y%qK{A6<>QHJAYvUV5;(PC6+1^0%{w2;Nj#lQBfUq ztbf^vV>H%)4>rd&;-nVw_BE=Rk#@C(P8ymr9M(&=genblT-@Isb zYpzYY1TVb`pp2P^o%8sljJjg-{~RyPZPY00BHRAsPJxd34_s5@ww*NY^Ff$8ZYE3hHLbNy%w+r^5W~n z5o7vb?KOd!bQ!xW0UXBo{ERgyGR6tM&>s@%$dvcTjdd4c^}K^X5k;|~EyqPMrogBt zAr@GPJrO?~f^N4hz@F3x*E95ug%o5A4fQ^tj_GF85ovy*;^JzrM1^YcD8tarA+eyU7+=Eolr?k18GQTa zx6NN>!&C)cs1}rvCx`WT36Df!_@ke+$Lm}jPXc4#*psm3h%MOv1zCYgGs_85;%%-is5Mv3|oSj&(& zW;a@J)tO>JCAZEC8R>TjwXo9QtzjXE6Xk)Iq4#B^`Ok2Vp11+dIV%eL$73%#i2 zlo<-pKulUV@;{I};FAZ$UP-W=Uo@RHEdStGcj{!F@VC>gva>IippIEBiMhi_i=o(e z9Cbgm%!GjqDO)_Pw>?VV{a7Zf0#aj64l65&A`fz6k%mDAcNtX(@rn&4lc&}6LXqK zXm576RC&m%Qk0D5jon@I_ibn@RqXxmna8|*{CIPs2tEb89c7ZAKFQWz3R0KLf+S}T zV3I$U9`VFJoQ)%@nD9-cT;81`(<1$6!%3scK#qFL^80y2!60>^cy{cpb#a0RqDcR+ z-&!@B)Ain5OlY#4w1I;HU)K9aR-hZQR}|vlMmvxcMuEv#2}jX1KoMb1TClUf1eS*$ z+D?ls6?bH0vi&ZP<6aJ{e1uXH`A~dw8_g+`dxSEBBa(QZl_fiNtWEZway2gov@cHL zbqnb#M<9ag@>oF~W+jDsqXC0eCoG(e6`l~XYw&Q-t1%_!KLwFUYRirpDhm~LD+(cn zygfrSJ!hd7G)6kaluvOK3qW6C?A?tlMyESQG+|3ep6{dV>vQg*_~ieAvVo2#Sq_Zn ziQcmy)B|fS;AiN*9>LRIrZ?7%?hL1QUA!`cnyroweRLI-x8J4Tx4qQgUx@9QW--Di z0w9zHhb)0rVh&4WydIpPvi1!6<^V@X`UuMOBj7VhdA5UTNXK!vANh~d}#fA zek+0$LY&17eT(GmWQgu{5yO>7B|FTG4_ltW_AoqEUQN?sS5-H=P(YD)u~fa}o46!8 zd0%SwR@Q&_jnueg>RMUu*9#9&gGtU;Z^J678HGx0YezOemJd%riPQKP-^m%w;hW+K zLGaPj7AW?*8$AFJ8n!7hrEEb3F4u=7MeWR%4f`(2=bSQ>VhLwG63yL~BV%CBv`R2a zNi556{ix6N@UhogiL#y~^we-k7w_`BjgmRLa{6VGg;DXR9zl7d{2juAp-C;uG1OV+yB)QWJY@XGl2LxQY0f@X z)fcms&ZW}42}iZ6?`x8n9hPjkeb9217z#5*6I^)RvF1!cCw5|Z(%vbj>E!-}81 zzz;zRy9c|%&XmhC#uORx?b4S+a#e7M&8`Q3OwV}g;2?GK7Q)f&O5j@|2Fph|ZLo)v zl}uKYV27{eR4MI$1!Ip+6cxj0*0LbCow4y0K;Q~nzB^Gdg@ojN7ND@njRyNbm&eIW znhsY3JJQHNx^g{fn{5TQA3qJu2!oVd*Hvqi$g+qTw69n-Vy}Sxnds3DSgn9a;4~y` zYQ^G1Ci*?-fAn|7IZ`V#cu3gp0Hrld=qCx}ijRMcA3_*u@hYUz5kLwyFa^_{{Q5xdch5af8ebaCbw|uo}5ZO>YT!v!_7jw~?rEa8AakWy6 z(I#U9VI=K)Mk4WZ>8RET3;lb^drr7Dl0O<~rBZw~*q(n9eNboh6(82w1+dL<*;EKm zX&83v_PJZRU}8q2$uvD{l^uNbTneKiQ~Zg?Bg!QB!)0fuNYc~I1VnhGi;7OES`+!= z^A~qOA;N8y)Q7yR`%w)a_0dHf0JIYXq{n40&!cWtBCjHdkk!>Pxj@^Bl&)!Xp~hFh zrcQ#5Bcz{ia+t= zfx-Y*v-;K$1#&S?7ziHM3!JLG#}t?Ia@sd}Z(#$o3Zz%L77@O(ZT&Hw<}V*G#R4C? zsYMHX<^EuhKW=X5B7H=3wcvf6Tf`= zg17Z$KQ^kR847ghfdq6wKI-jM?C%~m(? z;S*70e4>Zz%_rEhns#z0@U?6r&XjrG4`r3iekyFN%^V@)1t5J|AKtADp+a~HtR%*G z(NKM^Dm!|WjC zoOg6WepBv)Wg-?xP!w}NaOq<;!wt4x&3G|hyb)TD#y~{@?x;Fw8Q2YH4mXejk!eLKJa&3qrEN6)B7-yQ3A^wy> zVOA9|I#~ZqKzWLFy+^=8T`|+wkIi%x=oAj?wRT;;GywnyN@)Xy%)N#jYYUvd8eN@N zT$IObCscJXrgaW8-T%2eIN^(~h`t<`P=2Y3xyx%RFKWlz5}*vx(#(Nb77%@W3abFA zH%OF@^n$hB!(ziJI17kzxq!R^?v&KTa1$Kqw@4e_nztD&2u@j3u)q!1oGNHmhAA#j zDvx3wza6u`Y40$f?3PD-W!z-jr9K$bJjg`wB>ZY zFx2z`eq^JyI1*o1&QznOXK%vr6#??3fA#xAyt?>tAaB|wdCki;t@Pl)``Lnw2WP&l z6@P0rAhh^MA?o(Dn}1G7VEZ_|cXg+k$)xLfilwX;ri1= z@||6;AgWjBq z3v_skH}tXQS=LHBi=-PzBJ<7)tQm>DvwMSb&xbDl+SOUy{phprqj?)>)&cC&`4*a9o^rRFsTWW`cF9pNjvs)7)K9TslN%OIjpzouFbB^*w<= ztzx`K#!cKZsVi*Q^(0lnihSwJ&@=NF%@|36G06^W@6@&_CrbdkPJ#l zglXb^HVYpIR7bko(ETqS^4O^f{niaT)4Oc6Lzjc7>X0~{eS;p9d?v}x!l7l-TAoLne8$}L?k#$-MZE`}Zd@SAMU&kS%97juS zHsInVJB1)>2K=~gTU?+wn;4#hFVC|*%nA0uf`$}h?t)==bawZ!snmawUa}29RyB72 z`K;njAF~+%YbJhHXs!Jfaur;fu{t}snNS&sGGmS=lM;i)-+`DvNi7<7eG~k)F#$)X zwzwkp%pK-oY^cKh)Q@Lt>mT^aqqP7kiz=q@d*FGzPK}OP5#(YY$n$o=+pF0vs+UCuD@w1<5lc6cccm<`-jUo2gjqR;i!p6 zJ*!YY6Kv&X*$-@jV2nol;V3~hDE}cXJ7)ofN1ND=f?saV4l8E3rg_Bz1`lR( z2sS>AvPn`T#=0Qqui!FW)IH{kwM2`x7Ty@Ba*=nQthk~_E_IwS1kA6ZId`;GVWEU_ zx!s(GTtq9OleT!I&uOWV*wwZadwYGd02zw1!y05#0zJW>8+tFk)p#eauX7R=t6T@? zEq*>uq+&fN*p4*O5Xq!}lE-!meM@KE2#=sF@`0b=lM&a8l8BR1CaNm>y{M%pEv~s1 zGRYiY7ajvKKaBkH3~ZEu;jv0YqQrkZ*a8lx?0<9UY?Z;<=@=Weck!AwXtyLXheXtt z=49){@4okX7HDM9Z79lsKm`?zDa&dwz5fVT1(!}o7oydgIY<8ZD%C+?MGR?HB%z~XJ1kzMC@FdN=#v>?djL(Mw9j`uhtqkyuPt8+Ch&;Zh{cP$< zv(`9bW3X-beM+RZoVo)!$Cni%bkb`XN~zkzdc4f&|IW)2l z!fg%DVI{w5nlcw`vHzqDlFJ#gIB$6zHo#(r@oBPlG5>xjQo6zZrHehm`y4J+u$8u~ z$p5nh_w2bl+~08uM!rf|#xKUK5uCC9AtCcEAFzf{R}dk|RJ53uVKE?|@I!Z2D_%T| zY({VEB?U?dhap(p2Fy6SD)${HNh{w}r#n=JkR~TBw|fOuq8Wtc>wZ6ZBXUJ zg2pS-nT+BXVzKr7u^hEDdE=pAsO%ThyV9=JaN0D;{pW#9$Tl@?59)=yjkV@rk<`RV zC8H44_-c4ip3fxR7@oV_ovei^IW96kCyLbtvS=|o+@}6u8THS~E7e*;#_$+f6HR(FOYXzm4qnX&1^YxDDM$m{ z^6h?(E#qFHITO>1JEvFmDH3XO33ppiOkdgO`4+@iVr7#4D$MezyeH+~DIX+m4_yM` zEb*%T@MCG=A2TuWg<>3;m^2^AwP_dC((+d8sw3LBt4*I6FYqkDgR5EixD6)t3d!#E z)W(FrC_NpT8t-ON15`9jUM)XOuml=x?Ht0mO1OHXVZ7+#o)Uu=&1Q~|l$9w?=~pL8 zL#^i|G7d(L@IOfPSBNLt1u~|CCwnXA3e8&!mBEdVnUe&EU0D>W;bc0^mG~{179=Kg zr%++;8S=|~fF1be)B4VEhTWVV+Qu6nkaxs9m#3N79;WQ*=$<1xdHbVkBOQr1U^|8k zrHMtBc8RlFfbePzLa`-#2kVUu1)Eik3>F$7Ag$L5VxZ`*x}u8{w5NnyAVZTyFi4^M z({~<$T%0rE9+xj-qg_BCmGCgKBI2Cd)&CGwQB}JEnv<+w#y+QTGep`kr51dvc7c_UFE4!kS2_GVUxa{i9}v7q0TJQaFrkj!%I zBMPSZQUU?BlimvcAud2`#mAaxcQ+isKHhHbGqe|)KU#VVBCy{*Cu!CahjhffO+5Vv zcI>u~Dm^RXbQSfM8>@SpzllClhC}I!hp?gBM@Ij!exo_AyydWe{?to!Jn410T zsU&Wdy#$_QSM5d`SCgQsii0ii43G$owKDP<&N~8W(rEcQ?zyAwG3bcv(InxgEn2km z+yfglgtMWOfWv`@K6|IkXfS4qNGMX1tilmz+I-XsA-MGAAisNK?(A(8>MD`8IKkf3kC|JDF8Sk64{TWIUN}{zE=J zN;?2pY~Y>|qt#s1PGXO~mV$y(uzNfhjbxro$93;u5BM;sMyDcM zf0gBwO>DjV{2H$4(66pkllw&P9V*05`ATAM?~a1%{axCUB$IEfmhRf!EO|u1VRDJ0#c2 zi^pbvzihxcDg@M@x-{d9XEC@(H`#T`^igPt;$;-1;JjEb`&Ev5XBi74x^du7;A7S-<#zRINH^w*?&%o6}bvk4=onjHg;f4 z9w3OlGgX^U6V$ImDahZ5P2TDgO+r;!R)sQXJF9DC7}AUzg**iQ^co9l6vgrkX2W2( z0&!$+_8Qk8p2!-5p$MRDggc!?Fa^SHs8=L2W3}}`9GB;~g`cyBk!#Jfc<6D$J~H^~ znHiMFn|4vi7*m(-lKjiPGV;`L*lYocQxjac@$y?a5sac#?C7!sY%+ztgjUIB%@bf3 zUAii_na7kBGuK#eUap`pD#ku@x5C|*POyR+F*kTPViaFteAAOsJ@b&oq{)=*ZKB`EPD&1MJo0bMkf9{k6YkYAl> zZ)>SQIL>Zqj+?So8uTGL$Ftuxu3eP+e))P!vTk3jU3kVn-=b7Xy&r`I2p^|Ba?=yc z=PWrDEv6>#QY6~<3R$=kW1sRN+8ESLvUb3MSvbn-uOzWP6`W;R3c%bacWFN3my_Yt zbl_Spu4)KyXJiVY!JvbgokfkDERSe?HrD*(HA`ODaP{le&3>O4Iog?94&MLRN7<&O8WqyRMbp)ED1`uht7WL#x)Scmvw`f81 z>5|r25$!}#*@(pa(C4Cp1+G3S6qZq3-0($f?PuPO9HXv06Q=F3Lu^`U0{is|t_wOJ zo>^yo{w7+U=pSExJn=4DF0M@xMI^x)l>)(vb3+tagO-90YV(3RQ0i2a0uxY?WK_FY zd0f664pUwDaB34gF;^{4Rh-mDgb|4ROJ2b*=Htq|T)t&U#2O5GK~Wdyw1-2`Atg^u z4TlDTula7gOa5{}U3#*i1|E78b7p8Ai4WwDsd`U&S8wprML^Go*Fv-uNOyyUOOZ&p z*2#K%!p2sUQeREn8(z01*!@E~CR2QHSoq@uHGQ;|bNpV8LtKM@0KL|&C7j&M>6<8= zO=eRv)(>7hYrZYE8&(fgBi8A(~ax=rK3ne=fI2NcJZvnBPtTM4!VVqOFTNvW`^Aj=hFk(Cys*ceLr_(LW z@%5lF>8DGN*ZS4A7U@!OvtI;NJd;*kkxczMBG(lL21C_aBFKiARW)jINovU_%>^I%8 z+ix7sMFOu`dkpfI?upl{zxWuZsVo>Pg4&I;-9%vn%Tg#4Yrk$F4{_-Ud%JEnXL}@K zR9Gn+r0>eRMUjU`mugc|E=wMz*`NA;qSj|YcS}1mEtfNSf>I%-r85QDeQ6{kfRS;4 z#}dnJ9cJ$yXDO(%dJS#Z8v0_aklG-|>ifG;m{AZvvd{xnZbu0MhR*~9qLIhK_+iCF(#FulEfpqN6av_G< zfB^pRmF?^25F{*p=;q(dgfPFB`2IB2-yY7(nC9S!H2^=M&F(?g6`5xP3(Z3R!n7l* zKI?vAF6;ikrGKBEF`TqOa*kieq|>M49sF|UVqE#;^_U|QP4+v~hQ@L!QX|A$CpP7p zF=lkVx%-3^->P$db4KZA4;At-8{m45=j=OnA$EAQKu~E`DzArGY^c9Q7;(pHCvB@& zrP4z-+1x}shdZGx@m_~>!GmL=|DhQv2~eTl_BP3+XV|Z7aUrrbL|sm6#O9={x{i3N zH{$%GoINg_68sCly`3wtDc)HqS{F!r*onrE6H-T~1X(R1SPbWLOpO&+IEn6_(A1)O zTCXq!_mQ?l%3Mq{I;npHI_CEjl|KZJG7}0Q&VkDq!GuA$&r6Y{e};BQ9(!PMDVwx! zU+jXEy3H>R$B=8DP))6q~gHh}2u*#9)>yR>pXM zDB7oH_r_*~t(YH*KD3A~d)~VTVoHmi7@n^s5YZ!s)-O5XH`P)N$;9-D4+*a^ ziOIyu*TJ?EI6U~tm*Bj;N5K)hmXDllZON}NrW$R;_O4_|nQJ5{sTBuDcwVeE1|6Fr@9&TB3Gaj6k%Y=+ zK&SfhxQf$~uJ@wXb_%26p)mco>(>>^2nqMSJsJ{%aS6L%82o4B#A(-_@ zpkG<@j^hgcXcAE|`&dYq?wWw^j=eyTm7&}-kX88`-Biox1$zXJ2%K?J3(eD}^F+am zebp$>lC$aq<@oqF>TLN-w1^Q%9Fkv;Z&_-jgzidn(3rX#E51fW(2f>@R@yM8;qRl!P>AHnDB zoJPtVJ*F|GP>ubdgprV~>D=%e(b@1Rk(%bYOhWc<=T*O6Mu14XniI&|#3qkuL2{?r zH5O(6&T77qiF+y|*InMMqG2)2BMh8&BbmLsLdZ6)rlpLi=b8z%neuO|E>)GDQQ<`7 zwL}Do61%k8Gts+ojx{ed6}Ewh#E|t9)U(r`NGMapbw_JP)ev?HRH-})b`49I13mG0 zBhhQ*?`akSN zc#%x(H1tAk)wLLWCx)u@a6dziQd{ZGb-h0e;{2-{!$S@wxx4e6tT4E7TY^J2N*m5z zym=s@5AglQt%tCC7mIa?Kn)YdN)HP>%_1}LeYgo6$RYuK2au3msJJt6WgALzEHk6t zuK{Fd7@Tq^6raY_@#3Bw*^?YED6Ml+ZTDDi%- z)}+5a+i6QtS`nRwvLxhlkiP$f;V6~h(Jc)srR8+mD4eldXM`nbB0(dTeAw%e8d?mc z4F{SZM>p@2H8j@quqq;rJ-~xq9^1uoSSJaoU)RADK1k>OSvmP$vE=Cf42ouSg3r*Wler?0#u20>cu5N;4P ztn0F-fqrhscBoOeV#a!upwDyTc3;id^BP+Npjwlxa0?k^IkT4*7%L%>ESWh~aR_*X zVWW6UDs!{LnZ-DI)6nj;R}B^+wtOK}`Aqc2mRW*~GG_j zRxygWM!}ElSKz7?!tv704`5L{jkN3dlo&uPb5qa_xRz2C#-5lye>w@|5a+Eluj!O*!3IQQy%XCl6a zA#VIpR1A)&8=rC?FC6ysqoLE8!A(9<1tj!kj`zYW8Y7!ybM;-n!cL1Pso0`Ev9MTS zOBr6H96D9lHrcGL$?_(RzV4K8SDBi@N#(NLrMd1xx=A{gA=<#s z+~CpvEZslj4NFnS8gj8M`E*!a5rk@SBh$&za3A8NwOp{&g`bj-Wh@e5oem@jre#_AFGk5NzKeEI8YJCf>Noyxm%1~30M|Sb zTvW0UGjfU?l;ALb>7_P|PMaQ+zxqRrQT)Pvm5$An%G4j;rs(<~OHpo_h^H zQ1Y*qdwQL=__=qS`05r2@mhX94)&5;mpFZmc_EBjZ68HiR_jMYLH2;b=qRm*0A29; zjY>y8m8>Jb17CFQivy0n8Z!g!?Hf&H2+fOY;Wt%hH@MaKO}q}Wo^DrpVG`7^4-oDS z_q{$5EK1HY<0V<1m@HHy9+!Cg1U;sp7D7Jl1+Nr!s-vg_1plO%=r}f#bNyXm+n7P= z`NT{q$5!}ckDhKdCDc5%qI-ZgOZGMU!qvSm32gN+6Y5dd4Wv0N`PCsI3*u`}K( z2#sMwHw%4Y{6!#-R)(s*_3K`i{NS<>2Le2zdquS0*EvL9AQ)ak-VL#0$o-9+@}pp0 zU4*`IMZwywt~e^J$(6p*vv-_WNC@@yIfnj;rgzSy8YLV6p<#0NeLBY2?t}nxpE%ioZB9w%d4xuB>7ad4<}#?@@!_yLSb zPSHfslf_F@M6dx=KOaR$-Z|paLRk+&U)m<=R}x>M>0pT=HSsIdbEB3uI{D-WHN`x+ z_iWo}QUAkIq2gf1m|_qpa~tHx{fAS>fXf>>-=cRZ z^~>b);K(85A8kS$?jYe=Z_~-=d_r`qTbt zlOw__xC$jPZf!!`7t6F|pR9VTJan<0$_Qg-jZf{l7fFy6R)*n5u%a!yR7<4=bI-^N z{V%0sC|R)HEn~_`Rg3zpT;jjOW$x^>LSX>Y#ik^(jCHn^%uKpAlzR?=7!2$H(?H;k_|o?#W+d+=%p* zBHX63$cheE(efkurV25OOX90%m9qSHpsa`fm;GDtT1fub?5khFu}be->*$Q~ zn95wh0?U8(QBkQYgM*Kpfe#=wjN#gHTLxCkrT`7n`dFUb+C|6-!e}McGiXE@Cqs&S z+ovH|Roh76nyf&zPWpspiRoTR}f~f!DYsItq&{Wbs!NU8)1W`4|LX> zf%{Fi!+AM52dm{lKn>l-)1WgLUzs|6PbsmvGIw=p^H(fF54J-FyFGkn>?uIMr5&dn zu3CihXDi@Hx9FO2%0zL-K$*w;>D&jF!W47ulHiIo7Hccdl@z%?bDrYtLj!4jk}XQcDsc~@c-dPkCDBe zUxqUm${bl!Rq<>k)7-4qAl7WbX0(*--0e?*FgRzkA3Ab#k>Q6#;B9cP8!+MXW8Fxc zaumiXVc7@GB(Z{1rw!@d-sq&*>`!?QS?864E7@cUO%Zi5YUZ90zI`_ra29!uapLR` zw8(YM2C^Ny3U30I?L)LYQ4~sffJ?WelP_skrHVytYi6G&hCdtlg&0jtm^?2)NuwQ$ z^q5Q-1yuKNvz$nlDEir#+Fr2;-}M@@NEm|}T$CT9kcUs$y>RbGA>5v`JhIgSZFX?v zwX4J-Izm)u=vXkyGFcIkQRl|*z%sJ~l1jl9%8-CxWe-MHr| zx3Xg{APp`^iwxmXVEp^1$wF$8EBs+8pSW9#euLqwFb9*HNkrAqEe-xJF^b_w-5p_q zHGnsKFGo~q|tPDjU2)_`ejNxH6%^B7IwygXee^?pvX;|zaYpjF)a8UWF2 z^cNbpKLMYd<3duvcyf068P9@EfqOR}i=zWCV>j^rWecB%5v`0;?`0mT<@B(Ni;=A7 zMUAgmln*F&ULN8EbfG*UN$6##@1}TNuTOc6ug@q&?_a1IUz78$E^pd6laGq}MiU@h zqqyNu8=nBXx=kAJFCvf1qXUPe)OqgWt6BXFA5cpz5WKpLH0(dXGMj&Z!?#Fs3y2P+ zJ(`@yw+wBL&2;8`!~_KL%I`h3U-;zyaa>UUc3gtlDjbOIpx0oW06>`SzZ@6f|8`vN zX10I_7O1g*RsKd_-{|WbeSM>^Z}jzzzP{1dH~RWUU*G8K8-0DFuW$7AjlRCo*Ejn5 zMql6P>l=N2qpxrD^^Ly1(bqTn`bJ;h=<6GOeWR~$^!1ItzR}k=`uav+-{|WbeSM>^ zZ}jzzzP{1dH~RWUU*G8K8-0DFuW$7AjlRCo*Ejn5Mql6P>l=N2qpxrD^}h-F!g|T* z68$W$iR(VG*tdE8KQ#$9TTJH*0L%eszWz_0!vE(k{J&@9%?k|ikJtP90>A^Vzp(z> zq~IPf$iEALA@BhJB*4Yj|3Bw{P~^`90RDILzQq6Nbg?+E~%qIhk1-)BXb}De!^*b${fffPhf||9wRHuNePz`|qry_pcyO{|1^6fd7nb z{?`B_|18M&A3^>ZgPas7*uTnv|Jl9%IX~L`E7eB`g7DAj>kF&08vOryivE+V|6ND_ FUjQ>=xpDvi literal 0 HcmV?d00001 diff --git a/src/adapters/mod.rs b/src/adapters/mod.rs index 219a8d4..055e064 100644 --- a/src/adapters/mod.rs +++ b/src/adapters/mod.rs @@ -57,7 +57,7 @@ pub struct AdaptInfo<'a> { /// prefix every output line with this string to better indicate the file's location if it is in some archive pub line_prefix: &'a str, // pub adapt_subobject: &'a dyn Fn(AdaptInfo) -> Fallible<()>, - pub config: &'a mut PreprocConfig, + pub config: PreprocConfig, } pub fn extension_to_regex(extension: &str) -> Regex { diff --git a/src/adapters/sqlite.rs b/src/adapters/sqlite.rs index 5386aa6..71fd679 100644 --- a/src/adapters/sqlite.rs +++ b/src/adapters/sqlite.rs @@ -1,13 +1,9 @@ -use super::spawning::map_exe_error; use super::*; use failure::*; use lazy_static::lazy_static; -use rusqlite::types::{ToSql, ValueRef}; +use rusqlite::types::ValueRef; use rusqlite::*; -use serde::{Deserialize, Serialize}; use std::convert::TryInto; -use std::io::BufReader; -use std::process::*; static EXTENSIONS: &[&str] = &["db", "db3", "sqlite", "sqlite3"]; diff --git a/src/adapters/tar.rs b/src/adapters/tar.rs index ff808c9..2c6ff92 100644 --- a/src/adapters/tar.rs +++ b/src/adapters/tar.rs @@ -82,7 +82,7 @@ impl FileAdapter for TarAdapter { inp: &mut file, oup, line_prefix, - config, + config: config.clone(), }; rga_preproc(ai2)?; } diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index 38a505c..3096470 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -61,7 +61,8 @@ impl FileAdapter for ZipAdapter { continue; } eprintln!( - "{}|{}: {} bytes ({} bytes packed)", + "{}{}|{}: {} bytes ({} bytes packed)", + line_prefix, filepath_hint.to_string_lossy(), file.name(), file.size(), @@ -75,7 +76,7 @@ impl FileAdapter for ZipAdapter { oup, line_prefix, archive_recursion_depth: archive_recursion_depth + 1, - config, + config: config.clone(), })?; } Err(e) => return Err(e.into()), diff --git a/src/bin/rga-preproc.rs b/src/bin/rga-preproc.rs index 1026a9c..fad5284 100644 --- a/src/bin/rga-preproc.rs +++ b/src/bin/rga-preproc.rs @@ -4,7 +4,6 @@ use rga::preproc::*; use std::env; use std::fs::File; use std::io::BufReader; - fn main() -> Result<(), Error> { let path = { let filepath = std::env::args_os() @@ -28,7 +27,10 @@ fn main() -> Result<(), Error> { oup: &mut o, line_prefix: "", archive_recursion_depth: 0, - config: &mut PreprocConfig { cache }, + config: PreprocConfig { + cache, + max_archive_recursion: 3, + }, }; rga_preproc(ai) diff --git a/src/preproc.rs b/src/preproc.rs index 9bbdb87..423e79d 100644 --- a/src/preproc.rs +++ b/src/preproc.rs @@ -3,14 +3,16 @@ use crate::CachingWriter; use failure::Fallible; use failure::{format_err, Error}; use path_clean::PathClean; -use std::convert::AsRef; use std::io::BufWriter; // longest compressed conversion output to save in cache const MAX_DB_BLOB_LEN: usize = 2_000_000; const ZSTD_LEVEL: i32 = 12; +use std::sync::{Arc, RwLock}; +#[derive(Clone)] pub struct PreprocConfig { - pub cache: Option>, + pub cache: Option>>, + pub max_archive_recursion: i32, } /** * preprocess a file as defined in `ai`. @@ -27,11 +29,21 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<(), Error> { oup, line_prefix, config, + archive_recursion_depth, .. } = ai; + let PreprocConfig { + mut cache, + max_archive_recursion, + } = config; let filename = filepath_hint .file_name() .ok_or_else(|| format_err!("Empty filename"))?; + eprintln!("depth: {}", archive_recursion_depth); + if archive_recursion_depth >= config.max_archive_recursion { + writeln!(oup, "{}[rga: max archive recursion reached]", line_prefix)?; + return Ok(()); + } eprintln!("path_hint: {:?}", filepath_hint); @@ -49,7 +61,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<(), Error> { let meta = ad.metadata(); eprintln!("adapter: {}", &meta.name); let db_name = format!("{}.v{}", meta.name, meta.version); - if let Some(cache) = config.cache.as_mut() { + if let Some(cache) = cache.as_mut() { let cache_key: Vec = { let clean_path = filepath_hint.to_owned().clean(); let meta = std::fs::metadata(&filepath_hint)?; @@ -62,7 +74,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<(), Error> { bincode::serialize(&key).expect("could not serialize path") // key in the cache database }; - cache.get_or_run( + cache.write().unwrap().get_or_run( &db_name, &cache_key, Box::new(|| -> Fallible>> { @@ -76,8 +88,11 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<(), Error> { is_real_file, inp, oup: &mut compbuf, - archive_recursion_depth: 0, - config: &mut PreprocConfig { cache: None }, + archive_recursion_depth, + config: PreprocConfig { + cache: None, + max_archive_recursion, + }, })?; let compressed = compbuf .into_inner() @@ -104,8 +119,11 @@ pub fn rga_preproc(ai: AdaptInfo) -> Result<(), Error> { is_real_file, inp, oup, - archive_recursion_depth: 0, - config: &mut PreprocConfig { cache: None }, + archive_recursion_depth, + config: PreprocConfig { + cache: None, + max_archive_recursion, + }, })?; Ok(()) } diff --git a/src/preproc_cache.rs b/src/preproc_cache.rs index f041ac6..ce9ba3b 100644 --- a/src/preproc_cache.rs +++ b/src/preproc_cache.rs @@ -1,7 +1,8 @@ use failure::{format_err, Fallible}; +use std::sync::{Arc, RwLock}; -pub fn open() -> Fallible> { - Ok(Box::new(LmdbCache::open()?)) +pub fn open() -> Fallible>> { + Ok(Arc::new(RwLock::new(LmdbCache::open()?))) } pub trait PreprocCache { // possible without second lambda?