Archives
/
ripgrep-all
mirror of https://github.com/phiresky/ripgrep-all
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

switch failure -> anyhow, parse byte suffix in cache-max-blob-len

Browse Source
pull/66/head
phiresky 4 years ago
parent 5d97c2ad5f
commit f2d9393832
19 changed files with 189 additions and 173 deletions
Show Stats Download Patch File Download Diff File

160
Cargo.lock generated
Unescape Escape View File

@ -1,14 +1,5 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "addr2line"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "456d75cbb82da1ad150c8a9d97285ffcd21c9931dcb11e995903e7d75141b38b"
dependencies = [
"gimli",
]
[[package]]
name = "adler32"
version = "1.0.4"
@ -33,6 +24,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "anyhow"
version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85bb70cc08ec97ca5450e6eba421deeea5f172c0fc61f78b5357b2a8e8be195f"
[[package]]
name = "arrayref"
version = "0.3.6"
@ -62,19 +59,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
[[package]]
name = "backtrace"
version = "0.3.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0df2f85c8a2abbe3b7d7e748052fdd9b76a0458fdeb16ad4223f5eca78c7c130"
dependencies = [
"addr2line",
"cfg-if",
"libc",
"object",
"rustc-demangle",
]
[[package]]
name = "bincode"
version = "1.2.1"
@ -109,12 +93,13 @@ dependencies = [
[[package]]
name = "bzip2-sys"
version = "0.1.8+1.0.8"
version = "0.1.9+1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05305b41c5034ff0e93937ac64133d109b5a2660114ec45e9760bc6816d83038"
checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
@ -125,9 +110,9 @@ checksum = "c06509d1f4ffa658939bd23f076cd929ef218241363796551528e7eec69128c8"
[[package]]
name = "cc"
version = "1.0.53"
version = "1.0.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "404b1fe4f65288577753b17e3b36a04596ee784493ec249bf81c7f2d2acd751c"
checksum = "7bbb73db36c1246e9034e307d0fba23f9a2e251faa47ade70c1bd252220c8311"
dependencies = [
"jobserver",
]
@ -235,9 +220,9 @@ dependencies = [
[[package]]
name = "crossbeam-queue"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db"
checksum = "ab6bffe714b6bb07e42f201352c34f51fefd355ace793f9e638ebd52d23f98d2"
dependencies = [
"cfg-if",
"crossbeam-utils",
@ -254,6 +239,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "either"
version = "1.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
[[package]]
name = "encoding_rs"
version = "0.8.23"
@ -285,22 +276,12 @@ dependencies = [
"termcolor",
]
[[package]]
name = "exitfailure"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ff5bd832af37f366c6c194d813a11cd90ac484f124f079294f28e357ae40515"
dependencies = [
"failure",
]
[[package]]
name = "failure"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
dependencies = [
"backtrace",
"failure_derive",
]
@ -390,12 +371,6 @@ dependencies = [
"wasi",
]
[[package]]
name = "gimli"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcc8e0c9bce37868955864dbecd2b1ab2bdf967e6f28066d65aaac620444b65c"
[[package]]
name = "glob"
version = "0.3.0"
@ -440,6 +415,15 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "itertools"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "284f18f85651fe11e8a991b2adb42cb078325c996ed026d994719efcfca1d54b"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "0.4.5"
@ -463,9 +447,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.70"
version = "0.2.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3baa92041a6fec78c687fa0cc2b3fae8884f743d672cf551bed1d6dac6988d0f"
checksum = "9457b06509d27052635f90d6466700c65095fdf75409b3fbdd903e988b886f49"
[[package]]
name = "libsqlite3-sys"
@ -651,12 +635,6 @@ dependencies = [
"autocfg 1.0.0",
]
[[package]]
name = "object"
version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cbca9424c482ee628fa549d9c812e2cd22f1180b9222c9200fdfa6eb31aecb2"
[[package]]
name = "ordered-float"
version = "1.0.2"
@ -701,9 +679,9 @@ dependencies = [
[[package]]
name = "paste"
version = "0.1.12"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a229b1c58c692edcaa5b9b0948084f130f55d2dcc15b02fcc5340b2b4521476"
checksum = "d508492eeb1e5c38ee696371bf7b9fc33c83d46a7d451606b96458fbbbdc2dec"
dependencies = [
"paste-impl",
"proc-macro-hack",
@ -711,9 +689,9 @@ dependencies = [
[[package]]
name = "paste-impl"
version = "0.1.12"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e0bf239e447e67ff6d16a8bb5e4d4bd2343acf5066061c0e8e06ac5ba8ca68c"
checksum = "84f328a6a63192b333fce5fbb4be79db6758a4d518dfac6d54412f1492f72d32"
dependencies = [
"proc-macro-hack",
"proc-macro2",
@ -751,9 +729,9 @@ checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
[[package]]
name = "podio"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "780fb4b6698bbf9cf2444ea5d22411cef2953f0824b98f33cf454ec5615645bd"
checksum = "b18befed8bc2b61abc79a457295e7e838417326da1586050b919414073977f19"
[[package]]
name = "ppv-lite86"
@ -789,15 +767,15 @@ dependencies = [
[[package]]
name = "proc-macro-hack"
version = "0.5.15"
version = "0.5.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63"
checksum = "7e0456befd48169b9f13ef0f0ad46d492cf9d2dbb918bcf38e01eed4ce3ec5e4"
[[package]]
name = "proc-macro2"
version = "1.0.13"
version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53f5ffe53a6b28e37c9c1ce74893477864d64f74778a93a4beb43c8fa167f639"
checksum = "beae6331a816b1f65d04c45b078fd8e6c93e8071771f41b8163255bbd8d7c8fa"
dependencies = [
"unicode-xid",
]
@ -981,9 +959,9 @@ checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
[[package]]
name = "regex"
version = "1.3.7"
version = "1.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692"
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
dependencies = [
"aho-corasick",
"memchr",
@ -993,9 +971,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.6.17"
version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae"
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
[[package]]
name = "remove_dir_all"
@ -1010,6 +988,7 @@ dependencies = [
name = "ripgrep_all"
version = "0.9.7-alpha.0"
dependencies = [
"anyhow",
"bincode",
"bzip2",
"cachedir",
@ -1019,8 +998,6 @@ dependencies = [
"encoding_rs",
"encoding_rs_io",
"env_logger",
"exitfailure",
"failure",
"flate2",
"glob",
"lazy_static",
@ -1079,12 +1056,6 @@ dependencies = [
"time",
]
[[package]]
name = "rustc-demangle"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
[[package]]
name = "rustc_version"
version = "0.2.3"
@ -1096,9 +1067,9 @@ dependencies = [
[[package]]
name = "ryu"
version = "1.0.4"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "scopeguard"
@ -1123,18 +1094,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.110"
version = "1.0.111"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99e7b308464d16b56eba9964e4972a3eee817760ab60d88c3f86e1fecb08204c"
checksum = "c9124df5b40cbd380080b2cc6ab894c040a3070d995f5c9dc77e18c34a8ae37d"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.110"
version = "1.0.111"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "818fbf6bfa9a42d3bfcaca148547aa00c7b915bec71d1757aa2d44ca68771984"
checksum = "3f2c3ac8e6ca1e9c80b8be1023940162bf81ae3cffbb1809474152f2ce1eb250"
dependencies = [
"proc-macro2",
"quote",
@ -1209,9 +1180,9 @@ dependencies = [
[[package]]
name = "syn"
version = "1.0.22"
version = "1.0.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1425de3c33b0941002740a420b1a906a350b88d08b82b2c8a01035a3f9447bac"
checksum = "93a56fabc59dce20fe48b6c832cc249c713e7ed88fa28b0ee0a3bfcaae5fe4e2"
dependencies = [
"proc-macro2",
"quote",
@ -1243,9 +1214,9 @@ dependencies = [
[[package]]
name = "tar"
version = "0.4.26"
version = "0.4.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3"
checksum = "5c058ad0bd6ccb84faa24cc44d4fc99bee8a5d7ba9ff33aa4d993122d1aeeac2"
dependencies = [
"filetime",
"libc",
@ -1389,9 +1360,9 @@ checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11"
[[package]]
name = "vcpkg"
version = "0.2.8"
version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fc439f2794e98976c88a2a2dafce96b930fe8010b0a256b3c2199a773933168"
checksum = "55d1e41d56121e07f1e223db0a4def204e45c85425f6a16d462fd07c8d10d74c"
[[package]]
name = "vec_map"
@ -1401,9 +1372,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
[[package]]
name = "version_check"
version = "0.9.1"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed"
[[package]]
name = "wasi"
@ -1475,18 +1446,18 @@ dependencies = [
[[package]]
name = "zstd"
version = "0.5.1+zstd.1.4.4"
version = "0.5.2+zstd.1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c5d978b793ae64375b80baf652919b148f6a496ac8802922d9999f5a553194f"
checksum = "644352b10ce7f333d6e0af85bd4f5322dc449416dc1211c6308e95bca8923db4"
dependencies = [
"zstd-safe",
]
[[package]]
name = "zstd-safe"
version = "2.0.3+zstd.1.4.4"
version = "2.0.4+zstd.1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bee25eac9753cfedd48133fa1736cbd23b774e253d89badbeac7d12b23848d3f"
checksum = "7113c0c9aed2c55181f2d9f5b0a36e7d2c0183b11c058ab40b35987479efe4d7"
dependencies = [
"libc",
"zstd-sys",
@ -1494,11 +1465,12 @@ dependencies = [
[[package]]
name = "zstd-sys"
version = "1.4.15+zstd.1.4.4"
version = "1.4.16+zstd.1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89719b034dc22d240d5b407fb0a3fe6d29952c181cff9a9f95c0bd40b4f8f7d8"
checksum = "c442965efc45353be5a9b9969c9b0872fff6828c7e06d118dda2cb2d0bb11d5a"
dependencies = [
"cc",
"glob",
"itertools",
"libc",
]

21
Cargo.toml
Unescape Escape View File

@ -17,32 +17,31 @@ exclude = [
[dependencies]
tree_magic = { package = "tree_magic_fork", version = "0.2.2" }
regex = "1.3.6"
regex = "1.3.9"
rkv = "0.10.4"
cachedir = "0.1.1"
path-clean = "0.1.0"
bincode = "1.2.1"
serde = { version = "1.0.106", features = ["derive"] }
zstd = "0.5.1"
serde = { version = "1.0.111", features = ["derive"] }
zstd = "0.5.2"
lazy_static = "1.4.0"
serde_json = "1.0.51"
failure = "0.1.7"
serde_json = "1.0.53"
zip = "0.5.5"
crossbeam = "0.7.3"
clap = { version = "2.33.0", features = ["wrap_help"] }
clap = { version = "2.33.1", features = ["wrap_help"] }
log = "0.4.8"
env_logger = "0.7.1"
xz2 = "0.1.6"
flate2 = "1.0.14"
bzip2 = "0.3.3"
tar = "0.4.26"
tar = "0.4.28"
chrono = "0.4.11"
encoding_rs = "0.8.22"
encoding_rs = "0.8.23"
encoding_rs_io = "0.1.7"
rusqlite = { version = "0.23.1", features = ["vtab", "bundled"] }
size_format = "1.0.2"
structopt = "0.3.12"
paste = "0.1.10"
structopt = "0.3.14"
paste = "0.1.16"
tempfile = "3.1.0"
glob = "0.3.0"
exitfailure = "0.5.1"
anyhow = "1.0.31"

2
README.md
Unescape Escape View File

@ -211,7 +211,7 @@ The following adapters are disabled by default, and can be enabled using '--rga-
> \[default: 12\]
**\--rga-cache-max-blob-len** \<cache-max-blob-len\>
**\--rga-cache-max-blob-len**=\<cache-max-blob-len\>
> Max compressed size to cache
>

6
src/adapters.rs
Unescape Escape View File

@ -10,7 +10,7 @@ pub mod tesseract;
pub mod zip;
use crate::matching::*;
use crate::preproc::PreprocConfig;
use failure::*;
use anyhow::*;
use log::*;
use regex::Regex;
use std::borrow::Cow;
@ -58,7 +58,7 @@ pub trait FileAdapter: GetMetadata {
/// adapt a file.
///
/// detection_reason is the Matcher that was used to identify this file. Unless --rga-accurate was given, it is always a FastMatcher
fn adapt(&self, a: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()>;
fn adapt(&self, a: AdaptInfo, detection_reason: &SlowMatcher) -> Result<()>;
}
pub struct AdaptInfo<'a> {
/// file path. May not be an actual file on the file system (e.g. in an archive). Used for matching file extensions.
@ -107,7 +107,7 @@ pub fn get_all_adapters() -> AdaptersTuple {
*/
pub fn get_adapters_filtered<T: AsRef<str>>(
adapter_names: &[T],
) -> Fallible<Vec<Rc<dyn FileAdapter>>> {
) -> Result<Vec<Rc<dyn FileAdapter>>> {
let (def_enabled_adapters, def_disabled_adapters) = get_all_adapters();
let adapters = if !adapter_names.is_empty() {
let adapters_map: HashMap<_, _> = def_enabled_adapters

6
src/adapters/decompress.rs
Unescape Escape View File

@ -1,6 +1,6 @@
use super::*;
use crate::preproc::rga_preproc;
use failure::*;
use anyhow::*;
use lazy_static::lazy_static;
use std::path::PathBuf;
@ -46,7 +46,7 @@ impl GetMetadata for DecompressAdapter {
}
}
fn decompress_any<'a, R>(reason: &SlowMatcher, inp: &'a mut R) -> Fallible<Box<dyn Read + 'a>>
fn decompress_any<'a, R>(reason: &SlowMatcher, inp: &'a mut R) -> Result<Box<dyn Read + 'a>>
where
R: Read,
{
@ -91,7 +91,7 @@ fn get_inner_filename(filename: &Path) -> PathBuf {
}
impl FileAdapter for DecompressAdapter {
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
filepath_hint,
mut inp,

4
src/adapters/ffmpeg.rs
Unescape Escape View File

@ -1,6 +1,6 @@
use super::spawning::map_exe_error;
use super::*;
use failure::*;
use anyhow::*;
use lazy_static::lazy_static;
use serde::{Deserialize, Serialize};
use std::io::BufReader;
@ -48,7 +48,7 @@ struct FFprobeStream {
codec_type: String, // video,audio,subtitle
}
impl FileAdapter for FFmpegAdapter {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
is_real_file,
filepath_hint,

4
src/adapters/pdfpages.rs
Unescape Escape View File

@ -43,7 +43,7 @@ impl GetMetadata for PdfPagesAdapter {
/// A pdf is basically converted to a zip that has Page X.png files.
/// This way, something like tesseract can process the pages individually
impl FileAdapter for PdfPagesAdapter {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
filepath_hint,
is_real_file,
@ -114,7 +114,7 @@ fn split_by_seq<'a>(
split_seq: &'a [u8],
split_inx: usize,
read: &mut Read,
) -> Fallible<impl IntoIterator<Item = impl Read> + 'a> {
) -> Result<impl IntoIterator<Item = impl Read> + 'a> {
let regex = split_seq
.iter()
.map(|c| format!("\\x{:0>2x}", c))

2
src/adapters/poppler.rs
Unescape Escape View File

@ -42,7 +42,7 @@ impl SpawningFileAdapter for PopplerAdapter {
cmd.arg("-").arg("-");
cmd
}
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Fallible<()> {
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> {
// prepend Page X to each line
let mut page = 1;
for line in BufReader::new(inp).lines() {

14
src/adapters/spawning.rs
Unescape Escape View File

@ -1,6 +1,6 @@
use super::*;
use anyhow::*;
use encoding_rs_io::DecodeReaderBytesBuilder;
use failure::*;
use log::*;
use std::io::prelude::*;
use std::io::BufReader;
@ -20,7 +20,7 @@ pub fn postproc_line_prefix(
line_prefix: &str,
inp: &mut dyn Read,
oup: &mut dyn Write,
) -> Fallible<()> {
) -> Result<()> {
// TODO: parse these options from ripgrep's configuration
let encoding = None; // detect bom but usually assume utf8
let bom_sniffing = true;
@ -57,7 +57,7 @@ pub trait SpawningFileAdapter: GetMetadata {
fn get_exe(&self) -> &str;
fn command(&self, filepath_hint: &Path, command: Command) -> Command;
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Fallible<()> {
fn postproc(line_prefix: &str, inp: &mut dyn Read, oup: &mut dyn Write) -> Result<()> {
postproc_line_prefix(line_prefix, inp, oup)
}
}
@ -78,8 +78,8 @@ pub fn pipe_output(
oup: &mut (dyn Write + Send),
exe_name: &str,
help: &str,
cp: fn(line_prefix: &str, &mut dyn Read, &mut dyn Write) -> Fallible<()>,
) -> Fallible<()> {
cp: fn(line_prefix: &str, &mut dyn Read, &mut dyn Write) -> Result<()>,
) -> Result<()> {
let mut cmd = cmd
.stdin(Stdio::piped())
.stdout(Stdio::piped())
@ -90,7 +90,7 @@ pub fn pipe_output(
// TODO: how to handle this copying better?
// do we really need threads for this?
crossbeam::scope(|s| -> Fallible<()> {
crossbeam::scope(|s| -> Result<()> {
s.spawn(|_| cp(line_prefix, &mut stdo, oup).unwrap()); // errors?
std::io::copy(inp, &mut stdi)?;
drop(stdi); // NEEDED! otherwise deadlock
@ -109,7 +109,7 @@ impl<T> FileAdapter for T
where
T: SpawningFileAdapter,
{
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
filepath_hint,
mut inp,

4
src/adapters/sqlite.rs
Unescape Escape View File

@ -1,5 +1,5 @@
use super::*;
use failure::*;
use anyhow::Result;
use lazy_static::lazy_static;
use log::*;
use rusqlite::types::ValueRef;
@ -58,7 +58,7 @@ fn format_blob(b: ValueRef) -> String {
}
impl FileAdapter for SqliteAdapter {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
is_real_file,
filepath_hint,

4
src/adapters/tar.rs
Unescape Escape View File

@ -1,7 +1,7 @@
use super::*;
use crate::preproc::rga_preproc;
use ::tar::EntryType::Regular;
use failure::*;
use anyhow::*;
use lazy_static::lazy_static;
use log::*;
use std::path::PathBuf;
@ -36,7 +36,7 @@ impl GetMetadata for TarAdapter {
}
impl FileAdapter for TarAdapter {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
filepath_hint,
mut inp,

4
src/adapters/zip.rs
Unescape Escape View File

@ -1,7 +1,7 @@
use super::*;
use crate::preproc::rga_preproc;
use ::zip::read::ZipFile;
use failure::*;
use anyhow::*;
use lazy_static::lazy_static;
use log::*;
@ -47,7 +47,7 @@ fn is_dir(f: &ZipFile) -> bool {
}
impl FileAdapter for ZipAdapter {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Fallible<()> {
fn adapt(&self, ai: AdaptInfo, _detection_reason: &SlowMatcher) -> Result<()> {
let AdaptInfo {
filepath_hint,
mut inp,

47
src/args.rs
Unescape Escape View File

@ -1,16 +1,42 @@
use failure::Fallible;
use anyhow::*;
use log::*;
use serde::{Deserialize, Serialize};
use std::ffi::OsString;
use std::iter::IntoIterator;
use std::{iter::IntoIterator, str::FromStr};
use structopt::StructOpt;
#[derive(Debug, Deserialize, Serialize)]
struct ReadableBytesCount(i64);
fn parse_readable_bytes_str(s: &str) -> Result<i64, Error> {
let suffix = s.chars().last();
if let Some(suffix) = suffix {
match suffix {
'k' | 'M' | 'G' => i64::from_str(s.trim_end_matches(suffix))
.with_context(|| format!("Could not parse int"))
.map(|e| {
e * match suffix {
'k' => 1000,
'M' => 1000_000,
'G' => 1000_000_000,
_ => panic!("impossible"),
}
}),
_ => i64::from_str(s).with_context(|| format!("Could not parse int")),
}
} else {
Err(format_err!("empty byte input"))
}
}
fn is_default<T: Default + PartialEq>(t: &T) -> bool {
t == &T::default()
}
// ugly, but serde and structopt use different methods to define defaults
// ugly, but serde and structopt use different methods to define defaults, so need to declare defaults twice
macro_rules! set_default {
($name:ident, $val:expr, $type:ty) => {
paste::item! {
@ -25,7 +51,7 @@ macro_rules! set_default {
}
set_default!(cache_compression_level, 12, u32);
set_default!(cache_max_blob_len, 2000000, u32);
set_default!(cache_max_blob_len, 2000000, i64);
set_default!(max_archive_recursion, 4, i32);
#[derive(StructOpt, Debug, Deserialize, Serialize)]
@ -83,12 +109,13 @@ pub struct RgaArgs {
long = "--rga-cache-max-blob-len",
default_value = "2000000",
hidden_short_help = true,
require_equals = true
require_equals = true,
parse(try_from_str = parse_readable_bytes_str)
)]
/// Max compressed size to cache
///
/// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time.
pub cache_max_blob_len: u32,
/// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time. Allowed suffixes: k M G
pub cache_max_blob_len: i64,
#[serde(
default = "def_cache_compression_level",
@ -102,6 +129,8 @@ pub struct RgaArgs {
help = ""
)]
/// ZSTD compression level to apply to adapter outputs before storing in cache db
///
/// Ranges from 1 - 22
pub cache_compression_level: u32,
#[serde(
@ -133,7 +162,7 @@ pub struct RgaArgs {
static RGA_CONFIG: &str = "RGA_CONFIG";
pub fn parse_args<I>(args: I) -> Fallible<RgaArgs>
pub fn parse_args<I>(args: I) -> Result<RgaArgs>
where
I: IntoIterator,
I::Item: Into<OsString> + Clone,
@ -158,7 +187,7 @@ where
}
/// Split arguments into the ones we care about and the ones rg cares about
pub fn split_args() -> Fallible<(RgaArgs, Vec<OsString>)> {
pub fn split_args() -> Result<(RgaArgs, Vec<OsString>)> {
let mut app = RgaArgs::clap();
app.p.create_help_and_version();

2
src/bin/rga-preproc.rs
Unescape Escape View File

@ -4,7 +4,7 @@ use ripgrep_all as rga;
use std::fs::File;
fn main() -> Result<(), exitfailure::ExitFailure> {
fn main() -> anyhow::Result<()> {
env_logger::init();
let mut arg_arr: Vec<std::ffi::OsString> = std::env::args_os().collect();
let last = arg_arr.pop().expect("No filename specified");

6
src/bin/rga.rs
Unescape Escape View File

@ -1,4 +1,4 @@
use failure::Fallible;
use anyhow::Result;
use log::*;
use rga::adapters::spawning::map_exe_error;
use rga::adapters::*;
@ -9,7 +9,7 @@ use structopt::StructOpt;
use std::process::Command;
fn main() -> Result<(), exitfailure::ExitFailure> {
fn main() -> anyhow::Result<()> {
env_logger::init();
let (args, passthrough_args) = split_args()?;
@ -113,7 +113,7 @@ fn main() -> Result<(), exitfailure::ExitFailure> {
}
/// add the directory that contains `rga` to PATH, so rga-preproc can find pandoc etc (if we are on Windows where we include dependent binaries)
fn add_exe_to_path() -> Fallible<()> {
fn add_exe_to_path() -> Result<()> {
use std::env;
let mut exe = env::current_exe().expect("Could not get executable location");
// let preproc_exe = exe.with_file_name("rga-preproc");

8
src/caching_writer.rs
Unescape Escape View File

@ -1,4 +1,4 @@
use failure::Fallible;
use anyhow::Result;
use log::*;
use std::io::Write;
@ -13,11 +13,7 @@ pub struct CachingWriter<W: Write> {
out: W,
}
impl<W: Write> CachingWriter<W> {
pub fn new(
out: W,
max_cache_size: usize,
compression_level: i32,
) -> Fallible<CachingWriter<W>> {
pub fn new(out: W, max_cache_size: usize, compression_level: i32) -> Result<CachingWriter<W>> {
Ok(CachingWriter {
out,
max_cache_size,

4
src/matching.rs
Unescape Escape View File

@ -3,7 +3,7 @@
*/
use crate::adapters::*;
use failure::*;
use anyhow::*;
use regex::{Regex, RegexSet};
@ -49,7 +49,7 @@ pub fn extension_to_regex(extension: &str) -> Regex {
pub fn adapter_matcher<T: AsRef<str>>(
adapter_names: &[T],
slow: bool,
) -> Fallible<impl Fn(FileMeta) -> Option<(Rc<dyn FileAdapter>, SlowMatcher)>> {
) -> Result<impl Fn(FileMeta) -> Option<(Rc<dyn FileAdapter>, SlowMatcher)>> {
let adapters = get_adapters_filtered(adapter_names)?;
// need order later
let adapter_names: Vec<String> = adapters.iter().map(|e| e.metadata().name.clone()).collect();

8
src/preproc.rs
Unescape Escape View File

@ -2,7 +2,7 @@ use crate::adapters::*;
use crate::args::RgaArgs;
use crate::matching::*;
use crate::CachingWriter;
use failure::*;
use anyhow::*;
use log::*;
use path_clean::PathClean;
use std::convert::TryInto;
@ -22,7 +22,7 @@ pub struct PreprocConfig<'a> {
* If a cache is passed, read/write to it.
*
*/
pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> {
pub fn rga_preproc(ai: AdaptInfo) -> Result<()> {
let AdaptInfo {
filepath_hint,
is_real_file,
@ -98,7 +98,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> {
cache.write().unwrap().get_or_run(
&db_name,
&cache_key,
Box::new(|| -> Fallible<Option<Vec<u8>>> {
Box::new(|| -> Result<Option<Vec<u8>>> {
// wrapping BufWriter here gives ~10% perf boost
let mut compbuf = BufWriter::new(CachingWriter::new(
oup,
@ -120,7 +120,7 @@ pub fn rga_preproc(ai: AdaptInfo) -> Fallible<()> {
)?;
let compressed = compbuf
.into_inner()
.map_err(|_| "could not finish zstd")
.map_err(|_| "could not finish zstd") // can't use with_context here
.unwrap()
.finish()?;
if let Some(cached) = compressed {

56
src/preproc_cache.rs
Unescape Escape View File

@ -1,8 +1,11 @@
use failure::{format_err, Fallible};
use anyhow::{format_err, Context, Result};
use log::*;
use std::sync::{Arc, RwLock};
use std::{
fmt::Display,
sync::{Arc, RwLock},
};
pub fn open() -> Fallible<Arc<RwLock<dyn PreprocCache>>> {
pub fn open() -> Result<Arc<RwLock<dyn PreprocCache>>> {
Ok(Arc::new(RwLock::new(LmdbCache::open()?)))
}
pub trait PreprocCache {
@ -11,13 +14,13 @@ pub trait PreprocCache {
&mut self,
db_name: &str,
key: &[u8],
runner: Box<dyn FnOnce() -> Fallible<Option<Vec<u8>>> + 'a>,
callback: Box<dyn FnOnce(&[u8]) -> Fallible<()> + 'a>,
) -> Fallible<()>;
runner: Box<dyn FnOnce() -> Result<Option<Vec<u8>>> + 'a>,
callback: Box<dyn FnOnce(&[u8]) -> Result<()> + 'a>,
) -> Result<()>;
}
/// opens a LMDB cache
fn open_cache_db() -> Fallible<std::sync::Arc<std::sync::RwLock<rkv::Rkv>>> {
fn open_cache_db() -> Result<std::sync::Arc<std::sync::RwLock<rkv::Rkv>>> {
let app_cache = cachedir::CacheDirConfig::new("rga").get_cache_dir()?;
let db_arc = rkv::Manager::singleton()
@ -46,30 +49,42 @@ pub struct LmdbCache {
}
impl LmdbCache {
pub fn open() -> Fallible<LmdbCache> {
pub fn open() -> Result<LmdbCache> {
Ok(LmdbCache {
db_arc: open_cache_db()?,
})
}
}
#[derive(Debug)]
struct RkvErrWrap(rkv::StoreError);
impl Display for RkvErrWrap {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.fmt(f)
}
}
impl std::error::Error for RkvErrWrap {}
impl PreprocCache for LmdbCache {
// possible without second lambda?
fn get_or_run<'a>(
&mut self,
db_name: &str,
key: &[u8],
runner: Box<dyn FnOnce() -> Fallible<Option<Vec<u8>>> + 'a>,
callback: Box<dyn FnOnce(&[u8]) -> Fallible<()> + 'a>,
) -> Fallible<()> {
runner: Box<dyn FnOnce() -> Result<Option<Vec<u8>>> + 'a>,
callback: Box<dyn FnOnce(&[u8]) -> Result<()> + 'a>,
) -> Result<()> {
let db_env = self.db_arc.read().unwrap();
let db = db_env
.open_single(db_name, rkv::store::Options::create())
.map_err(|p| format_err!("could not open db store: {:?}", p))?;
.map_err(RkvErrWrap)
.with_context(|| format_err!("could not open cache db store"))?;
let reader = db_env.read().expect("could not get reader");
let cached = db
.get(&reader, &key)
.map_err(|p| format_err!("could not read from db: {:?}", p))?;
.map_err(RkvErrWrap)
.with_context(|| format_err!("could not read from db"))?;
match cached {
Some(rkv::Value::Blob(cached)) => {
@ -81,12 +96,17 @@ impl PreprocCache for LmdbCache {
debug!("did not get cached");
drop(reader);
if let Some(got) = runner()? {
let mut writer = db_env.write().map_err(|p| {
format_err!("could not open write handle to cache: {:?}", p)
})?;
let mut writer = db_env
.write()
.map_err(RkvErrWrap)
.with_context(|| format_err!("could not open write handle to cache"))?;
db.put(&mut writer, &key, &rkv::Value::Blob(&got))
.map_err(|p| format_err!("could not write to cache: {:?}", p))?;
writer.commit()?;
.map_err(RkvErrWrap)
.with_context(|| format_err!("could not write to cache"))?;
writer
.commit()
.map_err(RkvErrWrap)
.with_context(|| format!("could not write cache"))?;
}
}
};

Write Preview
Loading…
Cancel
Save