diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..d6dce30 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,27 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +**Describe the bug** + + +**To Reproduce** + +Attach example file: + +Run command: + +**Output** + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Operating System and Version** + + +**Output of `rga --version`** diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76c3208..07f646f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,75 +1,25 @@ # Based on https://github.com/actions-rs/meta/blob/master/recipes/quickstart.md # -# While our "example" application has the platform-specific code, -# for simplicity we are compiling and testing everything on the Ubuntu environment only. -# For multi-OS testing see the `cross.yml` workflow. +# While our "example" application has platform-specific code, +# for simplicity we are compiling and testing everything in a nix-on-Linux environment only. on: [push, pull_request] name: ci jobs: - check: - name: Check + nix-flake-check: + name: nix flake check runs-on: ubuntu-latest steps: - name: Checkout sources - uses: actions/checkout@v2 + uses: actions/checkout@v3 - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true + - name: Install nix + uses: cachix/install-nix-action@v21 - - name: Run cargo check - uses: actions-rs/cargo@v1 - with: - command: check + - name: Ensure the build succeeds + run: nix build - test: - name: Test Suite - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - - name: Run cargo test - uses: actions-rs/cargo@v1 - with: - command: test - - lints: - name: Lints - runs-on: ubuntu-latest - steps: - - name: Checkout sources - uses: actions/checkout@v2 - - - name: Install stable toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - components: rustfmt, clippy - - - name: Run cargo fmt - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - - - name: Run cargo clippy - uses: actions-rs/cargo@v1 - with: - command: clippy - args: -- -D warnings + - name: Run `nix flake check` to run formatters, linters, and tests + run: nix flake check --print-build-logs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0ad1c1a..610c391 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,14 +18,14 @@ on: # branches: # - ag/release tags: - - 'v[0-9]+.[0-9]+.[0-9]+' + - "v[0-9]+.[0-9]+.[0-9]+*" jobs: create-release: name: create-release runs-on: ubuntu-latest # env: - # Set to force version number, e.g., when no tag exists. - # RG_VERSION: TEST-0.0.0 + # Set to force version number, e.g., when no tag exists. + # RG_VERSION: TEST-0.0.0 steps: - name: Create artifacts directory run: mkdir artifacts @@ -62,7 +62,7 @@ jobs: build-release: name: build-release - needs: ['create-release'] + needs: ["create-release"] runs-on: ${{ matrix.os }} env: # For some builds, we use cross to test on 32-bit and big-endian @@ -78,124 +78,124 @@ jobs: matrix: build: [linux, linux-arm, macos, win-msvc] include: - - build: linux - os: ubuntu-18.04 - rust: nightly - target: x86_64-unknown-linux-musl - - build: linux-arm - os: ubuntu-18.04 - rust: nightly - target: arm-unknown-linux-gnueabihf - - build: macos - os: macos-latest - rust: nightly - target: x86_64-apple-darwin - - build: win-msvc - os: windows-2019 - rust: nightly - target: x86_64-pc-windows-msvc + - build: linux + os: ubuntu-22.04 + rust: nightly + target: x86_64-unknown-linux-musl + - build: linux-arm + os: ubuntu-22.04 + rust: nightly + target: arm-unknown-linux-gnueabihf + - build: macos + os: macos-latest + rust: nightly + target: x86_64-apple-darwin + - build: win-msvc + os: windows-2019 + rust: nightly + target: x86_64-pc-windows-msvc #- build: win-gnu # os: windows-2019 # rust: nightly-x86_64-gnu # target: x86_64-pc-windows-gnu steps: - - name: Checkout repository - uses: actions/checkout@v2 - with: - fetch-depth: 1 - - - name: Install packages (Ubuntu) - if: matrix.os == 'ubuntu-18.04' - run: | - ci/ubuntu-install-packages - - - name: Install packages (macOS) - if: matrix.os == 'macos-latest' - run: | - ci/macos-install-packages - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: ${{ matrix.rust }} - profile: minimal - override: true - target: ${{ matrix.target }} - - - name: Use Cross - # if: matrix.os != 'windows-2019' - run: | - cargo install cross - echo "CARGO=cross" >> $GITHUB_ENV - echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV - echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV - - - name: Show command used for Cargo - run: | - echo "cargo command is: ${{ env.CARGO }}" - echo "target flag is: ${{ env.TARGET_FLAGS }}" - echo "target dir is: ${{ env.TARGET_DIR }}" - - - name: Get release download URL - uses: actions/download-artifact@v1 - with: - name: artifacts - path: artifacts - - - name: Set release upload URL and release version - shell: bash - run: | - echo "RELEASE_UPLOAD_URL=$(cat artifacts/release-upload-url)" >> $GITHUB_ENV - echo "release upload url: $RELEASE_UPLOAD_URL" - echo "RELEASE_VERSION=$(cat artifacts/release-version)" >> $GITHUB_ENV - echo "release version: $RELEASE_VERSION" - - - name: Build release binary - run: ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }} - - - name: Strip release binary (linux and macos) - if: matrix.build == 'linux' || matrix.build == 'macos' - run: | - strip "target/${{ matrix.target }}/release/rga" "target/${{ matrix.target }}/release/rga-preproc" - - - name: Strip release binary (arm) - if: matrix.build == 'linux-arm' - run: | - docker run --rm -v \ - "$PWD/target:/target:Z" \ - rustembedded/cross:arm-unknown-linux-gnueabihf \ - arm-linux-gnueabihf-strip \ - /target/arm-unknown-linux-gnueabihf/release/rga \ - /target/arm-unknown-linux-gnueabihf/release/rga-preproc - - - name: Build archive - shell: bash - run: | - staging="ripgrep_all-${{ env.RELEASE_VERSION }}-${{ matrix.target }}" - mkdir -p "$staging"/doc - - cp {README.md,LICENSE.md} "$staging/" - cp CHANGELOG.md "$staging/doc/" - - if [ "${{ matrix.os }}" = "windows-2019" ]; then - cp "target/${{ matrix.target }}/release/rga.exe" "$staging/" - cp "target/${{ matrix.target }}/release/rga-preproc.exe" "$staging/" - 7z a "$staging.zip" "$staging" - echo "ASSET=$staging.zip" >> $GITHUB_ENV - else - cp "target/${{ matrix.target }}/release/rga" "$staging/" - cp "target/${{ matrix.target }}/release/rga-preproc" "$staging/" - tar czf "$staging.tar.gz" "$staging" - echo "ASSET=$staging.tar.gz" >> $GITHUB_ENV - fi - - - name: Upload release archive - uses: actions/upload-release-asset@v1.0.1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ env.RELEASE_UPLOAD_URL }} - asset_path: ${{ env.ASSET }} - asset_name: ${{ env.ASSET }} - asset_content_type: application/octet-stream + - name: Checkout repository + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Install packages (Ubuntu) + if: matrix.os == 'ubuntu-22.04' + run: | + ci/ubuntu-install-packages + + - name: Install packages (macOS) + if: matrix.os == 'macos-latest' + run: | + ci/macos-install-packages + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: ${{ matrix.rust }} + profile: minimal + override: true + target: ${{ matrix.target }} + + - name: Use Cross + shell: bash + run: | + cargo install cross + echo "CARGO=cross" >> $GITHUB_ENV + echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV + echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV + + - name: Show command used for Cargo + run: | + echo "cargo command is: ${{ env.CARGO }}" + echo "target flag is: ${{ env.TARGET_FLAGS }}" + echo "target dir is: ${{ env.TARGET_DIR }}" + + - name: Get release download URL + uses: actions/download-artifact@v1 + with: + name: artifacts + path: artifacts + + - name: Set release upload URL and release version + shell: bash + run: | + echo "RELEASE_UPLOAD_URL=$(cat artifacts/release-upload-url)" >> $GITHUB_ENV + echo "release upload url: $RELEASE_UPLOAD_URL" + echo "RELEASE_VERSION=$(cat artifacts/release-version)" >> $GITHUB_ENV + echo "release version: $RELEASE_VERSION" + + - name: Build release binary + run: ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }} + + - name: Strip release binary (linux and macos) + if: matrix.build == 'linux' || matrix.build == 'macos' + run: | + strip "target/${{ matrix.target }}/release/rga" "target/${{ matrix.target }}/release/rga-preproc" + + - name: Strip release binary (arm) + if: matrix.build == 'linux-arm' + run: | + docker run --rm -v \ + "$PWD/target:/target:Z" \ + rustembedded/cross:arm-unknown-linux-gnueabihf \ + arm-linux-gnueabihf-strip \ + /target/arm-unknown-linux-gnueabihf/release/rga \ + /target/arm-unknown-linux-gnueabihf/release/rga-preproc + + - name: Build archive + shell: bash + run: | + staging="ripgrep_all-${{ env.RELEASE_VERSION }}-${{ matrix.target }}" + mkdir -p "$staging"/doc + + cp {README.md,LICENSE.md} "$staging/" + cp CHANGELOG.md "$staging/doc/" + + if [ "${{ matrix.os }}" = "windows-2019" ]; then + cp "target/${{ matrix.target }}/release/rga.exe" "$staging/" + cp "target/${{ matrix.target }}/release/rga-preproc.exe" "$staging/" + 7z a "$staging.zip" "$staging" + echo "ASSET=$staging.zip" >> $GITHUB_ENV + else + cp "target/${{ matrix.target }}/release/rga" "$staging/" + cp "target/${{ matrix.target }}/release/rga-preproc" "$staging/" + tar czf "$staging.tar.gz" "$staging" + echo "ASSET=$staging.tar.gz" >> $GITHUB_ENV + fi + + - name: Upload release archive + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ env.RELEASE_UPLOAD_URL }} + asset_path: ${{ env.ASSET }} + asset_name: ${{ env.ASSET }} + asset_content_type: application/octet-stream diff --git a/Cargo.lock b/Cargo.lock index 59650cc..618d268 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -19,11 +28,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" -version = "0.7.20" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" +checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" dependencies = [ "memchr 2.5.0", ] @@ -43,6 +63,18 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + [[package]] name = "android_system_properties" version = "0.1.5" @@ -62,104 +94,156 @@ dependencies = [ ] [[package]] -name = "anyhow" -version = "1.0.69" +name = "anstream" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] [[package]] -name = "arrayref" -version = "0.3.6" +name = "anstyle" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" +dependencies = [ + "backtrace", +] [[package]] name = "async-compression" version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +dependencies = [ + "bzip2", + "flate2", + "futures-core", + "memchr 2.5.0", + "pin-project-lite", + "tokio", + "xz2", + "zstd 0.11.2+zstd.1.5.2", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "async-compression" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" dependencies = [ "brotli", - "bytes 0.5.6", "bzip2", "flate2", "futures-core", "futures-io", "memchr 2.5.0", - "pin-project-lite 0.2.9", - "tokio 0.2.25", - "tokio 0.3.7", - "tokio 1.25.0", + "pin-project-lite", + "tokio", "xz2", - "zstd", - "zstd-safe", + "zstd 0.12.4", + "zstd-safe 6.0.6", ] [[package]] name = "async-recursion" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b015a331cc64ebd1774ba119538573603427eaace0a1950c423ab971f903796" +checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] name = "async-stream" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" dependencies = [ "async-stream-impl", "futures-core", - "pin-project-lite 0.2.9", + "pin-project-lite", ] [[package]] name = "async-stream-impl" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09" dependencies = [ "proc-macro2", "quote", - "syn", -] - -[[package]] -name = "async_io_utilities" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b20cffc5590f4bf33f05f97a3ea587feba9c50d20325b401daa096b92ff7da0" -dependencies = [ - "tokio 1.25.0", + "syn 2.0.28", ] [[package]] name = "async_zip" -version = "0.0.9" +version = "0.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a36d43bdefc7215b2b3a97edd03b1553b7969ad76551025eedd3b913c645f6e" +checksum = "b2105142db9c6203b9dadc83b0553394589a6cb31b1449a3b46b42f47c3434d0" dependencies = [ - "async-compression", - "async_io_utilities", + "async-compression 0.3.15", "chrono", "crc32fast", + "log", + "pin-project", "thiserror", - "tokio 1.25.0", + "tokio", ] [[package]] @@ -179,6 +263,21 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4319208da049c43661739c5fade2ba182f09d1dc2299b32298d3a31692b17e12" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.13.1" @@ -206,6 +305,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" + [[package]] name = "brotli" version = "3.3.4" @@ -229,9 +334,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" +checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" [[package]] name = "bytecount" @@ -239,18 +344,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c" -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - [[package]] name = "bytes" version = "1.4.0" @@ -311,13 +404,13 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.23" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" dependencies = [ + "android-tzdata", "iana-time-zone", "js-sys", - "num-integer", "num-traits", "time", "wasm-bindgen", @@ -341,36 +434,37 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.6" +version = "4.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" +checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d" dependencies = [ - "bitflags 1.3.2", + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1" +dependencies = [ + "anstream", + "anstyle", "clap_lex", - "is-terminal", "strsim 0.10.0", - "termcolor", "terminal_size", ] [[package]] name = "clap_lex" -version = "0.3.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" -dependencies = [ - "os_str_bytes", -] +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" [[package]] -name = "codespan-reporting" -version = "0.11.1" +name = "colorchoice" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" -dependencies = [ - "termcolor", - "unicode-width", -] +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "convert_case" @@ -380,9 +474,9 @@ checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "crc32fast" @@ -409,9 +503,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -419,9 +513,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if", "crossbeam-epoch", @@ -430,9 +524,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7" dependencies = [ "autocfg", "cfg-if", @@ -453,72 +547,28 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" dependencies = [ "cfg-if", ] [[package]] name = "ctor" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" -dependencies = [ - "quote", - "syn", -] - -[[package]] -name = "cxx" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" -dependencies = [ - "cc", - "cxxbridge-flags", - "cxxbridge-macro", - "link-cplusplus", -] - -[[package]] -name = "cxx-build" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" -dependencies = [ - "cc", - "codespan-reporting", - "once_cell", - "proc-macro2", - "quote", - "scratch", - "syn", -] - -[[package]] -name = "cxxbridge-flags" -version = "1.0.91" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" - -[[package]] -name = "cxxbridge-macro" -version = "1.0.91" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" +checksum = "1f34ba9a9bcb8645379e9de8cb3ecfcf4d1c85ba66d90deb3259206fa5aa193b" dependencies = [ - "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] name = "data-encoding" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23d8666cb01533c39dde32bcbab8e227b4ed6679b2c925eba05feabea39508fb" +checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" [[package]] name = "derive_more" @@ -530,7 +580,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 1.0.109", ] [[package]] @@ -578,14 +628,14 @@ checksum = "558e40ea573c374cf53507fd240b7ee2f5477df7cfebdb97323ec61c719399c5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "dyn-clone" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" [[package]] name = "encoding_rs" @@ -607,12 +657,12 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" dependencies = [ - "atty", "humantime", + "is-terminal", "log", "regex", "termcolor", @@ -620,13 +670,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys", ] [[package]] @@ -653,23 +703,20 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "filetime" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" +checksum = "5cbc844cecaee9d4443931972e1289c8ff485cb4cc2767cb03ca139ed6885153" dependencies = [ "cfg-if", "libc", - "redox_syscall", - "windows-sys 0.45.0", + "redox_syscall 0.2.16", + "windows-sys", ] [[package]] @@ -680,9 +727,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743" dependencies = [ "crc32fast", "miniz_oxide", @@ -694,60 +741,51 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "form_urlencoded" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" -dependencies = [ - "percent-encoding", -] - [[package]] name = "futures-core" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" [[package]] name = "futures-io" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" [[package]] name = "futures-macro" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] name = "futures-sink" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" [[package]] name = "futures-task" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" [[package]] name = "futures-util" -version = "0.3.26" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", "futures-macro", "futures-task", - "pin-project-lite 0.2.9", + "pin-project-lite", "pin-utils", "slab", ] @@ -772,15 +810,21 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", "wasi 0.11.0+wasi-snapshot-preview1", ] +[[package]] +name = "gimli" +version = "0.27.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e" + [[package]] name = "glob" version = "0.3.1" @@ -793,16 +837,26 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", + "ahash 0.7.6", +] + +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +dependencies = [ + "ahash 0.8.3", + "allocator-api2", ] [[package]] name = "hashlink" -version = "0.8.1" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" +checksum = "312f66718a2d7789ffef4f4b7b213138ed9f1eb3aa1d0d82fc99f88fb3ffd26f" dependencies = [ - "hashbrown", + "hashbrown 0.14.0", ] [[package]] @@ -825,18 +879,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "humantime" @@ -846,106 +891,80 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "iana-time-zone" -version = "0.1.53" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows", ] [[package]] name = "iana-time-zone-haiku" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0703ae284fc167426161c2e3f1da3ea71d94b21bedbcc9494e92b28e334e3dca" -dependencies = [ - "cxx", - "cxx-build", -] - -[[package]] -name = "id-arena" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" - -[[package]] -name = "idna" -version = "0.3.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "cc", ] [[package]] name = "indexmap" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885e79c1fc4b10f0e172c475f458b7f7b93061064d98c3293e98c5ba0c8b399" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", "serde", ] -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "io-lifetimes" -version = "1.0.5" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ + "hermit-abi 0.3.2", "libc", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] name = "is-terminal" -version = "0.4.3" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ - "hermit-abi 0.3.1", - "io-lifetimes", - "rustix", - "windows-sys 0.45.0", + "hermit-abi 0.3.2", + "rustix 0.38.4", + "windows-sys", ] [[package]] name = "itoa" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.61" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ "wasm-bindgen", ] @@ -964,64 +983,38 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.139" +version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] name = "libsqlite3-sys" -version = "0.25.2" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" dependencies = [ "cc", "pkg-config", "vcpkg", ] -[[package]] -name = "link-cplusplus" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecd207c9c713c34f95a097a5b029ac2ce6010530c7b49d7fea24d977dede04f5" -dependencies = [ - "cc", -] - [[package]] name = "linux-raw-sys" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" - -[[package]] -name = "lmdb-rkv" -version = "0.14.0" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447a296f7aca299cfbb50f4e4f3d49451549af655fb7215d7f8c0c3d64bad42b" -dependencies = [ - "bitflags 1.3.2", - "byteorder", - "libc", - "lmdb-rkv-sys", -] +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] -name = "lmdb-rkv-sys" -version = "0.11.2" +name = "linux-raw-sys" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" -dependencies = [ - "cc", - "libc", - "pkg-config", -] +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -1029,12 +1022,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" [[package]] name = "lzma-sys" @@ -1090,18 +1080,18 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memoffset" -version = "0.7.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime2ext" @@ -1117,23 +1107,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -1212,52 +1201,37 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", ] [[package]] name = "num_cpus" -version = "1.15.0" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.2", "libc", ] [[package]] -name = "once_cell" -version = "1.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" - -[[package]] -name = "ordered-float" -version = "3.4.0" +name = "object" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d84eb1409416d254e4a9c8fa56cc24701755025b458f0fcd8e59e1f5f40c23bf" +checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" dependencies = [ - "num-traits", + "memchr 2.5.0", ] [[package]] -name = "os_str_bytes" -version = "6.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee" - -[[package]] -name = "output_vt100" -version = "0.1.3" +name = "once_cell" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628223faebab4e3e40667ee0b2336d34a5b960ff60ea743ddfdbcf7770bcfb66" -dependencies = [ - "winapi", -] +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "owning_ref" @@ -1280,34 +1254,28 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "paste" -version = "1.0.11" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "path-clean" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecba01bf2678719532c5e3059e0b5f0811273d94b397088b82e3bd0a78c78fdd" - -[[package]] -name = "percent-encoding" -version = "2.2.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef" [[package]] name = "petgraph" @@ -1320,16 +1288,30 @@ dependencies = [ ] [[package]] -name = "pin-project-lite" -version = "0.1.12" +name = "pin-project" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "257b64915a082f7811703966789728173279bdebb956b143dbcd23f6f970a777" +checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "4c40d25201921e5ff0c862a505c6557ea88568a4e3ace775ab55e93f2f4f9d57" [[package]] name = "pin-utils" @@ -1339,9 +1321,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "pretty-bytes" @@ -1355,13 +1337,11 @@ dependencies = [ [[package]] name = "pretty_assertions" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a25e9bcb20aa780fd0bb16b72403a9064d6b3f22f026946029acb941a50af755" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" dependencies = [ - "ctor", "diff", - "output_vt100", "yansi", ] @@ -1374,7 +1354,7 @@ dependencies = [ "proc-macro-error-attr", "proc-macro2", "quote", - "syn", + "syn 1.0.109", "version_check", ] @@ -1391,27 +1371,27 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.23" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" dependencies = [ "proc-macro2", ] [[package]] name = "quoted_printable" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a24039f627d8285853cc90dcddf8c1ebfaa91f834566948872b225b9a28ed1b6" +checksum = "5a3866219251662ec3b26fc217e3e05bf9c4f84325234dfb96bf0bf840889e49" [[package]] name = "redox_syscall" @@ -1422,6 +1402,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -1429,49 +1418,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regex" -version = "1.7.1" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" dependencies = [ "aho-corasick", "memchr 2.5.0", + "regex-automata", "regex-syntax", ] [[package]] -name = "regex-syntax" -version = "0.6.28" +name = "regex-automata" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +checksum = "b7b6d6190b7594385f61bd3911cd1be99dfddcfc365a4160cc2ab5bff4aed294" +dependencies = [ + "aho-corasick", + "memchr 2.5.0", + "regex-syntax", +] [[package]] -name = "remove_dir_all" -version = "0.5.3" +name = "regex-syntax" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "ripgrep_all" -version = "1.0.0-alpha.2" +version = "1.0.0-alpha.5" dependencies = [ "anyhow", - "async-compression", + "async-compression 0.4.1", "async-recursion", "async-stream", "async-trait", "async_zip", "bincode", - "bytes 1.4.0", - "clap 4.1.6", + "bytes", + "clap 4.3.19", "crossbeam", "crossbeam-channel", "ctor", @@ -1495,7 +1487,6 @@ dependencies = [ "pretty-bytes", "pretty_assertions", "regex", - "rkv", "rusqlite", "schemars", "serde", @@ -1503,7 +1494,8 @@ dependencies = [ "size_format", "structopt", "tempfile", - "tokio 1.25.0", + "tokio", + "tokio-rusqlite", "tokio-stream", "tokio-tar", "tokio-test", @@ -1511,36 +1503,13 @@ dependencies = [ "tree_magic_mini", ] -[[package]] -name = "rkv" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6006704273063c72952370ad236b8d58556dcc4f99a95ced4d9ad40f3e80a69" -dependencies = [ - "arrayref", - "bincode", - "bitflags 1.3.2", - "byteorder", - "id-arena", - "lazy_static", - "lmdb-rkv", - "log", - "ordered-float", - "paste", - "serde", - "serde_derive", - "thiserror", - "url", - "uuid", -] - [[package]] name = "rusqlite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01e213bc3ecb39ac32e81e51ebe31fd888a940515173e3a18a35f8c6e896422a" +checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.3.3", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -1548,6 +1517,12 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -1559,29 +1534,42 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.37.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "4d69718bf81c6127a49dc64e44a742e8bb9213c0ff8869a22c308f84c1d4ab06" dependencies = [ "bitflags 1.3.2", "errno", "io-lifetimes", "libc", - "linux-raw-sys", - "windows-sys 0.45.0", + "linux-raw-sys 0.3.8", + "windows-sys", +] + +[[package]] +name = "rustix" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys 0.4.3", + "windows-sys", ] [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "schemars" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a5fb6c61f29e723026dc8e923d94c694313212abbecbbe5f55a7748eec5b307" +checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" dependencies = [ "dyn-clone", "indexmap", @@ -1592,52 +1580,46 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.11" +version = "0.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f188d036977451159430f3b8dc82ec76364a42b7e289c2b18a9a18f4470058e9" +checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn", + "syn 1.0.109", ] [[package]] name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "scratch" -version = "1.0.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "semver" -version = "1.0.16" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" [[package]] name = "serde" -version = "1.0.152" +version = "1.0.179" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "0a5bf42b8d227d4abf38a1ddb08602e229108a517cd4e5bb28f9c7eaafdce5c0" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.179" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "741e124f5485c7e60c03b043f79f320bff3527f4bbf12cf3831750dc46a0ec2c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] @@ -1648,14 +1630,14 @@ checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c" dependencies = [ "itoa", "ryu", @@ -1683,24 +1665,24 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -1745,14 +1727,25 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" dependencies = [ "proc-macro2", "quote", @@ -1761,16 +1754,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.3.0" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" dependencies = [ "cfg-if", "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", + "redox_syscall 0.3.5", + "rustix 0.38.4", + "windows-sys", ] [[package]] @@ -1784,12 +1776,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c9afddd2cec1c0909f06b00ef33f94ab2cc0578c4a610aa208ddfec8aa2b43a" +checksum = "8e6bf6f19e9f8ed8d4048dc22981458ebcf406d67e94cd422e5ecd73d63b3237" dependencies = [ - "rustix", - "windows-sys 0.45.0", + "rustix 0.37.23", + "windows-sys", ] [[package]] @@ -1803,22 +1795,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", ] [[package]] @@ -1832,94 +1824,70 @@ dependencies = [ "winapi", ] -[[package]] -name = "tinyvec" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "tokio" -version = "0.2.25" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6703a273949a90131b290be1fe7b039d0fc884aa1935860dfcbe056f28cd8092" -dependencies = [ - "bytes 0.5.6", - "pin-project-lite 0.1.12", -] - -[[package]] -name = "tokio" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46409491c9375a693ce7032101970a54f8a2010efb77e13f70788f0d84489e39" -dependencies = [ - "autocfg", - "pin-project-lite 0.2.9", -] - -[[package]] -name = "tokio" -version = "1.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", - "bytes 1.4.0", + "backtrace", + "bytes", "libc", - "memchr 2.5.0", "mio", "num_cpus", "parking_lot", - "pin-project-lite 0.2.9", + "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "tokio-macros" -version = "1.8.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", +] + +[[package]] +name = "tokio-rusqlite" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aa66395f5ff117faee90c9458232c936405f9227ad902038000b74b3bc1feac" +dependencies = [ + "crossbeam-channel", + "rusqlite", + "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" dependencies = [ "futures-core", - "pin-project-lite 0.2.9", - "tokio 1.25.0", + "pin-project-lite", + "tokio", "tokio-util", ] [[package]] name = "tokio-tar" -version = "0.3.0" -source = "git+https://github.com/vorot93/tokio-tar#1bd30fbd1a219e8982571da48eb68f34317d1e15" +version = "0.3.1" +source = "git+https://github.com/vorot93/tokio-tar#87338a76092330bc6fe60de95d83eae5597332e1" dependencies = [ "filetime", "futures-core", "libc", - "redox_syscall", - "tokio 1.25.0", + "redox_syscall 0.3.5", + "tokio", "tokio-stream", "xattr", ] @@ -1931,27 +1899,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" dependencies = [ "async-stream", - "bytes 1.4.0", + "bytes", "futures-core", - "tokio 1.25.0", + "tokio", "tokio-stream", ] [[package]] name = "tokio-util" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-core", "futures-io", "futures-sink", "futures-util", - "hashbrown", - "pin-project-lite 0.2.9", + "hashbrown 0.12.3", + "pin-project-lite", "slab", - "tokio 1.25.0", + "tokio", "tracing", ] @@ -1962,15 +1930,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", - "pin-project-lite 0.2.9", + "pin-project-lite", "tracing-core", ] [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" dependencies = [ "once_cell", ] @@ -1995,26 +1963,11 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" -[[package]] -name = "unicode-bidi" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" - [[package]] name = "unicode-ident" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" - -[[package]] -name = "unicode-normalization" -version = "0.1.22" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" -dependencies = [ - "tinyvec", -] +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" [[package]] name = "unicode-segmentation" @@ -2029,21 +1982,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" [[package]] -name = "url" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "uuid" -version = "0.8.2" +name = "utf8parse" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "vcpkg" @@ -2077,9 +2019,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2087,24 +2029,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.28", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2112,22 +2054,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.28", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.84" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "winapi" @@ -2161,34 +2103,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "windows-sys" -version = "0.42.0" +name = "windows" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows-targets", ] [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.1" +version = "0.48.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -2201,51 +2137,51 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_msvc" -version = "0.42.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "xattr" -version = "0.2.3" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +checksum = "f4686009f71ff3e5c4dbcf1a282d0a44db3f021ba69350cd42086b3e5f1c6985" dependencies = [ "libc", ] @@ -2271,7 +2207,16 @@ version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "zstd" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" +dependencies = [ + "zstd-safe 6.0.6", ] [[package]] @@ -2284,11 +2229,21 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "6.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" +dependencies = [ + "libc", + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" +version = "2.0.8+zstd.1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" dependencies = [ "cc", "libc", diff --git a/Cargo.toml b/Cargo.toml index 59eac71..385bddc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [package] authors = ["phiresky "] description = "rga: ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc." -edition = "2018" +edition = "2021" exclude = [ "exampledir/*", ] @@ -11,56 +11,57 @@ license = "AGPL-3.0-or-later" name = "ripgrep_all" readme = "README.md" repository = "https://github.com/phiresky/ripgrep-all" -version = "1.0.0-alpha.2" +version = "1.0.0-alpha.5" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.32" -async-compression = {version = "0.3.15", features = ["all", "all-algorithms", "tokio"]} -async-stream = "0.3.3" -async-trait = "0.1.64" -async_zip = "0.0.9" -bincode = "1.3.1" -bytes = "1.2.1" -clap = {version = "4.0.18", features = ["wrap_help"]} -crossbeam = "0.8.1" -crossbeam-channel = "0.5.1" -derive_more = "0.99.9" +anyhow = {version = "1.0.71", features = ["backtrace"]} +async-compression = { version = "0.4.0", features = ["all", "all-algorithms", "tokio"] } +async-stream = "0.3.5" +async-trait = "0.1.68" +async_zip = {version = "0.0.12", features = ["full"]} +bincode = "1.3.3" +bytes = "1.4.0" +clap = {version = "4.3.0", features = ["wrap_help"]} +crossbeam = "0.8.2" +crossbeam-channel = "0.5.8" +derive_more = "0.99.17" directories-next = "2.0.0" dyn-clonable = "0.9.0" -dyn-clone = "1.0.2" -encoding_rs = "0.8.24" +dyn-clone = "1.0.11" +encoding_rs = "0.8.32" encoding_rs_io = "0.1.7" -env_logger = "0.9.0" -glob = "0.3.0" +env_logger = "0.10.0" +glob = "0.3.1" json_comments = "0.2.1" lazy_static = "1.4.0" -log = "0.4.11" +log = "0.4.17" mailbox = "0.2.0" mailparse = "0.14.0" -memchr = "2.3.3" +memchr = "2.5.0" mime2ext = "0.1.52" -paste = "1.0.0" -path-clean = "0.1.0" +paste = "1.0.12" +path-clean = "1.0.1" pretty-bytes = "0.2.2" -regex = "1.3.9" -rkv = "0.17" -rusqlite = {version = "0.28.0", features = ["vtab", "bundled"]} -schemars = {version = "0.8.0-alpha-4", features = ["preserve_order"]} -serde = {version = "1.0.115", features = ["derive"]} -serde_json = "1.0.57" +regex = "1.8.2" +rusqlite = {version = "0.29.0", features = ["vtab", "bundled"]} +schemars = {version = "0.8.12", features = ["preserve_order"]} +serde = {version = "1.0.163", features = ["derive"]} +serde_json = "1.0.96" size_format = "1.0.2" -structopt = "0.3.17" -tempfile = "3.1.0" -tokio = {version = "1.21.2", features = ["full"]} -tokio-stream = {version = "0.1.11", features = ["io-util", "tokio-util"]} +structopt = "0.3.26" +tempfile = "3.5.0" +tokio = {version = "1.28.1", features = ["full"]} +tokio-rusqlite = "0.4.0" +tokio-stream = {version = "0.1.14", features = ["io-util", "tokio-util"]} tokio-tar = { git = "https://github.com/vorot93/tokio-tar", version = "0.3.0" } -tokio-util = {version = "0.7.4", features = ["io", "full"]} -tree_magic = {package = "tree_magic_mini", version = "3.0.0"} +tokio-util = {version = "0.7.8", features = ["io", "full"]} +tree_magic = {package = "tree_magic_mini", version = "3.0.3"} [dev-dependencies] -async-recursion = "1.0.0" -ctor = "0.1.20" +async-recursion = "1.0.4" +ctor = "0.2.0" pretty_assertions = "1.3.0" +tempfile = "3.5.0" tokio-test = "0.4.2" diff --git a/README.md b/README.md index 929896e..fed1c17 100644 --- a/README.md +++ b/README.md @@ -33,45 +33,7 @@ demo/ ![rga-fzf](doc/rga-fzf.gif) -You can use rga interactively via fzf. Add the following to your ~/.{bash,zsh}rc: - -```bash -rga-fzf() { - RG_PREFIX="rga --files-with-matches" - local file - file="$( - FZF_DEFAULT_COMMAND="$RG_PREFIX '$1'" \ - fzf --sort --preview="[[ ! -z {} ]] && rga --pretty --context 5 {q} {}" \ - --phony -q "$1" \ - --bind "change:reload:$RG_PREFIX {q}" \ - --preview-window="70%:wrap" - )" && - echo "opening $file" && - xdg-open "$file" -} -``` - -And for your `~/.config/fish/config.fish`: -``` -function rga-fzf - set RG_PREFIX 'rga --files-with-matches' - if test (count $argv) -gt 1 - set RG_PREFIX "$RG_PREFIX $argv[1..-2]" - end - set -l file $file - set file ( - FZF_DEFAULT_COMMAND="$RG_PREFIX '$argv[-1]'" \ - fzf --sort \ - --preview='test ! -z {} && \ - rga --pretty --context 5 {q} {}' \ - --phony -q "$argv[-1]" \ - --bind "change:reload:$RG_PREFIX {q}" \ - --preview-window='50%:wrap' - ) && \ - echo "opening $file" && \ - open "$file" -end -``` +See [the wiki](https://github.com/phiresky/ripgrep-all/wiki/fzf-Integration) for instructions of integrating rga with fzf. ## INSTALLATION @@ -86,9 +48,11 @@ Linux x64, macOS and Windows binaries are available [in GitHub Releases][latestr `pacman -S ripgrep-all`. #### Nix + `nix-env -iA nixpkgs.ripgrep-all` #### Debian-based + download the [rga binary][latestrelease] and get the dependencies like this: `apt install ripgrep pandoc poppler-utils ffmpeg` @@ -117,7 +81,7 @@ If you get an error like `VCRUNTIME140.DLL could not be found`, you need to inst To install the dependencies that are each not strictly necessary but very useful: -`brew install pandoc poppler tesseract ffmpeg` +`brew install pandoc poppler ffmpeg` ### Compile from source @@ -131,58 +95,58 @@ rga should compile with stable Rust (v1.36.0+, check with `rustc --version`). To ## Available Adapters +rga works with _adapters_ that adapt various file formats. It comes with a few adapters integrated: + ``` rga --rga-list-adapters ``` +You can also add **custom adapters**. See [the wiki](https://github.com/phiresky/ripgrep-all/wiki) for more information. + Adapters: -- **ffmpeg** - Uses ffmpeg to extract video metadata/chapters and subtitles. - Extensions: `.mkv`, `.mp4`, `.avi` - -* **pandoc** - Uses pandoc to convert binary/unreadable text documents to plain markdown-like text. - Extensions: `.epub`, `.odt`, `.docx`, `.fb2`, `.ipynb` +- **pandoc** + Uses pandoc to convert binary/unreadable text documents to plain markdown-like text + Runs: pandoc --from= --to=plain --wrap=none --markdown-headings=atx + Extensions: .epub, .odt, .docx, .fb2, .ipynb -- **poppler** - Uses pdftotext (from poppler-utils) to extract plain text from PDF files. - Extensions: `.pdf` - Mime Types: `application/pdf` +- **poppler** + Uses pdftotext (from poppler-utils) to extract plain text from PDF files + Runs: pdftotext - - + Extensions: .pdf + Mime Types: application/pdf -- **zip** - Reads a zip file as a stream and recurses down into its contents. - Extensions: `.zip` - Mime Types: `application/zip` +- **postprocpagebreaks** + Adds the page number to each line for an input file that specifies page breaks as ascii page break character. + Mainly to be used internally by the poppler adapter. + Extensions: .asciipagebreaks -- **decompress** - Reads compressed file as a stream and runs a different extractor on the contents. - Extensions: `.tgz`, `.tbz`, `.tbz2`, `.gz`, `.bz2`, `.xz`, `.zst` - Mime Types: `application/gzip`, `application/x-bzip`, `application/x-xz`, `application/zstd` +- **ffmpeg** + Uses ffmpeg to extract video metadata/chapters, subtitles, lyrics, and other metadata + Extensions: .mkv, .mp4, .avi, .mp3, .ogg, .flac, .webm -- **tar** - Reads a tar file as a stream and recurses down into its contents. - Extensions: `.tar` +- **zip** + Reads a zip file as a stream and recurses down into its contents + Extensions: .zip, .jar + Mime Types: application/zip -* **sqlite** - Uses sqlite bindings to convert sqlite databases into a simple plain text format. - Extensions: `.db`, `.db3`, `.sqlite`, `.sqlite3` - Mime Types: `application/x-sqlite3` +- **decompress** + Reads compressed file as a stream and runs a different extractor on the contents. + Extensions: .tgz, .tbz, .tbz2, .gz, .bz2, .xz, .zst + Mime Types: application/gzip, application/x-bzip, application/x-xz, application/zstd -The following adapters are disabled by default, and can be enabled using `--rga-adapters=+pdfpages,tesseract`: +- **tar** + Reads a tar file as a stream and recurses down into its contents + Extensions: .tar -- **pdfpages** - Converts a pdf to its individual pages as png files. Only useful in combination with tesseract. - Extensions: `.pdf` - Mime Types: `application/pdf` +- **sqlite** + Uses sqlite bindings to convert sqlite databases into a simple plain text format + Extensions: .db, .db3, .sqlite, .sqlite3 + Mime Types: application/x-sqlite3 -- **tesseract** - Uses tesseract to run OCR on images to make them searchable. - May need `-j1` to prevent overloading the system. - Make sure you have tesseract installed. - Extensions: `.jpg`, `.png` +The following adapters are disabled by default, and can be enabled using '--rga-adapters=+foo,bar': ## USAGE: @@ -202,6 +166,17 @@ The following adapters are disabled by default, and can be enabled using `--rga- > Detection is only done on the first 8KiB of the file, since we can\'t > always seek on the input (in archives). +**\--rga-no-cache** + +> Disable caching of results +> +> By default, rga caches the extracted text, if it is small enough, to a +> database in \${XDG*CACHE_DIR-\~/.cache}/ripgrep-all on Linux, +> *\~/Library/Caches/ripgrep-all\_ on macOS, or +> C:\\Users\\username\\AppData\\Local\\ripgrep-all on Windows. This way, +> repeated searches on the same set of files will be much faster. If you +> pass this flag, all caching will be disabled. + **-h**, **\--help** > Prints help information @@ -210,15 +185,9 @@ The following adapters are disabled by default, and can be enabled using `--rga- > List all known adapters -**\--rga-no-cache** +**\--rga-print-config-schema** -> Disable caching of results -> -> By default, rga caches the extracted text, if it is small enough, to a -> database in \~/.cache/rga on Linux, _\~/Library/Caches/rga_ on macOS, -> or C:\\Users\\username\\AppData\\Local\\rga on Windows. This way, -> repeated searches on the same set of files will be much faster. If you -> pass this flag, all caching will be disabled. +> Print the JSON Schema of the configuration file **\--rg-help** @@ -242,24 +211,31 @@ The following adapters are disabled by default, and can be enabled using `--rga- > use all default adapters except for bar and baz. \"+bar,baz\" means > use all default adapters and also bar and baz. -**\--rga-cache-compression-level=**\ +**\--rga-cache-compression-level=**\ > ZSTD compression level to apply to adapter outputs before storing in > cache db > > Ranges from 1 - 22 \[default: 12\] -**\--rga-cache-max-blob-len=**\ +**\--rga-config-file=**\ + +**\--rga-max-archive-recursion=**\ + +> Maximum nestedness of archives to recurse into \[default: 4\] + +**\--rga-cache-max-blob-len=**\ > Max compressed size to cache > > Longest byte length (after compression) to store in cache. Longer -> adapter outputs will not be cached and recomputed every time. Allowed -> suffixes: k M G \[default: 2000000\] +> adapter outputs will not be cached and recomputed every time. +> +> Allowed suffixes on command line: k M G \[default: 2000000\] -**\--rga-max-archive-recursion=**\ +**\--rga-cache-path=**\ -> Maximum nestedness of archives to recurse into \[default: 4\] +> Path to store cache db \[default: /home/phire/.cache/ripgrep-all\] **-h** shows a concise overview, **\--help** shows more detail and advanced options. @@ -287,6 +263,7 @@ to debug the adapters. You can use the provided [`flake.nix`](./flake.nix) to setup all build- and run-time dependencies: + 1. Enable [Flakes](https://nixos.wiki/wiki/Flakes) in your Nix configuration. 1. Add [`direnv`](https://direnv.net/) to your profile: `nix profile install nixpkgs#direnv` diff --git a/doc/config.default.jsonc b/doc/config.default.jsonc index 28cd9a3..f45ccb6 100644 --- a/doc/config.default.jsonc +++ b/doc/config.default.jsonc @@ -7,7 +7,7 @@ // https://github.com/phiresky/ripgrep-all/blob/master/doc/config.default.jsonc // The config options are the same as the command line options, - // but with --rga- prefix removed and - replaced with _. + // but with --rga- prefix removed and - and . replaced with _. // e.g. --rga-no-cache becomes `"no_cache": true. // The only exception is the `custom_adapters` option, which can only be set in this file. diff --git a/doc/update-readme.sh b/doc/update-readme.sh index 8b2f0a8..1558eb8 100755 --- a/doc/update-readme.sh +++ b/doc/update-readme.sh @@ -5,7 +5,7 @@ content=$( $(cargo run --bin rga -- --rga-list-adapters) -$(help2man -N "cargo run --bin rga --" | pandoc -f man -t markdown --atx-headers | rg --multiline "## USAGE:(.|\n)*") +$(help2man -N "cargo run --bin rga --" | pandoc -f man -t markdown --markdown-headings=atx | rg --multiline "## USAGE:(.|\n)*") END ) diff --git a/exampledir/exif.png b/exampledir/exif.png new file mode 100644 index 0000000..52dc083 Binary files /dev/null and b/exampledir/exif.png differ diff --git a/flake.lock b/flake.lock index 95606a1..5df8a58 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "advisory-db": { "flake": false, "locked": { - "lastModified": 1670452192, - "narHash": "sha256-f8NIFbqSbCzpssgDUK4srfgKaVaMhDScEptw4uuxGAc=", + "lastModified": 1685821301, + "narHash": "sha256-4XRcnSboLJw1XKjDpg2jBU70jEw/8Bgx4nUmnq3kXbY=", "owner": "rustsec", "repo": "advisory-db", - "rev": "0a2faeb87195392b23333a8097309d29f2c5d31d", + "rev": "af3f3d503f82056785841bee49997bae65eba1c0", "type": "github" }, "original": { @@ -26,11 +26,11 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1670546681, - "narHash": "sha256-S33bhME0zPHPEZyZPCsrdQL/4WW/A020PwN+a3z7Q+I=", + "lastModified": 1684981077, + "narHash": "sha256-68X9cFm0RTZm8u0rXPbeBzOVUH5OoUGAfeHHVoxGd9o=", "owner": "ipetkov", "repo": "crane", - "rev": "63f80ee278897e72a1468090278716b5befa5128", + "rev": "35110cccf28823320f4fd697fcafcb5038683982", "type": "github" }, "original": { @@ -42,11 +42,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1668681692, - "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=", + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", "owner": "edolstra", "repo": "flake-compat", - "rev": "009399224d5e398d03b22badca40a37ac85412a1", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", "type": "github" }, "original": { @@ -58,11 +58,11 @@ "flake-compat_2": { "flake": false, "locked": { - "lastModified": 1668681692, - "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=", + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", "owner": "edolstra", "repo": "flake-compat", - "rev": "009399224d5e398d03b22badca40a37ac85412a1", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", "type": "github" }, "original": { @@ -72,12 +72,15 @@ } }, "flake-utils": { + "inputs": { + "systems": "systems" + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "lastModified": 1681202837, + "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=", "owner": "numtide", "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "rev": "cfacdce06f30d2b68473a46042957675eebb3401", "type": "github" }, "original": { @@ -87,27 +90,15 @@ } }, "flake-utils_2": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" + "inputs": { + "systems": "systems_2" }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_3": { "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "lastModified": 1685518550, + "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=", "owner": "numtide", "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef", "type": "github" }, "original": { @@ -139,48 +130,31 @@ }, "nixpkgs": { "locked": { - "lastModified": 1670525689, - "narHash": "sha256-YIjGzxrRQa5LYO0zlnH/ardcwXsRgsnHe3TkGkvCxbc=", + "lastModified": 1685860998, + "narHash": "sha256-ZexAPe8yvJaLvn5aVgjW0vY41RnmJGbgOdGBJk1yDIE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "f21f11aa2a02cb78651c6d57546c7d7541f9240c", + "rev": "45d47b647d7bbaede5121d731cbee78f6093b6d6", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } }, "nixpkgs-stable": { "locked": { - "lastModified": 1668984258, - "narHash": "sha256-0gDMJ2T3qf58xgcSbYoXiRGUkPWmKyr5C3vcathWhKs=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "cf63ade6f74bbc9d2a017290f1b2e33e8fbfa70a", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixos-22.05", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1668994630, - "narHash": "sha256-1lqx6HLyw6fMNX/hXrrETG1vMvZRGm2XVC9O/Jt0T6c=", + "lastModified": 1678872516, + "narHash": "sha256-/E1YwtMtFAu2KUQKV/1+KFuReYPANM2Rzehk84VxVoc=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "af50806f7c6ab40df3e6b239099e8f8385f6c78b", + "rev": "9b8e5abb18324c7fe9f07cb100c3cd4a29cda8b8", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixos-22.11", "repo": "nixpkgs", "type": "github" } @@ -188,17 +162,21 @@ "pre-commit-hooks": { "inputs": { "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_3", + "flake-utils": [ + "flake-utils" + ], "gitignore": "gitignore", - "nixpkgs": "nixpkgs_2", + "nixpkgs": [ + "nixpkgs" + ], "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1670413394, - "narHash": "sha256-M7sWqrKtOqUv9euX1t3HCxis8cPy9MNiZxQmUf0KF1o=", + "lastModified": 1685361114, + "narHash": "sha256-4RjrlSb+OO+e1nzTExKW58o3WRwVGpXwj97iCta8aj4=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "1303a1a76e9eb074075bfe566518c413f6fc104e", + "rev": "ca2fdbf3edda2a38140184da6381d49f8206eaf4", "type": "github" }, "original": { @@ -229,11 +207,11 @@ ] }, "locked": { - "lastModified": 1670034122, - "narHash": "sha256-EqmuOKucPWtMvCZtHraHr3Q3bgVszq1x2PoZtQkUuEk=", + "lastModified": 1683080331, + "narHash": "sha256-nGDvJ1DAxZIwdn6ww8IFwzoHb2rqBP4wv/65Wt5vflk=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "a0d5773275ecd4f141d792d3a0376277c0fc0b65", + "rev": "d59c3fa0cba8336e115b376c2d9e91053aa59e56", "type": "github" }, "original": { @@ -252,11 +230,11 @@ ] }, "locked": { - "lastModified": 1670552927, - "narHash": "sha256-lCE51eAGrAFS4k9W5aDGFpVtOAwQQ/rFMN80PCDh0vo=", + "lastModified": 1685846256, + "narHash": "sha256-G4aYK4VqlMHImvZ0lUnLHw1A+Cx28T0sBMvAKZBcGpk=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "a0fdafd18c9cf599fde17fbaf07dbb20fa57eecb", + "rev": "1ef3c6de6127a1cba94cc5492cdde52e33d06ea4", "type": "github" }, "original": { @@ -264,6 +242,36 @@ "repo": "rust-overlay", "type": "github" } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index b34a093..088c00c 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,7 @@ "ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc."; inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs"; crane = { url = "github:ipetkov/crane"; @@ -25,7 +25,13 @@ flake = false; }; - pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + pre-commit-hooks = { + url = "github:cachix/pre-commit-hooks.nix"; + inputs = { + nixpkgs.follows = "nixpkgs"; + flake-utils.follows = "flake-utils"; + }; + }; }; outputs = { self, nixpkgs, crane, flake-utils, rust-overlay, advisory-db @@ -36,14 +42,16 @@ inherit system; overlays = [ (import rust-overlay) ]; }; - inherit (pkgs) lib; craneLib = crane.lib.${system}; - src = craneLib.cleanCargoSource ./.; + src = pkgs.lib.cleanSourceWith { + src = craneLib.path ./.; + filter = pkgs.lib.cleanSourceFilter; + }; buildInputs = with pkgs; [ ffmpeg imagemagick pandoc poppler_utils ripgrep tesseract ] - ++ lib.optionals pkgs.stdenv.isDarwin [ + ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [ # Additional darwin specific inputs can be set here pkgs.libiconv ]; @@ -54,10 +62,7 @@ # Build the actual crate itself, reusing the dependency # artifacts from above. - rga = craneLib.buildPackage { - inherit cargoArtifacts src buildInputs; - doCheck = false; - }; + rga = craneLib.buildPackage { inherit cargoArtifacts src buildInputs; }; pre-commit = pre-commit-hooks.lib."${system}".run; in { @@ -97,18 +102,20 @@ hooks = { nixfmt.enable = true; rustfmt.enable = true; - cargo-check.enable = true; + typos = { + enable = true; + types = [ "text" ]; + excludes = [ "exampledir/.*" ]; + }; }; }; - } // lib.optionalAttrs (system == "x86_64-linux") { - # NB: cargo-tarpaulin only supports x86_64 systems - # Check code coverage (note: this will not upload coverage anywhere) - rga-coverage = - craneLib.cargoTarpaulin { inherit cargoArtifacts src; }; }; # `nix build` - packages.default = rga; + packages = { + inherit rga; # `nix build .#rga` + default = rga; # `nix build` + }; # `nix run` apps.default = flake-utils.lib.mkApp { drv = rga; }; diff --git a/src/adapters.rs b/src/adapters.rs index 22df997..9464adf 100644 --- a/src/adapters.rs +++ b/src/adapters.rs @@ -10,6 +10,7 @@ pub mod writing; pub mod zip; use crate::{adapted_iter::AdaptedFilesIterBox, config::RgaConfig, matching::*}; use anyhow::{format_err, Context, Result}; +use async_trait::async_trait; use custom::CustomAdapterConfig; use custom::BUILTIN_SPAWNING_ADAPTERS; use log::*; @@ -77,11 +78,17 @@ impl AdapterMeta { pub trait GetMetadata { fn metadata(&self) -> &AdapterMeta; } + +#[async_trait] pub trait FileAdapter: GetMetadata + Send + Sync { /// adapt a file. /// /// detection_reason is the Matcher that was used to identify this file. Unless --rga-accurate was given, it is always a FastMatcher - fn adapt(&self, a: AdaptInfo, detection_reason: &FileMatcher) -> Result; + async fn adapt( + &self, + a: AdaptInfo, + detection_reason: &FileMatcher, + ) -> Result; } pub struct AdaptInfo { diff --git a/src/adapters/custom.rs b/src/adapters/custom.rs index a35d631..b82c0aa 100644 --- a/src/adapters/custom.rs +++ b/src/adapters/custom.rs @@ -49,8 +49,9 @@ pub struct CustomAdapterConfig { pub args: Vec, /// The output path hint. The placeholders are the same as for `.args` /// - /// If not set, defaults to ${input_virtual_path}.txt + /// If not set, defaults to "${input_virtual_path}.txt" /// + /// Setting this is useful if the output format is not plain text (.txt) but instead some other format that should be passed to another adapter pub output_path_hint: Option, } @@ -128,7 +129,6 @@ lazy_static! { disabled_by_default: None, match_only_by_mime: None, output_path_hint: Some("${input_virtual_path}.txt.asciipagebreaks".into()) - // postprocessors: [{name: "add_page_numbers_by_pagebreaks"}] } ]; } @@ -143,15 +143,13 @@ pub fn map_exe_error(err: std::io::Error, exe_name: &str, help: &str) -> anyhow: } } -fn proc_wait(mut child: Child) -> impl AsyncRead { +fn proc_wait(mut child: Child, context: impl FnOnce() -> String) -> impl AsyncRead { let s = stream! { let res = child.wait().await?; if res.success() { yield std::io::Result::Ok(Bytes::new()); } else { - yield std::io::Result::Err(to_io_err( - format_err!("subprocess failed: {:?}", res), - )); + Err(format_err!("{:?}", res)).with_context(context).map_err(to_io_err)?; } }; StreamReader::new(s) @@ -164,6 +162,7 @@ pub fn pipe_output( exe_name: &str, help: &str, ) -> Result { + let cmd_log = format!("{:?}", cmd); // todo: perf let mut cmd = cmd .stdin(Stdio::piped()) .stdout(Stdio::piped()) @@ -177,10 +176,9 @@ pub fn pipe_output( tokio::io::copy(&mut z, &mut stdi).await?; std::io::Result::Ok(()) }); - - Ok(Box::pin( - stdo.chain(proc_wait(cmd).chain(join_handle_to_stream(join))), - )) + Ok(Box::pin(stdo.chain( + proc_wait(cmd, move || format!("subprocess: {cmd_log}")).chain(join_handle_to_stream(join)), + ))) } pub struct CustomSpawningFileAdapter { @@ -224,8 +222,9 @@ impl CustomSpawningFileAdapter { Ok(command) } } +#[async_trait] impl FileAdapter for CustomSpawningFileAdapter { - fn adapt<'a>( + async fn adapt( &self, ai: AdaptInfo, _detection_reason: &FileMatcher, @@ -314,7 +313,7 @@ mod test { let (a, d) = simple_adapt_info(&filepath, Box::pin(File::open(&filepath).await?)); // let r = adapter.adapt(a, &d)?; - let r = loop_adapt(&adapter, d, a)?; + let r = loop_adapt(&adapter, d, a).await?; let o = adapted_to_vec(r).await?; assert_eq!( String::from_utf8(o)?, @@ -368,7 +367,7 @@ PREFIX:Page 1: Path::new("foo.txt"), Box::pin(Cursor::new(Vec::from(input))), ); - let output = adapter.adapt(a, &d).unwrap(); + let output = adapter.adapt(a, &d).await.unwrap(); let oup = adapted_to_vec(output).await?; println!("output: {}", String::from_utf8_lossy(&oup)); diff --git a/src/adapters/decompress.rs b/src/adapters/decompress.rs index fe5b6c1..da28913 100644 --- a/src/adapters/decompress.rs +++ b/src/adapters/decompress.rs @@ -93,8 +93,13 @@ fn get_inner_filename(filename: &Path) -> PathBuf { filename.with_file_name(format!("{}{}", stem, new_extension)) } +#[async_trait] impl FileAdapter for DecompressAdapter { - fn adapt(&self, ai: AdaptInfo, detection_reason: &FileMatcher) -> Result { + async fn adapt( + &self, + ai: AdaptInfo, + detection_reason: &FileMatcher, + ) -> Result { Ok(one_file(AdaptInfo { filepath_hint: get_inner_filename(&ai.filepath_hint), is_real_file: false, @@ -137,7 +142,7 @@ mod tests { let filepath = test_data_dir().join("hello.gz"); let (a, d) = simple_adapt_info(&filepath, Box::pin(File::open(&filepath).await?)); - let r = adapter.adapt(a, &d)?; + let r = adapter.adapt(a, &d).await?; let o = adapted_to_vec(r).await?; assert_eq!(String::from_utf8(o)?, "hello\n"); Ok(()) @@ -150,7 +155,7 @@ mod tests { let filepath = test_data_dir().join("short.pdf.gz"); let (a, d) = simple_adapt_info(&filepath, Box::pin(File::open(&filepath).await?)); - let r = loop_adapt(&adapter, d, a)?; + let r = loop_adapt(&adapter, d, a).await?; let o = adapted_to_vec(r).await?; assert_eq!( String::from_utf8(o)?, diff --git a/src/adapters/ffmpeg.rs b/src/adapters/ffmpeg.rs index 79ffc1c..32298fe 100644 --- a/src/adapters/ffmpeg.rs +++ b/src/adapters/ffmpeg.rs @@ -14,13 +14,15 @@ use writing::WritingFileAdapter; // maybe todo: read list of extensions from // ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null // but really, the probability of getting useful information from a .flv is low -static EXTENSIONS: &[&str] = &["mkv", "mp4", "avi"]; +static EXTENSIONS: &[&str] = &["mkv", "mp4", "avi", "mp3", "ogg", "flac", "webm"]; lazy_static! { static ref METADATA: AdapterMeta = AdapterMeta { name: "ffmpeg".to_owned(), version: 1, - description: "Uses ffmpeg to extract video metadata/chapters and subtitles".to_owned(), + description: + "Uses ffmpeg to extract video metadata/chapters, subtitles, lyrics, and other metadata" + .to_owned(), recurses: false, fast_matchers: EXTENSIONS .iter() @@ -52,7 +54,7 @@ struct FFprobeOutput { } #[derive(Serialize, Deserialize)] struct FFprobeStream { - codec_type: String, // video,audio,subtitle + index: i32, // stream index } #[async_trait] @@ -78,17 +80,17 @@ impl WritingFileAdapter for FFmpegAdapter { } let inp_fname = filepath_hint; let spawn_fail = |e| map_exe_error(e, "ffprobe", "Make sure you have ffmpeg installed."); - let has_subtitles = { + let subtitle_streams = { let probe = Command::new("ffprobe") .args(vec![ "-v", - "error", + "error", // show all errors "-select_streams", - "s", + "s", // show only subtitle streams "-of", - "json", + "json", // use json as output format "-show_entries", - "stream=codec_type", + "stream=index", // show index of subtitle streams ]) .arg("-i") .arg(&inp_fname) @@ -96,10 +98,14 @@ impl WritingFileAdapter for FFmpegAdapter { .await .map_err(spawn_fail)?; if !probe.status.success() { - return Err(format_err!("ffprobe failed: {:?}", probe.status)); + return Err(format_err!( + "ffprobe failed: {:?}\n{}", + probe.status, + String::from_utf8_lossy(&probe.stderr) + )); } let p: FFprobeOutput = serde_json::from_slice(&probe.stdout)?; - !p.streams.is_empty() + p.streams }; { // extract file metadata (especially chapter names in a greppable format) @@ -124,6 +130,7 @@ impl WritingFileAdapter for FFmpegAdapter { .spawn()?; let mut lines = BufReader::new(probe.stdout.as_mut().unwrap()).lines(); while let Some(line) = lines.next_line().await? { + let line = line.replace("\\r\\n", "\n").replace("\\n", "\n"); // just unescape newlines async_writeln!(oup, "metadata: {line}")?; } let exit = probe.wait().await?; @@ -131,31 +138,35 @@ impl WritingFileAdapter for FFmpegAdapter { return Err(format_err!("ffprobe failed: {:?}", exit)); } } - if has_subtitles { - // extract subtitles - let mut cmd = Command::new("ffmpeg"); - cmd.arg("-hide_banner") - .arg("-loglevel") - .arg("panic") - .arg("-i") - .arg(&inp_fname) - .arg("-f") - .arg("webvtt") - .arg("-"); - let mut cmd = cmd.stdout(Stdio::piped()).spawn().map_err(spawn_fail)?; - let stdo = cmd.stdout.as_mut().expect("is piped"); - let time_re = Regex::new(r".*\d.*-->.*\d.*").unwrap(); - let mut time: String = "".to_owned(); - // rewrite subtitle times so they are shown as a prefix in every line - let mut lines = BufReader::new(stdo).lines(); - while let Some(line) = lines.next_line().await? { - // 09:55.195 --> 09:56.730 - if time_re.is_match(&line) { - time = line.to_owned(); - } else if line.is_empty() { - async_writeln!(oup)?; - } else { - async_writeln!(oup, "{time}: {line}")?; + if !subtitle_streams.is_empty() { + for probe_stream in subtitle_streams.iter() { + // extract subtitles + let mut cmd = Command::new("ffmpeg"); + cmd.arg("-hide_banner") + .arg("-loglevel") + .arg("panic") + .arg("-i") + .arg(&inp_fname) + .arg("-map") + .arg(format!("0:{}", probe_stream.index)) // 0 for first input + .arg("-f") + .arg("webvtt") + .arg("-"); + let mut cmd = cmd.stdout(Stdio::piped()).spawn().map_err(spawn_fail)?; + let stdo = cmd.stdout.as_mut().expect("is piped"); + let time_re = Regex::new(r".*\d.*-->.*\d.*").unwrap(); + let mut time: String = "".to_owned(); + // rewrite subtitle times so they are shown as a prefix in every line + let mut lines = BufReader::new(stdo).lines(); + while let Some(line) = lines.next_line().await? { + // 09:55.195 --> 09:56.730 + if time_re.is_match(&line) { + time = line.to_owned(); + } else if line.is_empty() { + async_writeln!(oup)?; + } else { + async_writeln!(oup, "{time}: {line}")?; + } } } } diff --git a/src/adapters/postproc.rs b/src/adapters/postproc.rs index 20e4c69..3b42cca 100644 --- a/src/adapters/postproc.rs +++ b/src/adapters/postproc.rs @@ -4,7 +4,11 @@ use anyhow::Result; use async_stream::stream; +use async_trait::async_trait; use bytes::Bytes; +use encoding_rs::Encoding; +use encoding_rs_io::DecodeReaderBytesBuilder; +use tokio_util::io::SyncIoBridge; use std::io::Cursor; use std::path::PathBuf; @@ -41,15 +45,16 @@ impl GetMetadata for PostprocPrefix { &METADATA } } +#[async_trait] impl FileAdapter for PostprocPrefix { - fn adapt<'a>( + async fn adapt( &self, a: super::AdaptInfo, _detection_reason: &crate::matching::FileMatcher, ) -> Result { let read = add_newline(postproc_prefix( &a.line_prefix, - postproc_encoding(&a.line_prefix, a.inp)?, + postproc_encoding(&a.line_prefix, a.inp).await?, )); // keep adapt info (filename etc) except replace inp let ai = AdaptInfo { @@ -74,50 +79,53 @@ impl Read for ReadErr { * Detects and converts encodings other than utf-8 to utf-8. * If the input stream does not contain valid text, returns the string `[rga: binary data]` instead */ -pub fn postproc_encoding( +async fn postproc_encoding( _line_prefix: &str, - inp: impl AsyncRead + Send + 'static, + inp: Pin>, ) -> Result>> { - Ok(Box::pin(inp)) - // panic!("todo: implement"); - /*// TODO: parse these options from ripgrep's configuration - let encoding = None; // detect bom but usually assume utf8 - let bom_sniffing = true; - let mut decode_builder = DecodeReaderBytesBuilder::new(); - // https://github.com/BurntSushi/ripgrep/blob/a7d26c8f144a4957b75f71087a66692d0b25759a/grep-searcher/src/searcher/mod.rs#L706 - // this detects utf-16 BOMs and transcodes to utf-8 if they are present - // it does not detect any other char encodings. that would require https://github.com/hsivonen/chardetng or similar but then binary detection is hard (?) - let inp = decode_builder - .encoding(encoding) - .utf8_passthru(true) - .strip_bom(bom_sniffing) - .bom_override(true) - .bom_sniffing(bom_sniffing) - .build(inp); - // check for binary content in first 8kB // read the first 8kB into a buffer, check for null bytes, then return the buffer concatenated with the rest of the file let mut fourk = Vec::with_capacity(1 << 13); let mut beginning = inp.take(1 << 13); - beginning.read_to_end(&mut fourk)?; - - if fourk.contains(&0u8) { - log::debug!("detected binary"); - let v = "[rga: binary data]"; - return Ok(Box::new(std::io::Cursor::new(v))); - /*let err = std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("{}[rga: binary data]", line_prefix), - ); - return Err(err).context(""); - return ReadErr { - err, - };*/ + beginning.read_to_end(&mut fourk).await?; + let has_binary = fourk.contains(&0u8); + + let enc = Encoding::for_bom(&fourk); + let inp = Cursor::new(fourk).chain(beginning.into_inner()); + match enc { + Some((enc, _)) if enc != encoding_rs::UTF_8 => { + // detected UTF16LE or UTF16BE, convert to UTF8 in separate thread + // TODO: parse these options from ripgrep's configuration + let encoding = None; // detect bom but usually assume utf8 + let bom_sniffing = true; + let mut decode_builder = DecodeReaderBytesBuilder::new(); + // https://github.com/BurntSushi/ripgrep/blob/a7d26c8f144a4957b75f71087a66692d0b25759a/grep-searcher/src/searcher/mod.rs#L706 + // this detects utf-16 BOMs and transcodes to utf-8 if they are present + // it does not detect any other char encodings. that would require https://github.com/hsivonen/chardetng or similar but then binary detection is hard (?) + let mut inp = decode_builder + .encoding(encoding) + .utf8_passthru(true) + .strip_bom(bom_sniffing) + .bom_override(true) + .bom_sniffing(bom_sniffing) + .build(SyncIoBridge::new(inp)); + let oup = tokio::task::spawn_blocking(move || -> Result> { + let mut oup = Vec::new(); + std::io::Read::read_to_end(&mut inp, &mut oup)?; + Ok(oup) + }) + .await??; + Ok(Box::pin(Cursor::new(oup))) + } + _ => { + if has_binary { + log::debug!("detected binary"); + return Ok(Box::pin(Cursor::new("[rga: binary data]"))); + } + Ok(Box::pin(inp)) + } } - Ok(Box::new( - std::io::Cursor::new(fourk).chain(beginning.into_inner()), - ))*/ } /// Adds the given prefix to each line in an `AsyncRead`. @@ -164,13 +172,14 @@ impl GetMetadata for PostprocPageBreaks { &METADATA } } +#[async_trait] impl FileAdapter for PostprocPageBreaks { - fn adapt<'a>( + async fn adapt( &self, a: super::AdaptInfo, _detection_reason: &crate::matching::FileMatcher, ) -> Result { - let read = postproc_pagebreaks(postproc_encoding(&a.line_prefix, a.inp)?); + let read = postproc_pagebreaks(postproc_encoding(&a.line_prefix, a.inp).await?); // keep adapt info (filename etc) except replace inp let ai = AdaptInfo { inp: Box::pin(read), @@ -282,7 +291,7 @@ mod tests { let fname = test_data_dir().join("twoblankpages.pdf"); let rd = File::open(&fname).await?; let (a, d) = simple_adapt_info(&fname, Box::pin(rd)); - let res = loop_adapt(&adapter, d, a)?; + let res = loop_adapt(&adapter, d, a).await?; let buf = adapted_to_vec(res).await?; @@ -327,7 +336,8 @@ PREFIX:Page 3: b: &str, ) -> Result<()> { let mut oup = Vec::new(); - let inp = postproc_encoding("", a)?; + let inp = Box::pin(Cursor::new(a)); + let inp = postproc_encoding("", inp).await?; if pagebreaks { postproc_pagebreaks(inp).read_to_end(&mut oup).await?; } else { @@ -341,6 +351,23 @@ PREFIX:Page 3: Ok(()) } + #[tokio::test] + async fn test_utf16() -> Result<()> { + let utf16lebom: &[u8] = &[ + 0xff, 0xfe, 0x68, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, 0x20, 0x00, + 0x77, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6c, 0x00, 0x64, 0x00, 0x20, 0x00, 0x3d, 0xd8, + 0xa9, 0xdc, 0x0a, 0x00, + ]; + let utf16bebom: &[u8] = &[ + 0xfe, 0xff, 0x00, 0x68, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x6c, 0x00, 0x6f, 0x00, 0x20, + 0x00, 0x77, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6c, 0x00, 0x64, 0x00, 0x20, 0xd8, 0x3d, + 0xdc, 0xa9, 0x00, 0x0a, + ]; + test_from_bytes(false, "", utf16lebom, "hello world 💩\n").await?; + test_from_bytes(false, "", utf16bebom, "hello world 💩\n").await?; + Ok(()) + } + #[tokio::test] async fn post1() -> Result<()> { let inp = "What is this\nThis is a test\nFoo"; @@ -362,20 +389,19 @@ PREFIX:Page 3: Ok(()) } - /* - todo: uncomment when fixed + #[tokio::test] - async fn test_binary_content() -> Result<()> { - test_from_strs( - false, - "foo:", - "this is a test \n\n \0 foo", - "foo:[rga: binary data]", - ) - .await?; - test_from_strs(false, "foo:", "\0", "foo:[rga: binary data]").await?; - Ok(()) - }*/ + async fn test_binary_content() -> Result<()> { + test_from_strs( + false, + "foo:", + "this is a test \n\n \0 foo", + "foo:[rga: binary data]", + ) + .await?; + test_from_strs(false, "foo:", "\0", "foo:[rga: binary data]").await?; + Ok(()) + } /*#[test] fn chardet() -> Result<()> { diff --git a/src/adapters/sqlite.rs b/src/adapters/sqlite.rs index 836b330..dba24a4 100644 --- a/src/adapters/sqlite.rs +++ b/src/adapters/sqlite.rs @@ -77,11 +77,13 @@ fn synchronous_dump_sqlite(ai: AdaptInfo, mut s: impl Write) -> Result<()> { return Ok(()); } let inp_fname = filepath_hint; - - let conn = Connection::open_with_flags(inp_fname, OpenFlags::SQLITE_OPEN_READ_ONLY)?; + let conn = Connection::open_with_flags(&inp_fname, OpenFlags::SQLITE_OPEN_READ_ONLY) + .with_context(|| format!("opening sqlite connection to {}", inp_fname.display()))?; let tables: Vec = conn - .prepare("select name from sqlite_master where type='table'")? - .query_map([], |r| r.get::<_, String>(0))? + .prepare("select name from sqlite_master where type='table'") + .context("while preparing query")? + .query_map([], |r| r.get::<_, String>(0)) + .context("while executing query")? .filter_map(|e| e.ok()) .collect(); debug!("db has {} tables", tables.len()); @@ -121,7 +123,9 @@ impl WritingFileAdapter for SqliteAdapter { oup: Pin>, ) -> Result<()> { let oup_sync = SyncIoBridge::new(oup); - tokio::task::spawn_blocking(|| synchronous_dump_sqlite(ai, oup_sync)).await??; + tokio::task::spawn_blocking(|| synchronous_dump_sqlite(ai, oup_sync)) + .await? + .context("in synchronous sqlite task")?; Ok(()) } } @@ -134,10 +138,10 @@ mod test { #[tokio::test] async fn simple() -> Result<()> { - let adapter: Box = Box::new(SqliteAdapter::default()); + let adapter: Box = Box::::default(); let fname = test_data_dir().join("hello.sqlite3"); let (a, d) = simple_fs_adapt_info(&fname).await?; - let res = adapter.adapt(a, &d)?; + let res = adapter.adapt(a, &d).await?; let buf = adapted_to_vec(res).await?; diff --git a/src/adapters/tar.rs b/src/adapters/tar.rs index 637829f..144bd20 100644 --- a/src/adapters/tar.rs +++ b/src/adapters/tar.rs @@ -6,6 +6,7 @@ use crate::{ }; use anyhow::*; use async_stream::stream; +use async_trait::async_trait; use lazy_static::lazy_static; use log::*; use std::path::PathBuf; @@ -45,8 +46,13 @@ impl GetMetadata for TarAdapter { } } +#[async_trait] impl FileAdapter for TarAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &FileMatcher) -> Result { + async fn adapt( + &self, + ai: AdaptInfo, + _detection_reason: &FileMatcher, + ) -> Result { let AdaptInfo { filepath_hint, inp, @@ -103,7 +109,7 @@ mod tests { let (a, d) = simple_adapt_info(&filepath, Box::pin(File::open(&filepath).await?)); let adapter = TarAdapter::new(); - let r = loop_adapt(&adapter, d, a).context("adapt")?; + let r = loop_adapt(&adapter, d, a).await.context("adapt")?; let o = adapted_to_vec(r).await.context("adapted_to_vec")?; assert_eq!( String::from_utf8(o).context("parsing utf8")?, diff --git a/src/adapters/writing.rs b/src/adapters/writing.rs index 548711c..b17152a 100644 --- a/src/adapters/writing.rs +++ b/src/adapters/writing.rs @@ -3,7 +3,7 @@ use std::pin::Pin; use crate::{adapted_iter::one_file, join_handle_to_stream, to_io_err}; use super::{AdaptInfo, FileAdapter, GetMetadata}; -use anyhow::Result; +use anyhow::{Context, Result}; use async_trait::async_trait; use tokio::io::{AsyncReadExt, AsyncWrite}; @@ -41,15 +41,17 @@ macro_rules! async_writeln { } pub(crate) use async_writeln; +#[async_trait] impl FileAdapter for T where T: WritingFileAdapter, { - fn adapt( + async fn adapt( &self, a: super::AdaptInfo, detection_reason: &crate::matching::FileMatcher, ) -> Result { + let name = self.metadata().name.clone(); let (w, r) = tokio::io::duplex(128 * 1024); let d2 = detection_reason.clone(); let archive_recursion_depth = a.archive_recursion_depth + 1; @@ -59,7 +61,10 @@ where let config = a.config.clone(); let joiner = tokio::spawn(async move { let x = d2; - T::adapt_write(a, &x, Box::pin(w)).await.map_err(to_io_err) + T::adapt_write(a, &x, Box::pin(w)) + .await + .with_context(|| format!("in {}.adapt_write", name)) + .map_err(to_io_err) }); Ok(one_file(AdaptInfo { diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index e6ee1bc..d1777cc 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -5,7 +5,7 @@ use async_stream::stream; use lazy_static::lazy_static; use log::*; -static EXTENSIONS: &[&str] = &["zip"]; +static EXTENSIONS: &[&str] = &["zip", "jar"]; lazy_static! { static ref METADATA: AdapterMeta = AdapterMeta { @@ -36,8 +36,13 @@ impl GetMetadata for ZipAdapter { } } +#[async_trait] impl FileAdapter for ZipAdapter { - fn adapt(&self, ai: AdaptInfo, _detection_reason: &FileMatcher) -> Result { + async fn adapt( + &self, + ai: AdaptInfo, + _detection_reason: &FileMatcher, + ) -> Result { // let (s, r) = mpsc::channel(1); let AdaptInfo { inp, @@ -52,11 +57,11 @@ impl FileAdapter for ZipAdapter { if is_real_file { use async_zip::read::fs::ZipFileReader; + let zip = ZipFileReader::new(&filepath_hint).await?; let s = stream! { - let zip = ZipFileReader::new(&filepath_hint).await?; - for i in 0..zip.entries().len() { - let reader = zip.entry_reader(i).await?; - let file = reader.entry(); + for i in 0..zip.file().entries().len() { + let file = zip.get_entry(i)?; + let reader = zip.entry(i).await?; if file.filename().ends_with('/') { continue; } @@ -98,10 +103,11 @@ impl FileAdapter for ZipAdapter { let mut zip = ZipFileReader::new(inp); let s = stream! { - while !zip.finished() { - if let Some(reader) = zip.entry_reader().await? { - let file = reader.entry(); + while let Some(mut entry) = zip.next_entry().await? { + let file = entry.entry(); if file.filename().ends_with('/') { + zip = entry.skip().await?; + continue; } debug!( @@ -114,6 +120,7 @@ impl FileAdapter for ZipAdapter { ); let new_line_prefix = format!("{}{}: ", line_prefix, file.filename()); let fname = PathBuf::from(file.filename()); + let reader = entry.reader(); tokio::pin!(reader); // SAFETY: this should be solvable without unsafe but idk how :( // the issue is that ZipEntryReader borrows from ZipFileReader, but we need to yield it here into the stream @@ -133,7 +140,8 @@ impl FileAdapter for ZipAdapter { postprocess, config: config.clone(), }); - } + zip = entry.done().await.context("going to next file in zip but entry was not read fully")?; + } }; @@ -182,7 +190,6 @@ impl<'a> AdaptedFilesIter for ZipAdaptIter<'a> { #[cfg(test)] mod test { use async_zip::{write::ZipFileWriter, Compression, ZipEntryBuilder}; - use super::*; use crate::{preproc::loop_adapt, test_utils::*}; @@ -213,7 +220,7 @@ mod test { async fn only_seek_zip_fs() -> Result<()> { let zip = test_data_dir().join("only-seek-zip.zip"); let (a, d) = simple_fs_adapt_info(&zip).await?; - let _v = adapted_to_vec(loop_adapt(&ZipAdapter::new(), d, a)?).await?; + let _v = adapted_to_vec(loop_adapt(&ZipAdapter::new(), d, a).await?).await?; // assert_eq!(String::from_utf8(v)?, ""); Ok(()) @@ -236,7 +243,7 @@ mod test { &PathBuf::from("outer.zip"), Box::pin(std::io::Cursor::new(zipfile)), ); - let buf = adapted_to_vec(loop_adapt(&adapter, d, a)?).await?; + let buf = adapted_to_vec(loop_adapt(&adapter, d, a).await?).await?; assert_eq!( String::from_utf8(buf)?, diff --git a/src/bin/rga-preproc.rs b/src/bin/rga-preproc.rs index 7af2ddb..2b25cef 100644 --- a/src/bin/rga-preproc.rs +++ b/src/bin/rga-preproc.rs @@ -43,7 +43,7 @@ async fn main() -> anyhow::Result<()> { // happens if e.g. ripgrep detects binary data in the pipe so it cancels reading debug!("output cancelled (broken pipe)"); } else { - Err(e).context("copying adapter output to stdout {}")?; + Err(e).context("copying adapter output to stdout")?; } } debug!("running adapter took {} total", print_dur(start)); diff --git a/src/caching_writer.rs b/src/caching_writer.rs index 048e774..7563f1d 100644 --- a/src/caching_writer.rs +++ b/src/caching_writer.rs @@ -1,17 +1,17 @@ -use std::pin::Pin; +use std::{future::Future, pin::Pin}; -use anyhow::Result; +use anyhow::{Context, Result}; use async_compression::tokio::write::ZstdEncoder; use async_stream::stream; +use crate::to_io_err; use log::*; use tokio::io::{AsyncRead, AsyncWriteExt}; use tokio_stream::StreamExt; use tokio_util::io::{ReaderStream, StreamReader}; -use crate::to_io_err; - -type FinishHandler = dyn FnOnce((u64, Option>)) -> Result<()> + Send; +type FinishHandler = + dyn FnOnce((u64, Option>)) -> Pin> + Send>> + Send; /** * wrap a AsyncRead so that it is passthrough, * but also the written data is compressed and written into a buffer, @@ -26,7 +26,7 @@ pub fn async_read_and_write_to_cache<'a>( let inp = Box::pin(inp); let mut zstd_writer = Some(ZstdEncoder::with_quality( Vec::new(), - async_compression::Level::Precise(compression_level as u32), + async_compression::Level::Precise(compression_level), )); let mut bytes_written = 0; @@ -64,7 +64,7 @@ pub fn async_read_and_write_to_cache<'a>( }; // EOF, finish! - on_finish(finish) + on_finish(finish).await.context("write_to_cache on_finish") .map_err(to_io_err)?; }; diff --git a/src/config.rs b/src/config.rs index df6979f..da173fe 100644 --- a/src/config.rs +++ b/src/config.rs @@ -108,6 +108,7 @@ impl FromStr for CacheMaxBlobLen { rename_all = "kebab-case", about = env!("CARGO_PKG_DESCRIPTION"), author = env!("CARGO_PKG_HOMEPAGE"), + long_about="rga: ripgrep, but also search in PDFs, E-Books, Office documents, zip, tar.gz, etc.", // TODO: long_about does not seem to work to only show this on short help after_help = "-h shows a concise overview, --help shows more detail and advanced options.\n\nAll other options not shown here are passed directly to rg, especially [PATTERN] and [PATH ...]", usage = "rga [RGA OPTIONS] [RG OPTIONS] PATTERN [PATH ...]" @@ -197,9 +198,9 @@ pub struct CacheConfig { /// Disable caching of results /// /// By default, rga caches the extracted text, if it is small enough, - /// to a database in ~/.cache/rga on Linux, - /// ~/Library/Caches/rga on macOS, - /// or C:\Users\username\AppData\Local\rga on Windows. + /// to a database in ${XDG_CACHE_DIR-~/.cache}/ripgrep-all on Linux, + /// ~/Library/Caches/ripgrep-all on macOS, + /// or C:\Users\username\AppData\Local\ripgrep-all on Windows. /// This way, repeated searches on the same set of files will be much faster. /// If you pass this flag, all caching will be disabled. #[serde(default, skip_serializing_if = "is_default")] @@ -208,7 +209,9 @@ pub struct CacheConfig { /// Max compressed size to cache /// - /// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time. Allowed suffixes: k M G + /// Longest byte length (after compression) to store in cache. Longer adapter outputs will not be cached and recomputed every time. + /// + /// Allowed suffixes on command line: k M G #[serde(default, skip_serializing_if = "is_default")] #[structopt( default_value, diff --git a/src/preproc.rs b/src/preproc.rs index 3f632c3..f3b01ec 100644 --- a/src/preproc.rs +++ b/src/preproc.rs @@ -3,25 +3,28 @@ use crate::adapters::*; use crate::caching_writer::async_read_and_write_to_cache; use crate::config::RgaConfig; use crate::matching::*; +use crate::preproc_cache::CacheKey; use crate::recurse::concat_read_streams; use crate::{ - preproc_cache::{LmdbCache, PreprocCache}, + preproc_cache::{open_cache_db, PreprocCache}, print_bytes, }; use anyhow::*; use async_compression::tokio::bufread::ZstdDecoder; use async_stream::stream; +// use futures::future::{BoxFuture, FutureExt}; use log::*; -use path_clean::PathClean; use postproc::PostprocPrefix; +use std::future::Future; use std::io::Cursor; use std::path::Path; +use std::pin::Pin; use std::sync::Arc; use tokio::io::AsyncBufRead; use tokio::io::AsyncBufReadExt; use tokio::io::BufReader; -type ActiveAdapters = Vec>; +pub type ActiveAdapters = Vec>; async fn choose_adapter( config: &RgaConfig, @@ -120,36 +123,6 @@ pub async fn rga_preproc(ai: AdaptInfo) -> Result { .with_context(|| format!("run_adapter({})", &path_hint_copy.to_string_lossy())) } -fn compute_cache_key( - filepath_hint: &Path, - adapter: &dyn FileAdapter, - active_adapters: ActiveAdapters, -) -> Result> { - let clean_path = filepath_hint.to_owned().clean(); - let meta = std::fs::metadata(filepath_hint) - .with_context(|| format!("reading metadata for {}", filepath_hint.to_string_lossy()))?; - let modified = meta.modified().expect("weird OS that can't into mtime"); - - if adapter.metadata().recurses { - let active_adapters_cache_key = active_adapters - .iter() - .map(|a| (a.metadata().name.clone(), a.metadata().version)) - .collect::>(); - let key = (active_adapters_cache_key, clean_path, modified); - debug!("Cache key (with recursion): {:?}", key); - bincode::serialize(&key).context("could not serialize path") - } else { - let key = ( - adapter.metadata().name.clone(), - adapter.metadata().version, - clean_path, - modified, - ); - debug!("Cache key (no recursion): {:?}", key); - bincode::serialize(&key).context("could not serialize path") - } -} - async fn adapt_caching( ai: AdaptInfo, adapter: Arc, @@ -166,41 +139,44 @@ async fn adapt_caching( ai.filepath_hint.to_string_lossy(), &meta.name ); - let db_name = format!("{}.v{}", meta.name, meta.version); let cache_compression_level = ai.config.cache.compression_level; let cache_max_blob_len = ai.config.cache.max_blob_len; - let cache = if ai.is_real_file { - LmdbCache::open(&ai.config.cache)? + let cache = if ai.is_real_file && !ai.config.cache.disabled { + Some(open_cache_db(Path::new(&ai.config.cache.path.0)).await?) } else { None }; let mut cache = cache.context("No cache?")?; - let cache_key: Vec = - compute_cache_key(&ai.filepath_hint, adapter.as_ref(), active_adapters)?; + let cache_key = CacheKey::new(&ai.filepath_hint, adapter.as_ref(), &active_adapters)?; // let dbg_ctx = format!("adapter {}", &adapter.metadata().name); - let cached = cache.get(&db_name, &cache_key)?; + let cached = cache.get(&cache_key).await.context("cache.get")?; match cached { Some(cached) => Ok(Box::pin(ZstdDecoder::new(Cursor::new(cached)))), None => { debug!("cache MISS, running adapter with caching..."); - let inp = loop_adapt(adapter.as_ref(), detection_reason, ai)?; + let inp = loop_adapt(adapter.as_ref(), detection_reason, ai).await?; let inp = concat_read_streams(inp); let inp = async_read_and_write_to_cache( inp, cache_max_blob_len.0, cache_compression_level.0, Box::new(move |(uncompressed_size, compressed)| { - debug!( - "uncompressed output: {}", - print_bytes(uncompressed_size as f64) - ); - if let Some(cached) = compressed { - debug!("compressed output: {}", print_bytes(cached.len() as f64)); - cache.set(&db_name, &cache_key, &cached)? - } - Ok(()) + Box::pin(async move { + debug!( + "uncompressed output: {}", + print_bytes(uncompressed_size as f64) + ); + if let Some(cached) = compressed { + debug!("compressed output: {}", print_bytes(cached.len() as f64)); + cache + .set(&cache_key, cached) + .await + .context("writing to cache")? + } + Ok(()) + }) }), )?; @@ -213,21 +189,34 @@ pub fn loop_adapt( adapter: &dyn FileAdapter, detection_reason: FileMatcher, ai: AdaptInfo, +) -> Pin> + Send + '_>> { + Box::pin(async move { loop_adapt_inner(adapter, detection_reason, ai).await }) +} +pub async fn loop_adapt_inner( + adapter: &dyn FileAdapter, + detection_reason: FileMatcher, + ai: AdaptInfo, ) -> anyhow::Result { let fph = ai.filepath_hint.clone(); - let inp = adapter.adapt(ai, &detection_reason).with_context(|| { - format!( - "adapting {} via {} failed", - fph.to_string_lossy(), - adapter.metadata().name - ) - })?; + let inp = adapter.adapt(ai, &detection_reason).await; + let inp = if adapter.metadata().name == "postprocprefix" { + // don't add confusing error context + inp? + } else { + inp.with_context(|| { + format!( + "adapting {} via {} failed", + fph.to_string_lossy(), + adapter.metadata().name + ) + })? + }; let s = stream! { for await file in inp { match buf_choose_adapter(file?).await? { Ret::Recurse(ai, adapter, detection_reason, _active_adapters) => { if ai.archive_recursion_depth >= ai.config.max_archive_recursion.0 { - let s = format!("{}[rga: max archive recursion reached ({})]", ai.line_prefix, ai.archive_recursion_depth).into_bytes(); + let s = format!("{}[rga: max archive recursion reached ({})]\n", ai.line_prefix, ai.archive_recursion_depth).into_bytes(); yield Ok(AdaptInfo { inp: Box::pin(Cursor::new(s)), ..ai @@ -243,7 +232,7 @@ pub fn loop_adapt( ai.filepath_hint.to_string_lossy(), &adapter.metadata().name ); - for await ifile in loop_adapt(adapter.as_ref(), detection_reason, ai)? { + for await ifile in loop_adapt(adapter.as_ref(), detection_reason, ai).await? { yield ifile; } } diff --git a/src/preproc_cache.rs b/src/preproc_cache.rs index 764213d..d0eed0e 100644 --- a/src/preproc_cache.rs +++ b/src/preproc_cache.rs @@ -1,135 +1,188 @@ -use crate::{config::CacheConfig, print_bytes, print_dur}; -use anyhow::{format_err, Context, Result}; -use log::*; -use rkv::backend::{BackendEnvironmentBuilder, LmdbEnvironment}; -use std::{fmt::Display, path::Path, time::Instant}; +use crate::{adapters::FileAdapter, preproc::ActiveAdapters}; +use anyhow::{Context, Result}; +use path_clean::PathClean; +use rusqlite::{named_params, OptionalExtension}; +use std::{path::Path, time::UNIX_EPOCH}; +use tokio_rusqlite::Connection; -pub trait PreprocCache: Send + Sync { - /*/// gets cache at specified key. - /// if cache hit, return the resulting data - /// else, run the given lambda, and store its result in the cache if present - fn get_or_run<'a>( - &mut self, - db_name: &str, - key: &[u8], - debug_name: &str, - runner: Box Result>> + 'a>, - ) -> Result>>;*/ - - fn get(&self, db_name: &str, key: &[u8]) -> Result>>; - fn set(&mut self, db_name: &str, key: &[u8], value: &[u8]) -> Result<()>; +#[derive(Clone)] +pub struct CacheKey { + adapter: String, + adapter_version: i32, + active_adapters: String, + file_path: String, + file_mtime_unix_ms: i64, } - -/// opens a LMDB cache -fn open_cache_db( - path: &Path, -) -> Result>>> { - std::fs::create_dir_all(path)?; - // use rkv::backend::LmdbEnvironmentFlags; - - rkv::Manager::::singleton() - .write() - .map_err(|_| format_err!("could not write cache db manager"))? - .get_or_create(path, |p| { - let mut builder = rkv::Rkv::environment_builder::(); - builder - .set_flags(rkv::EnvironmentFlags::NO_SYNC) - .set_flags(rkv::EnvironmentFlags::WRITE_MAP) // not durable cuz it's a cache - // i'm not sure why NO_TLS is needed. otherwise LMDB transactions (open readers) will keep piling up until it fails with - // LmdbError(ReadersFull). Those "open readers" stay even after the corresponding processes exit. - // hope setting this doesn't break integrity - .set_flags(rkv::EnvironmentFlags::NO_TLS) - // sometimes, this seems to cause the data.mdb file to appear as 2GB in size (with holes), but sometimes not? - .set_map_size(2 * 1024 * 1024 * 1024) - .set_max_dbs(100) - .set_max_readers(128); - rkv::Rkv::from_builder(p, builder) +impl CacheKey { + pub fn new( + filepath_hint: &Path, + adapter: &dyn FileAdapter, + active_adapters: &ActiveAdapters, + ) -> Result { + let meta = std::fs::metadata(filepath_hint) + .with_context(|| format!("reading metadata for {}", filepath_hint.to_string_lossy()))?; + let modified = meta.modified().expect("weird OS that can't into mtime"); + let file_mtime_unix_ms = modified.duration_since(UNIX_EPOCH)?.as_millis() as i64; + let active_adapters = if adapter.metadata().recurses { + serde_json::to_string( + &active_adapters + .iter() + .map(|a| format!("{}.v{}", a.metadata().name, a.metadata().version)) + .collect::>(), + )? + } else { + "null".to_string() + }; + Ok(CacheKey { + adapter: adapter.metadata().name.clone(), + adapter_version: adapter.metadata().version, + file_path: filepath_hint.clean().to_string_lossy().to_string(), + file_mtime_unix_ms, + active_adapters, }) - .map_err(|e| format_err!("could not get/create cache db: {}", e)) + } } -pub struct LmdbCache { - db_arc: std::sync::Arc>>, +#[async_trait::async_trait] +pub trait PreprocCache { + async fn get(&self, key: &CacheKey) -> Result>>; + async fn set(&mut self, key: &CacheKey, value: Vec) -> Result<()>; } -impl LmdbCache { - pub fn open(config: &CacheConfig) -> Result> { - if config.disabled { - return Ok(None); - } - let path = Path::new(&config.path.0); - Ok(Some(LmdbCache { - db_arc: open_cache_db(path)?, - })) +async fn connect_pragmas(db: &Connection) -> Result<()> { + // https://phiresky.github.io/blog/2020/sqlite-performance-tuning/ + //let want_page_size = 32768; + //db.execute(&format!("pragma page_size = {};", want_page_size)) + // .context("setup pragma 1")?; + db.call(|db| { + db.execute_batch( + " + pragma journal_mode = WAL; + pragma foreign_keys = on; + pragma temp_store = memory; + pragma synchronous = off; -- integrity isn't very important here + pragma mmap_size = 30000000000; + + create table if not exists preproc_cache ( + adapter text not null, + adapter_version integer not null, + created_unix_ms integer not null default (unixepoch() * 1000), + active_adapters text not null, -- 'null' if adapter cannot recurse + file_path text not null, + file_mtime_unix_ms integer not null, + text_content_zstd blob not null + ) strict; + + create unique index if not exists preproc_cache_idx on preproc_cache (adapter, adapter_version, file_path, active_adapters); + ", + ) + }) + .await.context("connect_pragmas")?; + let jm: i64 = db + .call(|db| db.pragma_query_value(None, "application_id", |r| r.get(0))) + .await?; + if jm != 924716026 { + // (probably) newly created db + create_pragmas(db).await.context("create_pragmas")?; } + Ok(()) } -#[derive(Debug)] -struct RkvErrWrap(rkv::StoreError); -impl Display for RkvErrWrap { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } +async fn create_pragmas(db: &Connection) -> Result<()> { + db.call(|db| { + db.execute_batch( + " + pragma application_id = 924716026; + pragma user_version = 2; -- todo: on upgrade clear db if version is unexpected + ", + ) + }) + .await?; + Ok(()) } -impl std::error::Error for RkvErrWrap {} +struct SqliteCache { + db: Connection, +} +impl SqliteCache { + async fn new(path: &Path) -> Result { + let db = Connection::open(path.join("cache.sqlite3")).await?; + connect_pragmas(&db).await?; -impl PreprocCache for LmdbCache { - fn get(&self, db_name: &str, key: &[u8]) -> Result>> { - let start = Instant::now(); - let db_env = self - .db_arc - .read() - .map_err(|_| anyhow::anyhow!("Could not open lock, some lock writer panicked"))?; - let db = db_env - .open_single(db_name, rkv::store::Options::create()) - .map_err(RkvErrWrap) - .context("could not open cache db store")?; + Ok(SqliteCache { db }) + } +} - let reader = db_env.read().expect("could not get reader"); - let cached = db - .get(&reader, key) - .map_err(RkvErrWrap) - .context("could not read from db")?; +#[async_trait::async_trait] +impl PreprocCache for SqliteCache { + async fn get(&self, key: &CacheKey) -> Result>> { + let key = (*key).clone(); // todo: without cloning + Ok(self + .db + .call(move |db| { + db.query_row( + "select text_content_zstd from preproc_cache where + adapter = :adapter + and adapter_version = :adapter_version + and active_adapters = :active_adapters + and file_path = :file_path + and file_mtime_unix_ms = :file_mtime_unix_ms + ", + named_params! { + ":adapter": &key.adapter, + ":adapter_version": &key.adapter_version, + ":active_adapters": &key.active_adapters, + ":file_path": &key.file_path, + ":file_mtime_unix_ms": &key.file_mtime_unix_ms + }, + |r| r.get::<_, Vec>(0), + ) + .optional() + }) + .await + .context("reading from cache")?) + } - match cached { - Some(rkv::Value::Blob(cached)) => { - debug!( - "cache HIT, reading {} (compressed) from cache", - print_bytes(cached.len() as f64) - ); - debug!("reading from cache took {}", print_dur(start)); - Ok(Some(Vec::from(cached))) - } - Some(_) => Err(format_err!("Integrity: value not blob"))?, - None => Ok(None), - } + async fn set(&mut self, key: &CacheKey, value: Vec) -> Result<()> { + let key = (*key).clone(); // todo: without cloning + Ok(self + .db + .call(move |db| { + db.execute( + "insert into preproc_cache (adapter, adapter_version, active_adapters, file_path, file_mtime_unix_ms, text_content_zstd) values + (:adapter, :adapter_version, :active_adapters, :file_path, :file_mtime_unix_ms, :text_content_zstd) + on conflict (adapter, adapter_version, active_adapters, file_path) do update set + file_mtime_unix_ms = :file_mtime_unix_ms, + created_unix_ms = unixepoch() * 1000, + text_content_zstd = :text_content_zstd", + named_params! { + ":adapter": &key.adapter, + ":adapter_version": &key.adapter_version, + ":active_adapters": &key.active_adapters, + ":file_path": &key.file_path, + ":file_mtime_unix_ms": &key.file_mtime_unix_ms, + ":text_content_zstd": value + } + ).map(|_| ()) + }) + .await?) } - fn set(&mut self, db_name: &str, key: &[u8], got: &[u8]) -> Result<()> { - let start = Instant::now(); - debug!("writing {} to cache", print_bytes(got.len() as f64)); - let db_env = self - .db_arc - .read() - .map_err(|_| anyhow::anyhow!("Could not open lock, some lock writer panicked"))?; +} +/// opens a default cache +pub async fn open_cache_db(path: &Path) -> Result { + std::fs::create_dir_all(path)?; + SqliteCache::new(path).await +} - let db = db_env - .open_single(db_name, rkv::store::Options::create()) - .map_err(RkvErrWrap) - .context("could not open cache db store")?; +#[cfg(test)] +mod test { - let mut writer = db_env - .write() - .map_err(RkvErrWrap) - .with_context(|| format_err!("could not open write handle to cache"))?; + use crate::preproc_cache::*; - db.put(&mut writer, key, &rkv::Value::Blob(got)) - .map_err(RkvErrWrap) - .with_context(|| format_err!("could not write to cache"))?; - writer - .commit() - .map_err(RkvErrWrap) - .with_context(|| "could not write cache".to_string())?; - debug!("writing to cache took {}", print_dur(start)); + #[tokio::test] + async fn test_read_write() -> anyhow::Result<()> { + let path = tempfile::tempdir()?; + let _db = open_cache_db(&path.path().join("foo.sqlite3")).await?; + // db.set(); Ok(()) } }