From 816e27802ae07cab057cc3412a787be39f210e8c Mon Sep 17 00:00:00 2001 From: phiresky Date: Sat, 18 Feb 2023 23:10:38 +0100 Subject: [PATCH] fix writing file adapter --- Cargo.lock | 320 +++++++++++++++++++++++----------------- Cargo.toml | 1 + src/adapters.rs | 8 +- src/adapters/ffmpeg.rs | 44 +++--- src/adapters/sqlite.rs | 6 +- src/adapters/writing.rs | 107 ++++++++++---- src/adapters/zip.rs | 4 +- src/lib.rs | 2 +- 8 files changed, 296 insertions(+), 196 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c326e56..589fb42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,9 +63,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" [[package]] name = "arrayref" @@ -89,7 +89,7 @@ dependencies = [ "pin-project-lite 0.2.9", "tokio 0.2.25", "tokio 0.3.7", - "tokio 1.23.0", + "tokio 1.25.0", "xz2", "zstd", "zstd-safe", @@ -97,9 +97,9 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cda8f4bcc10624c4e85bc66b3f452cca98cfa5ca002dc83a16aad2367641bea" +checksum = "3b015a331cc64ebd1774ba119538573603427eaace0a1950c423ab971f903796" dependencies = [ "proc-macro2", "quote", @@ -108,19 +108,31 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +checksum = "ad445822218ce64be7a341abfb0b1ea43b5c23aa83902542a4542e78309d8e5e" dependencies = [ "async-stream-impl", "futures-core", + "pin-project-lite 0.2.9", ] [[package]] name = "async-stream-impl" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +checksum = "e4655ae1a7b0cdf149156f780c5bf3f1352bc53cbd9e0a361a7ef7b22947e965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", @@ -133,7 +145,7 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b20cffc5590f4bf33f05f97a3ea587feba9c50d20325b401daa096b92ff7da0" dependencies = [ - "tokio 1.23.0", + "tokio 1.25.0", ] [[package]] @@ -147,7 +159,7 @@ dependencies = [ "chrono", "crc32fast", "thiserror", - "tokio 1.23.0", + "tokio 1.25.0", ] [[package]] @@ -195,9 +207,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.3.2" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ad2d4653bf5ca36ae797b1f4bb4dbddb60ce49ca4aed8a2ce4829f60425b80" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -205,9 +217,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.11.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" +checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytecount" @@ -229,15 +241,15 @@ checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "bzip2" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6afcd980b5f3a45017c57e57a2fcccbb351cc43a356ce117ef760ef8052b89b0" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" dependencies = [ "bzip2-sys", "libc", @@ -256,9 +268,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" dependencies = [ "jobserver", ] @@ -301,9 +313,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.0.32" +version = "4.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7db700bc935f9e43e88d00b0850dae18a63773cfbec6d8e070fccf7fef89a39" +checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" dependencies = [ "bitflags", "clap_lex", @@ -315,9 +327,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4198f73e42b4936b35b5bb248d81d2b595ecb170da0bac7655c54eedfa8da8" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" dependencies = [ "os_str_bytes", ] @@ -432,9 +444,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.85" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add3fc1717409d029b20c5b6903fc0c0b02fa6741d820054f4a2efa5e5816fd" +checksum = "86d3488e7665a7a483b57e25bdd90d0aeb2bc7608c8d0346acf2ad3f1caf1d62" dependencies = [ "cc", "cxxbridge-flags", @@ -444,9 +456,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.85" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c87959ba14bc6fbc61df77c3fcfe180fc32b93538c4f1031dd802ccb5f2ff0" +checksum = "48fcaf066a053a41a81dfb14d57d99738b767febb8b735c3016e469fac5da690" dependencies = [ "cc", "codespan-reporting", @@ -459,15 +471,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.85" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69a3e162fde4e594ed2b07d0f83c6c67b745e7f28ce58c6df5e6b6bef99dfb59" +checksum = "a2ef98b8b717a829ca5603af80e1f9e2e48013ab227b68ef37872ef84ee479bf" [[package]] name = "cxxbridge-macro" -version = "1.0.85" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e7e2adeb6a0d4a282e581096b06e1791532b7d576dcde5ccd9382acf55db8e6" +checksum = "086c685979a698443656e5cf7856c95c642295a38599f12fb1ff76fb28d19892" dependencies = [ "proc-macro2", "quote", @@ -543,9 +555,9 @@ checksum = "c9b0705efd4599c15a38151f4721f7bc388306f61084d3bfd50bd07fbca5cb60" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if", ] @@ -607,23 +619,23 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] [[package]] name = "filetime" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -659,21 +671,21 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -682,21 +694,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-core", "futures-macro", @@ -737,9 +749,9 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "hashbrown" @@ -786,6 +798,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "humantime" version = "2.1.0" @@ -854,24 +872,24 @@ dependencies = [ [[package]] name = "io-lifetimes" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "is-terminal" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28dfb6c8100ccc63462345b67d1bbc3679177c75ee4bf59bf29c8b1d110b8189" +checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" dependencies = [ - "hermit-abi 0.2.6", + "hermit-abi 0.3.1", "io-lifetimes", "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -891,9 +909,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -1027,21 +1045,21 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] name = "nom" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1123,9 +1141,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "ordered-float" @@ -1163,15 +1181,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1194,9 +1212,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "petgraph" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", "indexmap", @@ -1274,9 +1292,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.49" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -1312,9 +1330,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a" +checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" dependencies = [ "aho-corasick", "memchr", @@ -1344,10 +1362,11 @@ dependencies = [ "async-compression", "async-recursion", "async-stream", + "async-trait", "async_zip", "bincode", - "bytes 1.3.0", - "clap 4.0.32", + "bytes 1.4.0", + "clap 4.1.6", "crossbeam", "crossbeam-channel", "ctor", @@ -1376,7 +1395,7 @@ dependencies = [ "size_format", "structopt", "tempfile", - "tokio 1.23.0", + "tokio 1.25.0", "tokio-stream", "tokio-tar", "tokio-test", @@ -1432,16 +1451,16 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.5" +version = "0.36.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3807b5d10909833d3e9acd1eb5fb988f79376ff10fce42937de71a449c4c588" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1526,9 +1545,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" dependencies = [ "itoa", "ryu", @@ -1537,9 +1556,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" dependencies = [ "libc", ] @@ -1642,21 +1661,21 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.3" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" dependencies = [ "winapi-util", ] [[package]] name = "terminal_size" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb20089a8ba2b69debd491f8d2d023761cbf196e999218c591fa1e7e15a21907" +checksum = "4c9afddd2cec1c0909f06b00ef33f94ab2cc0578c4a610aa208ddfec8aa2b43a" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1710,9 +1729,9 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" @@ -1736,12 +1755,12 @@ dependencies = [ [[package]] name = "tokio" -version = "1.23.0" +version = "1.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" +checksum = "c8e00990ebabbe4c14c08aca901caed183ecd5c09562a12c824bb53d3c3fd3af" dependencies = [ "autocfg", - "bytes 1.3.0", + "bytes 1.4.0", "libc", "memchr", "mio", @@ -1751,7 +1770,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1773,7 +1792,7 @@ checksum = "d660770404473ccd7bc9f8b28494a811bc18542b915c0855c51e8f419d5223ce" dependencies = [ "futures-core", "pin-project-lite 0.2.9", - "tokio 1.23.0", + "tokio 1.25.0", "tokio-util", ] @@ -1786,7 +1805,7 @@ dependencies = [ "futures-core", "libc", "redox_syscall", - "tokio 1.23.0", + "tokio 1.25.0", "tokio-stream", "xattr", ] @@ -1798,19 +1817,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" dependencies = [ "async-stream", - "bytes 1.3.0", + "bytes 1.4.0", "futures-core", - "tokio 1.23.0", + "tokio 1.25.0", "tokio-stream", ] [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ - "bytes 1.3.0", + "bytes 1.4.0", "futures-core", "futures-io", "futures-sink", @@ -1818,7 +1837,7 @@ dependencies = [ "hashbrown", "pin-project-lite 0.2.9", "slab", - "tokio 1.23.0", + "tokio 1.25.0", "tracing", ] @@ -1864,9 +1883,9 @@ checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" [[package]] name = "unicode-bidi" -version = "0.3.8" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" @@ -1885,9 +1904,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -1944,9 +1963,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -1954,9 +1973,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -1969,9 +1988,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1979,9 +1998,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -1992,9 +2011,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "winapi" @@ -2042,47 +2061,71 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "xattr" @@ -2129,10 +2172,11 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.4+zstd.1.5.2" +version = "2.0.7+zstd.1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa202f2ef00074143e219d15b62ffc317d17cc33909feac471c044087cad7b0" +checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" dependencies = [ "cc", "libc", + "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 0859993..7d2f468 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ version = "0.9.7-alpha.0" anyhow = "1.0.32" async-compression = {version = "0.3.15", features = ["all", "all-algorithms", "tokio"]} async-stream = "0.3.3" +async-trait = "0.1.64" async_zip = "0.0.9" bincode = "1.3.1" bytes = "1.2.1" diff --git a/src/adapters.rs b/src/adapters.rs index 93bd5af..7f97050 100644 --- a/src/adapters.rs +++ b/src/adapters.rs @@ -1,11 +1,11 @@ pub mod custom; pub mod decompress; -// pub mod ffmpeg; +pub mod ffmpeg; pub mod postproc; use std::sync::Arc; pub mod sqlite; pub mod tar; -// pub mod writing; +pub mod writing; pub mod zip; use crate::{adapted_iter::AdaptedFilesIterBox, config::RgaConfig, matching::*}; use anyhow::{format_err, Context, Result}; @@ -112,11 +112,11 @@ pub fn get_all_adapters(custom_adapters: Option>) -> Ad let internal_adapters: Vec> = vec![ Arc::new(PostprocPageBreaks::default()), - //Rc::new(ffmpeg::FFmpegAdapter::new()), + Arc::new(ffmpeg::FFmpegAdapter::new()), Arc::new(zip::ZipAdapter::new()), Arc::new(decompress::DecompressAdapter::new()), Arc::new(tar::TarAdapter::new()), - //Rc::new(sqlite::SqliteAdapter::new()), + Arc::new(sqlite::SqliteAdapter::new()), ]; adapters.extend( BUILTIN_SPAWNING_ADAPTERS diff --git a/src/adapters/ffmpeg.rs b/src/adapters/ffmpeg.rs index a45c7ff..79ffc1c 100644 --- a/src/adapters/ffmpeg.rs +++ b/src/adapters/ffmpeg.rs @@ -1,12 +1,15 @@ -use super::spawning::map_exe_error; use super::*; +use super::{custom::map_exe_error, writing::async_writeln}; use anyhow::*; +use async_trait::async_trait; use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; -use std::io::BufReader; -use std::process::*; -use writing::{WritingFileAdapter, WritingFileAdapterTrait}; +use std::process::Stdio; +use tokio::io::AsyncWrite; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::process::Command; +use writing::WritingFileAdapter; // todo: // maybe todo: read list of extensions from // ffmpeg -demuxers | tail -n+5 | awk '{print $2}' | while read demuxer; do echo MUX=$demuxer; ffmpeg -h demuxer=$demuxer | grep 'Common extensions'; done 2>/dev/null @@ -33,8 +36,8 @@ lazy_static! { pub struct FFmpegAdapter; impl FFmpegAdapter { - pub fn new() -> WritingFileAdapter { - WritingFileAdapter::new(Box::new(FFmpegAdapter)) + pub fn new() -> FFmpegAdapter { + FFmpegAdapter } } impl GetMetadata for FFmpegAdapter { @@ -51,12 +54,13 @@ struct FFprobeOutput { struct FFprobeStream { codec_type: String, // video,audio,subtitle } -impl WritingFileAdapterTrait for FFmpegAdapter { - fn adapt_write( - &self, + +#[async_trait] +impl WritingFileAdapter for FFmpegAdapter { + async fn adapt_write( ai: AdaptInfo, _detection_reason: &FileMatcher, - oup: &mut dyn Write, + mut oup: Pin>, ) -> Result<()> { let AdaptInfo { is_real_file, @@ -69,7 +73,7 @@ impl WritingFileAdapterTrait for FFmpegAdapter { // it would require using a BufReader to read at least part of the file to memory // but really when would you want to search for videos within archives? // So instead, we only run this adapter if the file is a actual file on disk for now - writeln!(oup, "{}[rga: skipping video in archive]", line_prefix,)?; + async_writeln!(oup, "{line_prefix}[rga: skipping video in archive]\n")?; return Ok(()); } let inp_fname = filepath_hint; @@ -89,12 +93,13 @@ impl WritingFileAdapterTrait for FFmpegAdapter { .arg("-i") .arg(&inp_fname) .output() + .await .map_err(spawn_fail)?; if !probe.status.success() { return Err(format_err!("ffprobe failed: {:?}", probe.status)); } let p: FFprobeOutput = serde_json::from_slice(&probe.stdout)?; - p.streams.iter().count() > 0 + !p.streams.is_empty() }; { // extract file metadata (especially chapter names in a greppable format) @@ -117,10 +122,11 @@ impl WritingFileAdapterTrait for FFmpegAdapter { .arg(&inp_fname) .stdout(Stdio::piped()) .spawn()?; - for line in BufReader::new(probe.stdout.as_mut().unwrap()).lines() { - writeln!(oup, "metadata: {}", line?)?; + let mut lines = BufReader::new(probe.stdout.as_mut().unwrap()).lines(); + while let Some(line) = lines.next_line().await? { + async_writeln!(oup, "metadata: {line}")?; } - let exit = probe.wait()?; + let exit = probe.wait().await?; if !exit.success() { return Err(format_err!("ffprobe failed: {:?}", exit)); } @@ -141,15 +147,15 @@ impl WritingFileAdapterTrait for FFmpegAdapter { let time_re = Regex::new(r".*\d.*-->.*\d.*").unwrap(); let mut time: String = "".to_owned(); // rewrite subtitle times so they are shown as a prefix in every line - for line in BufReader::new(stdo).lines() { - let line = line?; + let mut lines = BufReader::new(stdo).lines(); + while let Some(line) = lines.next_line().await? { // 09:55.195 --> 09:56.730 if time_re.is_match(&line) { time = line.to_owned(); } else if line.is_empty() { - oup.write_all(b"\n")?; + async_writeln!(oup)?; } else { - writeln!(oup, "{}: {}", time, line)?; + async_writeln!(oup, "{time}: {line}")?; } } } diff --git a/src/adapters/sqlite.rs b/src/adapters/sqlite.rs index aaac120..ce677bf 100644 --- a/src/adapters/sqlite.rs +++ b/src/adapters/sqlite.rs @@ -39,6 +39,11 @@ lazy_static! { #[derive(Default, Clone)] pub struct SqliteAdapter; +impl SqliteAdapter { + pub fn new() -> SqliteAdapter { + SqliteAdapter + } +} impl GetMetadata for SqliteAdapter { fn metadata(&self) -> &AdapterMeta { &METADATA @@ -147,7 +152,6 @@ mod test { use super::*; use crate::test_utils::*; use pretty_assertions::assert_eq; - #[tokio::test] async fn simple() -> Result<()> { diff --git a/src/adapters/writing.rs b/src/adapters/writing.rs index 95dc0c0..3a61ea2 100644 --- a/src/adapters/writing.rs +++ b/src/adapters/writing.rs @@ -1,49 +1,94 @@ -use super::{FileAdapter, GetMetadata, ReadBox}; +use std::pin::Pin; + +use crate::adapted_iter::one_file; + +use super::{AdaptInfo, FileAdapter, GetMetadata}; use anyhow::Result; +use async_trait::async_trait; use tokio::io::AsyncWrite; -// use async_trait::async_trait; -pub trait WritingFileAdapter: GetMetadata + Send + Clone { - fn adapt_write( - &self, +#[async_trait] +pub trait WritingFileAdapter: GetMetadata + Send + Sync + Clone { + async fn adapt_write( a: super::AdaptInfo, detection_reason: &crate::matching::FileMatcher, - oup: &mut (dyn AsyncWrite), + oup: Pin>, ) -> Result<()>; } -/* struct PipedReadWriter { - inner: ReadBox, - pipe_thread: Thread, +macro_rules! async_writeln { + ($dst: expr) => { + { + tokio::io::AsyncWriteExt::write_all(&mut $dst, b"\n").await + } + }; + ($dst: expr, $fmt: expr) => { + { + use std::io::Write; + let mut buf = Vec::::new(); + writeln!(buf, $fmt)?; + tokio::io::AsyncWriteExt::write_all(&mut $dst, &buf).await + } + }; + ($dst: expr, $fmt: expr, $($arg: tt)*) => { + { + use std::io::Write; + let mut buf = Vec::::new(); + writeln!(buf, $fmt, $( $arg )*)?; + tokio::io::AsyncWriteExt::write_all(&mut $dst, &buf).await + } + }; } +pub(crate) use async_writeln; -impl<'a> Read for PipedReadWriter<'a> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - todo!() - } -}*/ - -impl FileAdapter for WritingFileAdapter { +impl FileAdapter for T +where + T: WritingFileAdapter, +{ fn adapt( + &self, + a: super::AdaptInfo, + detection_reason: &crate::matching::FileMatcher, + ) -> Result { + let (w, r) = tokio::io::duplex(128 * 1024); + let d2 = detection_reason.clone(); + let archive_recursion_depth = a.archive_recursion_depth + 1; + let filepath_hint = format!("{}.txt", a.filepath_hint.to_string_lossy()); + let postprocess = a.postprocess; + let line_prefix = a.line_prefix.clone(); + let config = a.config.clone(); + tokio::spawn(async move { + let x = d2; + T::adapt_write(a, &x, Box::pin(w)).await.unwrap() + }); + + Ok(one_file(AdaptInfo { + is_real_file: false, + filepath_hint: filepath_hint.into(), + archive_recursion_depth, + config, + inp: Box::pin(r), + line_prefix, + postprocess, + })) + } + /*fn adapt( &self, ai_outer: super::AdaptInfo, detection_reason: &crate::matching::FileMatcher, ) -> anyhow::Result { - let (r, w) = crate::pipe::pipe(); - let cc = self.inner.clone(); let detc = detection_reason.clone(); - panic!("ooo"); - // cc.adapt_write(ai_outer, detc, ) - /*tokio::spawn(move || { - let mut oup = w; - let ai = ai_outer; - let res = cc.adapt_write(ai, &detc, &mut oup); - if let Err(e) = res { - oup.write_err(std::io::Error::new(std::io::ErrorKind::Other, e)) - .expect("could not write err"); - } - }); */ + panic!("ooo");*/ + // cc.adapt_write(ai_outer, detc, ) + /*tokio::spawn(move || { + let mut oup = w; + let ai = ai_outer; + let res = cc.adapt_write(ai, &detc, &mut oup); + if let Err(e) = res { + oup.write_err(std::io::Error::new(std::io::ErrorKind::Other, e)) + .expect("could not write err"); + } + }); */ - //Ok(Box::new(r)) - } + //Ok(Box::new(r)) } diff --git a/src/adapters/zip.rs b/src/adapters/zip.rs index 8d4f848..e6ee1bc 100644 --- a/src/adapters/zip.rs +++ b/src/adapters/zip.rs @@ -57,7 +57,7 @@ impl FileAdapter for ZipAdapter { for i in 0..zip.entries().len() { let reader = zip.entry_reader(i).await?; let file = reader.entry(); - if file.filename().ends_with("/") { + if file.filename().ends_with('/') { continue; } debug!( @@ -101,7 +101,7 @@ impl FileAdapter for ZipAdapter { while !zip.finished() { if let Some(reader) = zip.entry_reader().await? { let file = reader.entry(); - if file.filename().ends_with("/") { + if file.filename().ends_with('/') { continue; } debug!( diff --git a/src/lib.rs b/src/lib.rs index d00a25a..af10567 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -81,7 +81,7 @@ fn init() { pub fn join_handle_to_stream(join: JoinHandle>) -> impl AsyncRead { let st = stream! { join.await.map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))??; - yield std::io::Result::Ok((&b""[..])) + yield std::io::Result::Ok(&b""[..]) }; StreamReader::new(st)