|
|
|
@ -2,7 +2,6 @@ use super::*;
|
|
|
|
|
use crate::print_bytes;
|
|
|
|
|
use anyhow::*;
|
|
|
|
|
use async_stream::stream;
|
|
|
|
|
use async_zip::read::stream::ZipFileReader;
|
|
|
|
|
use lazy_static::lazy_static;
|
|
|
|
|
use log::*;
|
|
|
|
|
|
|
|
|
@ -47,13 +46,16 @@ impl FileAdapter for ZipAdapter {
|
|
|
|
|
postprocess,
|
|
|
|
|
line_prefix,
|
|
|
|
|
config,
|
|
|
|
|
is_real_file,
|
|
|
|
|
..
|
|
|
|
|
} = ai;
|
|
|
|
|
let mut zip = ZipFileReader::new(inp);
|
|
|
|
|
if is_real_file {
|
|
|
|
|
use async_zip::read::fs::ZipFileReader;
|
|
|
|
|
|
|
|
|
|
let s = stream! {
|
|
|
|
|
while !zip.finished() {
|
|
|
|
|
if let Some(reader) = zip.entry_reader().await? {
|
|
|
|
|
let s = stream! {
|
|
|
|
|
let zip = ZipFileReader::new(&filepath_hint).await?;
|
|
|
|
|
for i in 0..zip.entries().len() {
|
|
|
|
|
let reader = zip.entry_reader(i).await?;
|
|
|
|
|
let file = reader.entry();
|
|
|
|
|
if file.filename().ends_with("/") {
|
|
|
|
|
continue;
|
|
|
|
@ -88,10 +90,55 @@ impl FileAdapter for ZipAdapter {
|
|
|
|
|
config: config.clone(),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
Ok(Box::pin(s))
|
|
|
|
|
} else {
|
|
|
|
|
use async_zip::read::stream::ZipFileReader;
|
|
|
|
|
let mut zip = ZipFileReader::new(inp);
|
|
|
|
|
|
|
|
|
|
let s = stream! {
|
|
|
|
|
while !zip.finished() {
|
|
|
|
|
if let Some(reader) = zip.entry_reader().await? {
|
|
|
|
|
let file = reader.entry();
|
|
|
|
|
if file.filename().ends_with("/") {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
debug!(
|
|
|
|
|
"{}{}|{}: {} ({} packed)",
|
|
|
|
|
line_prefix,
|
|
|
|
|
filepath_hint.display(),
|
|
|
|
|
file.filename(),
|
|
|
|
|
print_bytes(file.uncompressed_size() as f64),
|
|
|
|
|
print_bytes(file.compressed_size() as f64)
|
|
|
|
|
);
|
|
|
|
|
let new_line_prefix = format!("{}{}: ", line_prefix, file.filename());
|
|
|
|
|
let fname = PathBuf::from(file.filename());
|
|
|
|
|
tokio::pin!(reader);
|
|
|
|
|
// SAFETY: this should be solvable without unsafe but idk how :(
|
|
|
|
|
// the issue is that ZipEntryReader borrows from ZipFileReader, but we need to yield it here into the stream
|
|
|
|
|
// but then it can't borrow from the ZipFile
|
|
|
|
|
let reader2 = unsafe {
|
|
|
|
|
std::intrinsics::transmute::<
|
|
|
|
|
Pin<&mut (dyn AsyncRead + Send)>,
|
|
|
|
|
Pin<&'static mut (dyn AsyncRead + Send)>,
|
|
|
|
|
>(reader)
|
|
|
|
|
};
|
|
|
|
|
yield Ok(AdaptInfo {
|
|
|
|
|
filepath_hint: fname,
|
|
|
|
|
is_real_file: false,
|
|
|
|
|
inp: Box::pin(reader2),
|
|
|
|
|
line_prefix: new_line_prefix,
|
|
|
|
|
archive_recursion_depth: archive_recursion_depth + 1,
|
|
|
|
|
postprocess,
|
|
|
|
|
config: config.clone(),
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
Ok(Box::pin(s))
|
|
|
|
|
Ok(Box::pin(s))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@ -163,14 +210,23 @@ mod test {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn only_seek_zip() -> Result<()> {
|
|
|
|
|
async fn only_seek_zip_fs() -> Result<()> {
|
|
|
|
|
let zip = test_data_dir().join("only-seek-zip.zip");
|
|
|
|
|
let (a, d) = simple_adapt_info(&zip, Box::pin(File::open(&zip).await?));
|
|
|
|
|
let (a, d) = simple_fs_adapt_info(&zip).await?;
|
|
|
|
|
let v = adapted_to_vec(loop_adapt(&ZipAdapter::new(), d, a)?).await?;
|
|
|
|
|
assert_eq!(String::from_utf8(v)?, "");
|
|
|
|
|
// assert_eq!(String::from_utf8(v)?, "");
|
|
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
/*#[tokio::test]
|
|
|
|
|
async fn only_seek_zip_mem() -> Result<()> {
|
|
|
|
|
let zip = test_data_dir().join("only-seek-zip.zip");
|
|
|
|
|
let (a, d) = simple_adapt_info(&zip, Box::pin(File::open(&zip).await?));
|
|
|
|
|
let v = adapted_to_vec(loop_adapt(&ZipAdapter::new(), d, a)?).await?;
|
|
|
|
|
// assert_eq!(String::from_utf8(v)?, "");
|
|
|
|
|
|
|
|
|
|
Ok(())
|
|
|
|
|
}*/
|
|
|
|
|
#[tokio::test]
|
|
|
|
|
async fn recurse() -> Result<()> {
|
|
|
|
|
let zipfile = create_zip("outer.txt", "outer text file", true).await?;
|
|
|
|
|