From 2f34e13355d81007de338551a5229e130249822c Mon Sep 17 00:00:00 2001 From: Benedikt Terhechte Date: Fri, 8 Oct 2021 15:24:22 +0200 Subject: [PATCH] Somehow it is much faster now.. --- README.md | 3 +- src/bin/cli.rs | 56 +++++++++++-------- src/database/database.rs | 5 +- .../formats/gmailbackup/filesystem.rs | 4 +- src/importer/formats/importer.rs | 2 +- src/importer/formats/shared/database.rs | 2 + src/importer/formats/shared/filesystem.rs | 2 + 7 files changed, 44 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index ad71855..81bf859 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ ## Fast It currently parses 632383 emails in ~160 seconds, so roughly `4.000` emails per second. This excludes (for now) attachments. +Update: It currently parses 632115 emails in ~56 seconds, so roughly `11.000` emails per second. This excludes (for now) attachments. ## Open Issues - [ ] make the rectangles look nicer @@ -13,4 +14,4 @@ It currently parses 632383 emails in ~160 seconds, so roughly `4.000` emails per - [ ] support apple mail - [ ] try `iced` or `druid` as well - [ ] maybe add blocking versions of the calls, too (in model) -- [ ] abstract over `Fields` and backend \ No newline at end of file +- [ ] abstract over `Fields` and backend diff --git a/src/bin/cli.rs b/src/bin/cli.rs index 6a9d56e..59e7077 100644 --- a/src/bin/cli.rs +++ b/src/bin/cli.rs @@ -8,7 +8,7 @@ use std::{ use gmaildb::{ self, - importer::{Progress, State}, + importer::{Adapter, State}, }; fn main() -> Result<()> { @@ -23,31 +23,15 @@ fn main() -> Result<()> { let mut stdout = stdout(); loop { - match adapter.finished() { - Ok(State { - finishing: true, .. - }) => { - println!("Finishing import..."); + match handle_adapter(&adapter) { + Ok(true) => break, + Ok(false) => (), + Err(e) => { + println!("Execution Error:\n{:?}", &e); + panic!(); } - Ok(State { done: true, .. }) => { - break; - } - _ => (), - }; - match adapter.read_count() { - Ok(Progress { count, total }) => { - print!("\rReading {}/{}...", count, total); - } - _ => (), - }; - match adapter.write_count() { - Ok(Progress { count, total }) => { - print!("\rWriting to DB {}/{}...", count, total); - } - _ => (), - }; + } stdout.flush().unwrap(); - sleep(Duration::from_millis(50)); } match handle.join() { @@ -55,6 +39,30 @@ fn main() -> Result<()> { Ok(Err(e)) => println!("Error: {:?}", e), _ => (), } + println!("\rDone"); Ok(()) } + +fn handle_adapter(adapter: &Adapter) -> Result { + let State { done, finishing } = adapter.finished()?; + if done { + return Ok(true); + } + if finishing { + print!("\rFinishing up..."); + } else { + let write = adapter.write_count()?; + if write.count > 0 { + print!("\rWriting to DB {}/{}...", write.count, write.total); + } else { + let read = adapter.read_count()?; + print!( + "\rReading Emails {}%...", + ((read.count as f32 / read.total as f32) * 100.0) as usize + ); + } + } + sleep(Duration::from_millis(50)); + Ok(false) +} diff --git a/src/database/database.rs b/src/database/database.rs index 22480ac..913b876 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -29,10 +29,9 @@ impl Database { Self::create_tables(&connection)?; - //#[cfg(feature = "trace-sql")] + #[cfg(feature = "trace-sql")] connection.trace(Some(|query| { - //tracing::trace!("SQL: {}", &query); - println!("SQL: {}", &query); + tracing::trace!("SQL: {}", &query); })); Ok(Database { diff --git a/src/importer/formats/gmailbackup/filesystem.rs b/src/importer/formats/gmailbackup/filesystem.rs index ea3d7ff..ea36ba3 100644 --- a/src/importer/formats/gmailbackup/filesystem.rs +++ b/src/importer/formats/gmailbackup/filesystem.rs @@ -1,5 +1,6 @@ use eyre::Result; use rayon::prelude::*; +use tracing::trace; use super::super::shared::filesystem::folders_in; use super::super::{Message, MessageSender}; @@ -24,7 +25,8 @@ fn read_folder(path: &Path, sender: MessageSender) -> Result> if path.is_dir() { return None; } - sender.send(Message::ReadOne); + trace!("Reading {}", &path.display()); + sender.send(Message::ReadOne).unwrap(); RawEmailEntry::new(path) }) //.take(50) diff --git a/src/importer/formats/importer.rs b/src/importer/formats/importer.rs index 268f161..85a715d 100644 --- a/src/importer/formats/importer.rs +++ b/src/importer/formats/importer.rs @@ -17,7 +17,7 @@ impl<'a, Format: ImporterFormat + 'static> Importer<'a, Format> { } pub fn import(self) -> Result<(MessageReceiver, JoinHandle>)> { - let Importer { config, format } = self; + let Importer { format, .. } = self; let (sender, receiver) = unbounded(); let config = self.config.clone(); diff --git a/src/importer/formats/shared/database.rs b/src/importer/formats/shared/database.rs index 19e7752..c93a21a 100644 --- a/src/importer/formats/shared/database.rs +++ b/src/importer/formats/shared/database.rs @@ -65,6 +65,7 @@ pub fn into_database( bail!("Channel Failure {:?}", &e); } + tracing::trace!("Waiting for database handle..."); let output = match handle.join() { Ok(Ok(count)) => Ok(count), Ok(Err(e)) => Err(e), @@ -73,6 +74,7 @@ pub fn into_database( // Tell the caller that we're done processing. This will allow leaving the // display loop + tracing::trace!("Messaging Done"); if let Err(e) = tx.send(Message::Done) { bail!("Channel Failure {:?}", &e); } diff --git a/src/importer/formats/shared/filesystem.rs b/src/importer/formats/shared/filesystem.rs index 33f94d4..23716aa 100644 --- a/src/importer/formats/shared/filesystem.rs +++ b/src/importer/formats/shared/filesystem.rs @@ -1,5 +1,6 @@ use eyre::{bail, eyre, Result}; use rayon::prelude::*; +use tracing::trace; use std::io::Read; use std::path::{Path, PathBuf}; @@ -37,6 +38,7 @@ where return None; } let sender = sender.clone(); + trace!("Reading folder {}", path.display()); action(&path, sender) .map_err(|e| tracing::error!("{} {:?}", &path.display(), &e)) .ok()