Somehow it is much faster now..

README.md

@@ -3,6 +3,7 @@
 ## Fast
 
 It currently parses 632383 emails in ~160 seconds, so roughly `4.000` emails per second. This excludes (for now) attachments.
+Update: It currently parses 632115 emails in ~56 seconds, so roughly `11.000` emails per second. This excludes (for now) attachments.
 
 ## Open Issues
 - [ ] make the rectangles look nicer
@@ -13,4 +14,4 @@ It currently parses 632383 emails in ~160 seconds, so roughly `4.000` emails per
 - [ ] support apple mail
 - [ ] try `iced` or `druid` as well
 - [ ] maybe add blocking versions of the calls, too (in model)
-- [ ] abstract over `Fields` and backend
+- [ ] abstract over `Fields` and backend

src/bin/cli.rs

@@ -8,7 +8,7 @@ use std::{
 
 use gmaildb::{
     self,
-    importer::{Progress, State},
+    importer::{Adapter, State},
 };
 
 fn main() -> Result<()> {
@@ -23,31 +23,15 @@ fn main() -> Result<()> {
     let mut stdout = stdout();
 
     loop {
-        match adapter.finished() {
-            Ok(State {
-                finishing: true, ..
-            }) => {
-                println!("Finishing import...");
+        match handle_adapter(&adapter) {
+            Ok(true) => break,
+            Ok(false) => (),
+            Err(e) => {
+                println!("Execution Error:\n{:?}", &e);
+                panic!();
             }
-            Ok(State { done: true, .. }) => {
-                break;
-            }
-            _ => (),
-        };
-        match adapter.read_count() {
-            Ok(Progress { count, total }) => {
-                print!("\rReading {}/{}...", count, total);
-            }
-            _ => (),
-        };
-        match adapter.write_count() {
-            Ok(Progress { count, total }) => {
-                print!("\rWriting to DB {}/{}...", count, total);
-            }
-            _ => (),
-        };
+        }
         stdout.flush().unwrap();
-        sleep(Duration::from_millis(50));
     }
 
     match handle.join() {
@@ -55,6 +39,30 @@ fn main() -> Result<()> {
         Ok(Err(e)) => println!("Error: {:?}", e),
         _ => (),
     }
+    println!("\rDone");
 
     Ok(())
 }
+
+fn handle_adapter(adapter: &Adapter) -> Result<bool> {
+    let State { done, finishing } = adapter.finished()?;
+    if done {
+        return Ok(true);
+    }
+    if finishing {
+        print!("\rFinishing up...");
+    } else {
+        let write = adapter.write_count()?;
+        if write.count > 0 {
+            print!("\rWriting to DB {}/{}...", write.count, write.total);
+        } else {
+            let read = adapter.read_count()?;
+            print!(
+                "\rReading Emails {}%...",
+                ((read.count as f32 / read.total as f32) * 100.0) as usize
+            );
+        }
+    }
+    sleep(Duration::from_millis(50));
+    Ok(false)
+}

src/database/database.rs

@@ -29,10 +29,9 @@ impl Database {
 
         Self::create_tables(&connection)?;
 
-        //#[cfg(feature = "trace-sql")]
+        #[cfg(feature = "trace-sql")]
         connection.trace(Some(|query| {
-            //tracing::trace!("SQL: {}", &query);
-            println!("SQL: {}", &query);
+            tracing::trace!("SQL: {}", &query);
         }));
 
         Ok(Database {

src/importer/formats/gmailbackup/filesystem.rs

@@ -1,5 +1,6 @@
 use eyre::Result;
 use rayon::prelude::*;
+use tracing::trace;
 
 use super::super::shared::filesystem::folders_in;
 use super::super::{Message, MessageSender};
@@ -24,7 +25,8 @@ fn read_folder(path: &Path, sender: MessageSender) -> Result<Vec<RawEmailEntry>>
             if path.is_dir() {
                 return None;
             }
-            sender.send(Message::ReadOne);
+            trace!("Reading {}", &path.display());
+            sender.send(Message::ReadOne).unwrap();
             RawEmailEntry::new(path)
         })
         //.take(50)

src/importer/formats/importer.rs

@@ -17,7 +17,7 @@ impl<'a, Format: ImporterFormat + 'static> Importer<'a, Format> {
     }
 
     pub fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<usize>>)> {
-        let Importer { config, format } = self;
+        let Importer { format, .. } = self;
         let (sender, receiver) = unbounded();
 
         let config = self.config.clone();

src/importer/formats/shared/database.rs

@@ -65,6 +65,7 @@ pub fn into_database<Mail: ParseableEmail + 'static>(
         bail!("Channel Failure {:?}", &e);
     }
 
+    tracing::trace!("Waiting for database handle...");
     let output = match handle.join() {
         Ok(Ok(count)) => Ok(count),
         Ok(Err(e)) => Err(e),
@@ -73,6 +74,7 @@ pub fn into_database<Mail: ParseableEmail + 'static>(
 
     // Tell the caller that we're done processing. This will allow leaving the
     // display loop
+    tracing::trace!("Messaging Done");
     if let Err(e) = tx.send(Message::Done) {
         bail!("Channel Failure {:?}", &e);
     }

src/importer/formats/shared/filesystem.rs

@@ -1,5 +1,6 @@
 use eyre::{bail, eyre, Result};
 use rayon::prelude::*;
+use tracing::trace;
 
 use std::io::Read;
 use std::path::{Path, PathBuf};
@@ -37,6 +38,7 @@ where
                 return None;
             }
             let sender = sender.clone();
+            trace!("Reading folder {}", path.display());
             action(&path, sender)
                 .map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
                 .ok()