Add support for AppleMail

Cargo.lock

@@ -538,7 +538,7 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 [[package]]
 name = "email-parser"
 version = "0.5.0"
-source = "git+https://github.com/terhechte/email-parser#da8582a266385b8f1b2f7e150ac891e3f850d41a"
+source = "git+https://github.com/terhechte/email-parser#dba59d86771f7df67bb9e7f3a2c4b1e36b02d19b"
 dependencies = [
  "textcode",
  "timezone-abbreviations",
@@ -552,10 +552,10 @@ checksum = "cca5179aa9d15128cebb79bb56dda73a79cc66b402056ff19a992e54b365e15c"
 
 [[package]]
 name = "emlx"
-version = "0.1.0"
+version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
+source = "git+https://github.com/terhechte/emlx#44c2f278551d9e7a9ae0c3c3207c4471da3049fe"
-checksum = "5f41d97755f64845fb52dbff2acc01853ba51343e2076c4ab3099bb669d0e71e"
 dependencies = [
+ "email-parser",
  "plist",
  "thiserror",
 ]

Cargo.toml

@@ -13,7 +13,7 @@ tracing-subscriber = "0.2.24"
 rusqlite = {version = "0.25.3", features = ["chrono", "trace", "serde_json"]}
 regex = "1.5.3"
 flate2 = "1.0.22"
-email-parser = { git = "https://github.com/terhechte/email-parser", features = ["sender", "to", "in-reply-to", "date", "subject", "mime", "allow-duplicate-headers"]}
+email-parser = { git = "https://github.com/terhechte/email-parser", features = ["sender", "to", "in-reply-to", "date", "subject", "mime", "allow-duplicate-headers", "compatibility-fixes"]}
 rayon = "1.5.1"
 chrono = "0.4.19"
 lazy_static = "*"
@@ -27,7 +27,7 @@ num-format = "0.4.0"
 strum = "0.21"
 strum_macros = "0.21"
 lru = { version = "0.7.0", optional = true }
-emlx = "0.1"
+emlx = { git = "https://github.com/terhechte/emlx", features = []}
 walkdir = "*"
 
 [features]

src/database/database.rs

@@ -4,10 +4,7 @@ use eyre::{bail, Report, Result};
 use rusqlite::{self, params, Connection, Statement};
 
 use core::panic;
-use std::{
+use std::{path::Path, thread::JoinHandle};
-    path::{Path, PathBuf},
-    thread::JoinHandle,
-};
 
 use super::{query::Query, query_result::QueryResult, sql::*, DBMessage};
 use crate::{database::RowConversion, importer::EmailEntry};

src/database/db_message.rs

@@ -1,5 +1,3 @@
-use std::path::PathBuf;
-
 use eyre::Report;
 
 use crate::importer::EmailEntry;

src/database/sql.rs

@@ -45,7 +45,7 @@ VALUES
 
 pub const QUERY_ERRORS: &str = r#"
 INSERT INTO errors
-    (message, path)
+    (message)
 VALUES
-    (?, ?)
+    (?)
 "#;

src/importer/formats/apple_mail/filesystem.rs

@@ -4,43 +4,46 @@
 
 use eyre::Result;
 use rayon::prelude::*;
-use tracing::trace;
 use walkdir::WalkDir;
 
-use super::super::shared::filesystem::folders_in;
+use super::super::shared::filesystem::emails_in;
-use super::super::{Message, MessageSender};
+use super::super::MessageSender;
-use super::raw_email::RawEmailEntry;
 use crate::types::Config;
 
-use std::path::Path;
+use super::mail::Mail;
+use std::path::PathBuf;
 
-fn test_walkdir() {
+pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<Mail>> {
-    for entry in WalkDir::new("foo").int_par_iter().filter_map(|e| e.ok()) {
+    // As `walkdir` does not support `par_iter` (see https://www.reddit.com/r/rust/comments/6eif7r/walkdir_users_we_need_you/)
-        println!("{}", entry.path().display());
+    // - -we first collect all folders,
-    }
+    // then all sub-folders in those ending in mboxending in .mbox and then iterate over them in paralell
-}
+    let folders: Vec<PathBuf> = WalkDir::new(&config.emails_folder_path)
-
-pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
-    Ok(folders_in(&config.emails_folder_path, sender, read_folder)?)
-}
-
-fn read_folder(path: &Path, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
-    let result = Ok(std::fs::read_dir(path)?
         .into_iter()
-        .par_bridge()
+        .filter_map(|e| match e {
-        .filter_map(|entry| {
+            Ok(n)
-            let path = entry
+                if n.path().is_dir()
-                .map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
+                    && n.path()
-                .ok()?
+                        .to_str()
-                .path();
+                        .map(|e| e.contains(".mbox"))
-            if path.is_dir() {
+                        .unwrap_or(false) =>
-                return None;
+            {
+                Some(n.path().to_path_buf())
             }
-            trace!("Reading {}", &path.display());
+            _ => None,
-            RawEmailEntry::new(path)
         })
-        .collect());
+        .collect();
-    // We're done reading the folder
+    let mails = folders
-    sender.send(Message::ReadOne).unwrap();
+        .into_par_iter()
-    result
+        .filter_map(
+            |path| match emails_in(path.clone(), sender.clone(), Mail::new) {
+                Ok(n) => Some(n),
+                Err(e) => {
+                    tracing::error!("{} {:?}", path.display(), &e);
+                    None
+                }
+            },
+        )
+        .flatten()
+        .collect();
+    Ok(mails)
 }

src/importer/formats/apple_mail/mail.rs

@@ -18,7 +18,12 @@ pub struct Mail {
 }
 
 impl Mail {
-    pub fn new(path: &Path) -> Result<Self> {
+    pub fn new<P: AsRef<Path>>(path: P) -> Option<Self> {
+        let path = path.as_ref();
+        let name = path.file_name()?.to_str()?;
+        if !name.ends_with(".emlx") {
+            return None;
+        }
         // find the folder ending with `.mbox` in the path
         let ext = ".mbox";
         let label = path
@@ -27,7 +32,7 @@ impl Mail {
             .flatten()
             .find(|s| s.ends_with(ext))
             .map(|s| s.replace(ext, "").to_string());
-        Ok(Self {
+        Some(Self {
             path: path.to_path_buf(),
             is_seen: false,
             label,

src/importer/formats/apple_mail/mod.rs

@@ -1,8 +1,6 @@
+mod filesystem;
 mod mail;
 
-/// FIXME: Not sure if the number changes with each macOS release?
-const DEFAULT_FOLDER: &str = "~/Library/Mail/V8/";
-
 use super::{Config, ImporterFormat, MessageSender, Result};
 
 #[derive(Default)]
@@ -11,6 +9,6 @@ pub struct AppleMail {}
 impl ImporterFormat for AppleMail {
     type Item = mail::Mail;
     fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
-        panic!()
+        filesystem::read_emails(config, sender)
     }
 }

src/importer/formats/gmailbackup/filesystem.rs

@@ -1,35 +1,14 @@
 use eyre::Result;
-use rayon::prelude::*;
-use tracing::trace;
 
-use super::super::shared::filesystem::folders_in;
+use super::super::shared::filesystem::{emails_in, folders_in};
-use super::super::{Message, MessageSender};
+use super::super::MessageSender;
 use super::raw_email::RawEmailEntry;
 use crate::types::Config;
 
-use std::path::Path;
-
 pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
-    Ok(folders_in(&config.emails_folder_path, sender, read_folder)?)
+    Ok(folders_in(
-}
+        &config.emails_folder_path,
-
+        sender,
-fn read_folder(path: &Path, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
+        |path, sender| emails_in(path, sender, RawEmailEntry::new),
-    let result = Ok(std::fs::read_dir(path)?
+    )?)
-        .into_iter()
-        .par_bridge()
-        .filter_map(|entry| {
-            let path = entry
-                .map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
-                .ok()?
-                .path();
-            if path.is_dir() {
-                return None;
-            }
-            trace!("Reading {}", &path.display());
-            RawEmailEntry::new(path)
-        })
-        .collect());
-    // We're done reading the folder
-    sender.send(Message::ReadOne).unwrap();
-    result
 }

src/importer/formats/mod.rs

@@ -5,6 +5,7 @@ mod gmailbackup;
 mod importer;
 pub mod shared;
 
+pub use apple_mail::AppleMail;
 pub use gmailbackup::Gmail;
 
 pub use crate::types::Config;

src/importer/formats/shared/database.rs

@@ -4,7 +4,7 @@ use crate::types::Config;
 
 use super::super::{Message, MessageSender};
 
-use eyre::{bail, Context, Result};
+use eyre::{bail, Result};
 use rayon::prelude::*;
 
 pub fn into_database<Mail: ParseableEmail + 'static>(
@@ -33,23 +33,20 @@ pub fn into_database<Mail: ParseableEmail + 'static>(
         //.par_iter()
         .par_iter_mut()
         // parsing them
-        .map(|raw_mail| {
+        .map(|raw_mail| parse_email(raw_mail))
-            // Due to lifetime issues, we can't use raw_mail.path() or raw_mail.path().display()
-            // or raw_mail.path().to_path_buf().display() as all of those retain a reference to
-            // `raw_mail`. So we just format the context into a string
-            parse_email(raw_mail).with_context(|| format!("{}", raw_mail.path().display()))
-        })
         // and inserting them into SQLite
         .for_each(|entry| {
-            if let Err(e) = tx.send(Message::WriteOne) {
+            // Try to write the message into the database
-                tracing::error!("Channel Failure: {:?}", &e);
-            }
             if let Err(e) = match entry {
                 Ok(mail) => sender.send(DBMessage::Mail(mail)),
                 Err(e) => sender.send(DBMessage::Error(e)),
             } {
                 tracing::error!("Error Inserting into Database: {:?}", &e);
             }
+            // Signal the write
+            if let Err(e) = tx.send(Message::WriteOne) {
+                tracing::error!("Channel Failure: {:?}", &e);
+            }
         });
 
     // Tell SQLite there's no more work coming. This will exit the listening loop

src/importer/formats/shared/filesystem.rs

@@ -2,7 +2,7 @@ use eyre::{bail, Result};
 use rayon::prelude::*;
 use tracing::trace;
 
-use std::path::Path;
+use std::path::{Path, PathBuf};
 
 use super::super::{Message, MessageSender};
 
@@ -15,7 +15,7 @@ pub fn folders_in<FolderAction, ActionResult, P>(
 ) -> Result<Vec<ActionResult>>
 where
     P: AsRef<Path>,
-    FolderAction: Fn(&Path, MessageSender) -> Result<Vec<ActionResult>> + Send + Sync,
+    FolderAction: Fn(PathBuf, MessageSender) -> Result<Vec<ActionResult>> + Send + Sync,
     ActionResult: Send,
 {
     let folder = folder.as_ref();
@@ -40,10 +40,37 @@ where
             }
             let sender = sender.clone();
             trace!("Reading folder {}", path.display());
-            action(&path, sender)
+            action(path.clone(), sender)
-                .map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
+                .map_err(|e| tracing::error!("{} {:?}", path.display(), &e))
                 .ok()
         })
         .flatten()
         .collect())
 }
+
+pub fn emails_in<O, F, P: AsRef<Path>>(path: P, sender: MessageSender, make: F) -> Result<Vec<O>>
+where
+    F: Fn(PathBuf) -> Option<O>,
+    F: Send + Sync + 'static,
+    O: Send + Sync,
+{
+    let path = path.as_ref();
+    let result = Ok(std::fs::read_dir(path)?
+        .into_iter()
+        .par_bridge()
+        .filter_map(|entry| {
+            let path = entry
+                .map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
+                .ok()?
+                .path();
+            if path.is_dir() {
+                return None;
+            }
+            trace!("Reading {}", &path.display());
+            make(path)
+        })
+        .collect());
+    // We're done reading the folder
+    sender.send(Message::ReadOne).unwrap();
+    result
+}

src/importer/formats/shared/parse.rs

@@ -2,8 +2,7 @@ use chrono::prelude::*;
 use email_parser::address::{Address, EmailAddress, Mailbox};
 use eyre::{eyre, Result};
 
-use std::borrow::{Borrow, Cow};
+use std::borrow::Cow;
-use std::convert::{TryFrom, TryInto};
 use std::path::Path;
 
 use super::email::{EmailEntry, EmailMeta};
@@ -24,11 +23,15 @@ pub trait ParseableEmail: Send + Sized + Sync {
 }
 
 pub fn parse_email<Entry: ParseableEmail>(entry: &mut Entry) -> Result<EmailEntry> {
-    entry.prepare()?;
+    if let Err(e) = entry.prepare() {
+        tracing::error!("Prepare Error: {:?}", e);
+        return Err(e);
+    }
     let content = entry.message()?;
     match email_parser::email::Email::parse(&content) {
         Ok(email) => {
             let path = entry.path();
+            tracing::trace!("Parsing {}", path.display());
             let (sender_name, _, sender_local_part, sender_domain) =
                 mailbox_to_string(&email.sender);
 
@@ -69,13 +72,18 @@ pub fn parse_email<Entry: ParseableEmail>(entry: &mut Entry) -> Result<EmailEntr
             })
         }
         Err(error) => {
-            //let content_string = String::from_utf8(content.clone())?;
+            let error = eyre!(
-            //println!("{}|{}", &error, &raw_entry.eml_path.display());
+                "Could not parse email (trace to see contents): {:?} [{}]",
-            Err(eyre!(
-                "Could not parse email: {:?} [{}]",
                 &error,
                 entry.path().display()
-            ))
+            );
+            tracing::error!("{:?}", &error);
+            if let Ok(content_string) = String::from_utf8(content.into_owned()) {
+                tracing::trace!("Contents:\n{}\n---\n", content_string);
+            } else {
+                tracing::trace!("Contents:\nInvalid UTF8\n---\n");
+            }
+            Err(error)
         }
     }
 }

src/importer/mod.rs

@@ -38,3 +38,7 @@ pub type MessageReceiver = crossbeam_channel::Receiver<Message>;
 pub fn gmail_importer(config: &Config) -> Importer<formats::Gmail> {
     Importer::new(config, formats::Gmail::default())
 }
+
+pub fn applemail_importer(config: &Config) -> Importer<formats::AppleMail> {
+    Importer::new(config, formats::AppleMail::default())
+}