Removed warnings, started implementing apple mail

pull/1/head
Benedikt Terhechte 3 years ago
parent 2f34e13355
commit 6bdc412833

57
Cargo.lock generated

@ -117,6 +117,12 @@ dependencies = [
"rustc-demangle",
]
[[package]]
name = "base64"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -544,6 +550,16 @@ version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cca5179aa9d15128cebb79bb56dda73a79cc66b402056ff19a992e54b365e15c"
[[package]]
name = "emlx"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f41d97755f64845fb52dbff2acc01853ba51343e2076c4ab3099bb669d0e71e"
dependencies = [
"plist",
"thiserror",
]
[[package]]
name = "epaint"
version = "0.14.0"
@ -745,6 +761,7 @@ dependencies = [
"crossbeam-channel",
"eframe",
"email-parser",
"emlx",
"eyre",
"flate2",
"lazy_static",
@ -762,6 +779,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"treemap",
"walkdir",
]
[[package]]
@ -812,6 +830,16 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
[[package]]
name = "indexmap"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "instant"
version = "0.1.11"
@ -896,6 +924,15 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "line-wrap"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f30344350a2a51da54c1d53be93fade8a237e545dbcc4bdbe635413f2117cab9"
dependencies = [
"safemem",
]
[[package]]
name = "lock_api"
version = "0.4.5"
@ -1347,6 +1384,20 @@ version = "0.3.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
[[package]]
name = "plist"
version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a38d026d73eeaf2ade76309d0c65db5a35ecf649e3cec428db316243ea9d6711"
dependencies = [
"base64",
"chrono",
"indexmap",
"line-wrap",
"serde",
"xml-rs",
]
[[package]]
name = "ppv-lite86"
version = "0.2.10"
@ -1554,6 +1605,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "safemem"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
[[package]]
name = "same-file"
version = "1.0.6"

@ -27,6 +27,8 @@ num-format = "0.4.0"
strum = "0.21"
strum_macros = "0.21"
lru = { version = "0.7.0", optional = true }
emlx = "0.1"
walkdir = "*"
[features]
default = ["gui"]

@ -54,12 +54,12 @@ fn handle_adapter(adapter: &Adapter) -> Result<bool> {
} else {
let write = adapter.write_count()?;
if write.count > 0 {
print!("\rWriting to DB {}/{}...", write.count, write.total);
print!("\rWriting emails to DB {}/{}...", write.count, write.total);
} else {
let read = adapter.read_count()?;
print!(
"\rReading Emails {}%...",
((read.count as f32 / read.total as f32) * 100.0) as usize
(read.count as f32 / read.total as f32) * 100.0
);
}
}

@ -113,9 +113,7 @@ impl Database {
counter += 1;
insert_mail(&mut mail_prepared, &mail)
}
DBMessage::Error(report, path) => {
insert_error(&mut error_prepared, &report, &path)
}
DBMessage::Error(report) => insert_error(&mut error_prepared, &report),
DBMessage::Done => {
tracing::trace!("Received DBMessage::Done");
break;
@ -184,8 +182,8 @@ fn insert_mail(statement: &mut Statement, entry: &EmailEntry) -> Result<()> {
Ok(())
}
fn insert_error(statement: &mut Statement, message: &Report, path: &PathBuf) -> Result<()> {
statement.execute(params![message.to_string(), path.display().to_string()])?;
tracing::trace!("Insert Error {}", &path.display());
fn insert_error(statement: &mut Statement, message: &Report) -> Result<()> {
statement.execute(params![message.to_string()])?;
tracing::trace!("Insert Error {}", message);
Ok(())
}

@ -9,7 +9,7 @@ pub enum DBMessage {
/// Send for a successfuly parsed mail
Mail(EmailEntry),
/// Send for any kind of error during reading / parsing
Error(Report, PathBuf),
Error(Report),
/// Send once all parsing is done.
/// This is used to break out of the receiving loop
Done,

@ -21,8 +21,7 @@ CREATE TABLE IF NOT EXISTS emails (
pub const TBL_ERRORS: &str = r#"
CREATE TABLE IF NOT EXISTS errors (
message TEXT NOT NULL,
path TEXT NOT NULL
message TEXT NOT NULL
);"#;
pub const QUERY_EMAILS: &str = r#"

@ -0,0 +1,46 @@
//! We use a stubbornly stupid algorithm where we just
//! recursively drill down into the appropriate folder
//! until we find `emlx` files and return those.
use eyre::Result;
use rayon::prelude::*;
use tracing::trace;
use walkdir::WalkDir;
use super::super::shared::filesystem::folders_in;
use super::super::{Message, MessageSender};
use super::raw_email::RawEmailEntry;
use crate::types::Config;
use std::path::Path;
fn test_walkdir() {
for entry in WalkDir::new("foo").int_par_iter().filter_map(|e| e.ok()) {
println!("{}", entry.path().display());
}
}
pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
Ok(folders_in(&config.emails_folder_path, sender, read_folder)?)
}
fn read_folder(path: &Path, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
let result = Ok(std::fs::read_dir(path)?
.into_iter()
.par_bridge()
.filter_map(|entry| {
let path = entry
.map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
.ok()?
.path();
if path.is_dir() {
return None;
}
trace!("Reading {}", &path.display());
RawEmailEntry::new(path)
})
.collect());
// We're done reading the folder
sender.send(Message::ReadOne).unwrap();
result
}

@ -0,0 +1,64 @@
use emlx::parse_emlx;
use eyre::Result;
use std::borrow::Cow;
use std::path::{Path, PathBuf};
use super::super::shared::email::EmailMeta;
use super::super::shared::parse::ParseableEmail;
pub struct Mail {
path: PathBuf,
// This is parsed out of the `emlx` as it is parsed
is_seen: bool,
// This is parsed out of the `path`
label: Option<String>,
// Maildata
data: Vec<u8>,
}
impl Mail {
pub fn new(path: &Path) -> Result<Self> {
// find the folder ending with `.mbox` in the path
let ext = ".mbox";
let label = path
.iter()
.map(|e| e.to_str())
.flatten()
.find(|s| s.ends_with(ext))
.map(|s| s.replace(ext, "").to_string());
Ok(Self {
path: path.to_path_buf(),
is_seen: false,
label,
data: Vec::new(),
})
}
}
impl ParseableEmail for Mail {
fn prepare(&mut self) -> Result<()> {
let data = std::fs::read(self.path.as_path())?;
let parsed = parse_emlx(&data)?;
self.is_seen = !parsed.flags.is_read;
self.data = parsed.message.to_vec();
Ok(())
}
fn message<'a>(&'a self) -> Result<Cow<'a, [u8]>> {
Ok(Cow::Borrowed(self.data.as_slice()))
}
fn path(&self) -> &Path {
self.path.as_path()
}
fn meta(&self) -> Result<Option<EmailMeta>> {
let tags = match self.label {
Some(ref n) => vec![n.clone()],
None => vec![],
};
let meta = EmailMeta {
tags,
is_seen: self.is_seen,
};
Ok(Some(meta))
}
}

@ -1,2 +1,16 @@
mod mail;
/// FIXME: Not sure if the number changes with each macOS release?
const DEFAULT_FOLDER: &str = "~/Library/Mail/V8/";
use super::{Config, ImporterFormat, MessageSender, Result};
#[derive(Default)]
pub struct AppleMail {}
impl ImporterFormat for AppleMail {
type Item = mail::Mail;
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
panic!()
}
}

@ -14,7 +14,7 @@ pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<RawEmai
}
fn read_folder(path: &Path, sender: MessageSender) -> Result<Vec<RawEmailEntry>> {
Ok(std::fs::read_dir(path)?
let result = Ok(std::fs::read_dir(path)?
.into_iter()
.par_bridge()
.filter_map(|entry| {
@ -26,9 +26,10 @@ fn read_folder(path: &Path, sender: MessageSender) -> Result<Vec<RawEmailEntry>>
return None;
}
trace!("Reading {}", &path.display());
sender.send(Message::ReadOne).unwrap();
RawEmailEntry::new(path)
})
//.take(50)
.collect())
.collect());
// We're done reading the folder
sender.send(Message::ReadOne).unwrap();
result
}

@ -94,6 +94,9 @@ impl RawEmailEntry {
}
impl ParseableEmail for RawEmailEntry {
fn prepare(&mut self) -> Result<()> {
Ok(())
}
fn message<'a>(&'a self) -> Result<Cow<'a, [u8]>> {
Ok(Cow::Owned(self.read()?))
}

@ -1,9 +1,9 @@
use super::{shared, Config, ImporterFormat};
use super::{Message, MessageReceiver, MessageSender};
use super::MessageReceiver;
use crossbeam_channel::{self, unbounded, Receiver, Sender};
use eyre::{Report, Result};
use crossbeam_channel::{self, unbounded};
use eyre::Result;
use std::thread::JoinHandle;
pub struct Importer<'a, Format: ImporterFormat> {

@ -1,22 +1,15 @@
use super::email::{EmailEntry, EmailMeta};
use super::parse::{parse_email, ParseableEmail};
use crate::database::{DBMessage, Database};
use crate::types::Config;
use super::super::{Message, MessageSender};
use chrono::prelude::*;
use email_parser::address::{Address, EmailAddress, Mailbox};
use eyre::{bail, eyre, Result};
use eyre::{bail, Context, Result};
use rayon::prelude::*;
use std::thread::JoinHandle;
use std::convert::{TryFrom, TryInto};
use std::path::Path;
pub fn into_database<Mail: ParseableEmail + 'static>(
config: &Config,
emails: Vec<Mail>,
mut emails: Vec<Mail>,
tx: MessageSender,
) -> Result<usize> {
let total = emails.len();
@ -37,17 +30,23 @@ pub fn into_database<Mail: ParseableEmail + 'static>(
// Iterate over the mails..
emails
// in paralell..
.par_iter()
//.par_iter()
.par_iter_mut()
// parsing them
.map(|raw_mail| (raw_mail.path(), parse_email(raw_mail)))
.map(|raw_mail| {
// Due to lifetime issues, we can't use raw_mail.path() or raw_mail.path().display()
// or raw_mail.path().to_path_buf().display() as all of those retain a reference to
// `raw_mail`. So we just format the context into a string
parse_email(raw_mail).with_context(|| format!("{}", raw_mail.path().display()))
})
// and inserting them into SQLite
.for_each(|(path, entry)| {
.for_each(|entry| {
if let Err(e) = tx.send(Message::WriteOne) {
tracing::error!("Channel Failure: {:?}", &e);
}
if let Err(e) = match entry {
Ok(mail) => sender.send(DBMessage::Mail(mail)),
Err(e) => sender.send(DBMessage::Error(e, path.to_path_buf())),
Err(e) => sender.send(DBMessage::Error(e)),
} {
tracing::error!("Error Inserting into Database: {:?}", &e);
}

@ -1,12 +1,13 @@
use eyre::{bail, eyre, Result};
use eyre::{bail, Result};
use rayon::prelude::*;
use tracing::trace;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::path::Path;
use super::super::{Message, MessageSender};
/// Call `FolderAction` on all files in all sub folders in
/// folder `folder`.
pub fn folders_in<FolderAction, ActionResult, P>(
folder: P,
sender: MessageSender,

@ -1,6 +1,6 @@
use chrono::prelude::*;
use email_parser::address::{Address, EmailAddress, Mailbox};
use eyre::{bail, eyre, Result};
use eyre::{eyre, Result};
use std::borrow::{Borrow, Cow};
use std::convert::{TryFrom, TryInto};
@ -11,6 +11,9 @@ use super::email::{EmailEntry, EmailMeta};
/// Different `importer`s can implement this trait to provide the necessary
/// data to parse their data into a `EmailEntry`.
pub trait ParseableEmail: Send + Sized + Sync {
/// This will be called once before `message`, `path` and `meta`
/// are called. It can be used to perform parsing operations
fn prepare(&mut self) -> Result<()>;
/// The message content as bytes
fn message<'a>(&'a self) -> Result<Cow<'a, [u8]>>;
/// The original path of the email in the filesystem
@ -20,74 +23,63 @@ pub trait ParseableEmail: Send + Sized + Sync {
fn meta(&self) -> Result<Option<EmailMeta>>;
}
pub fn parse_email<Entry: ParseableEmail>(entry: &Entry) -> Result<EmailEntry> {
pub fn parse_email<Entry: ParseableEmail>(entry: &mut Entry) -> Result<EmailEntry> {
entry.prepare()?;
let content = entry.message()?;
parse_email_parser(entry, content.borrow())
}
fn parse_email_parser<Entry: ParseableEmail>(
raw_entry: &Entry,
content: &[u8],
) -> Result<EmailEntry> {
match email_parser::email::Email::parse(&content) {
Ok(email) => (raw_entry, email).try_into(),
Ok(email) => {
let path = entry.path();
let (sender_name, _, sender_local_part, sender_domain) =
mailbox_to_string(&email.sender);
let datetime = emaildatetime_to_chrono(&email.date);
let subject = email.subject.map(|e| e.to_string()).unwrap_or_default();
let to_count = match email.to.as_ref() {
Some(n) => n.len(),
None => 0,
};
let to = match email.to.as_ref().map(|v| v.first()).flatten() {
Some(n) => address_to_name_string(n),
None => None,
};
let to_group = to.as_ref().map(|e| e.0.clone()).flatten();
let to_first = to.as_ref().map(|e| (e.1.clone(), e.2.clone()));
let is_reply = email.in_reply_to.map(|v| !v.is_empty()).unwrap_or(false);
let meta = entry.meta()?;
// FIXME: This is filled out at a later stage
let is_send = false;
Ok(EmailEntry {
path: path.to_path_buf(),
sender_domain,
sender_local_part,
sender_name,
datetime,
subject,
meta,
is_reply,
to_count,
to_group,
to_first,
is_send,
})
}
Err(error) => {
//let content_string = String::from_utf8(content.clone())?;
//println!("{}|{}", &error, &raw_entry.eml_path.display());
Err(eyre!(
"Could not parse email: {:?} [{}]",
&error,
raw_entry.path().display()
entry.path().display()
))
}
}
}
impl<'a, Entry: ParseableEmail> TryFrom<(&Entry, email_parser::email::Email<'a>)> for EmailEntry {
type Error = eyre::Report;
fn try_from(content: (&Entry, email_parser::email::Email)) -> Result<Self, Self::Error> {
let (entry, email) = content;
let path = entry.path();
let (sender_name, _, sender_local_part, sender_domain) = mailbox_to_string(&email.sender);
let datetime = emaildatetime_to_chrono(&email.date);
let subject = email.subject.map(|e| e.to_string()).unwrap_or_default();
let to_count = match email.to.as_ref() {
Some(n) => n.len(),
None => 0,
};
let to = match email.to.as_ref().map(|v| v.first()).flatten() {
Some(n) => address_to_name_string(n),
None => None,
};
let to_group = to.as_ref().map(|e| e.0.clone()).flatten();
let to_first = to.as_ref().map(|e| (e.1.clone(), e.2.clone()));
let is_reply = email.in_reply_to.map(|v| !v.is_empty()).unwrap_or(false);
let meta = entry.meta()?;
// FIXME: This is filled out at a later stage
let is_send = false;
Ok(EmailEntry {
path: path.to_path_buf(),
sender_domain,
sender_local_part,
sender_name,
datetime,
subject,
meta,
is_reply,
to_count,
to_group,
to_first,
is_send,
})
}
}
/// Returns a conversion from address to the fields we care about:
/// ([group name], display name, email address)
fn address_to_name_string(address: &Address) -> Option<(Option<String>, String, String)> {

@ -4,8 +4,7 @@ use std::sync::{Arc, RwLock};
use std::thread::JoinHandle;
use super::formats::{Importer, ImporterFormat};
use super::{Message, MessageReceiver};
use crate::types::Config;
use super::Message;
#[derive(Clone, Debug, Copy, Default)]
struct Data {

Loading…
Cancel
Save