mirror of https://github.com/terhechte/postsack
Initial Workspace restructuring compiles
Benedikt Terhechte
2 years ago
parent
7a0be16578
commit
5dfd1b9630
@ -1,2 +1,3 @@
|
||||
/target
|
||||
target
|
||||
.DS_Store
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,3 @@
|
||||
/target
|
||||
target
|
||||
.DS_Store
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "postsack"
|
||||
version = "0.2.0"
|
||||
edition = "2021"
|
||||
description = "Provides a high level visual overview of swaths of email"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[package.metadata.bundle]
|
||||
name = "Postsack"
|
||||
identifier = "com.stylemac.postsack"
|
||||
icon = ["icons/Icon.icns", "icons/icon-win-256.png", "icons/icon-win-32.png", "icons/icon-win-16.png"]
|
||||
version = "1.0.0"
|
||||
copyright = "Copyright (c) Benedikt Terhechte (2021). All rights reserved."
|
||||
category = "Developer Tool"
|
||||
short_description = "Provides a high level visual overview of swaths of email"
|
||||
osx_minimum_system_version = "10.14"
|
||||
|
||||
[dependencies]
|
||||
ps-gui = { path = "../ps-gui" }
|
||||
ps-core = { path = "../ps-core" }
|
||||
|
||||
[profile.dev]
|
||||
split-debuginfo = "unpacked"
|
||||
|
||||
#[profile.release]
|
||||
#lto = "fat"
|
||||
#codegen-units = 1
|
||||
#panic = "abort"
|
||||
@ -0,0 +1,6 @@
|
||||
fn main() {
|
||||
#[cfg(debug_assertions)]
|
||||
ps_core::setup_tracing();
|
||||
|
||||
ps_gui::run_ui();
|
||||
}
|
||||
@ -0,0 +1,3 @@
|
||||
/target
|
||||
target
|
||||
.DS_Store
|
||||
@ -1,13 +1,29 @@
|
||||
mod database;
|
||||
mod importer;
|
||||
mod model;
|
||||
pub mod message_adapter;
|
||||
pub mod model;
|
||||
mod types;
|
||||
|
||||
pub use database::database_like::DatabaseLike;
|
||||
pub use database::db_message::DBMessage;
|
||||
pub use database::query::{Field, OtherQuery, Query, ValueField, AMOUNT_FIELD_NAME};
|
||||
pub use database::query_result::QueryResult;
|
||||
pub use database::query::{Field, Filter, OtherQuery, Query, ValueField, AMOUNT_FIELD_NAME};
|
||||
pub use database::query_result::{QueryResult, QueryRow};
|
||||
pub use types::{Config, EmailEntry, EmailMeta, FormatType};
|
||||
|
||||
pub use crossbeam_channel;
|
||||
pub use importer::{Importerlike, Message, MessageReceiver, MessageSender};
|
||||
|
||||
// Tracing
|
||||
|
||||
use tracing_subscriber::fmt;
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
pub fn setup_tracing() {
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
std::env::set_var("RUST_LOG", "error")
|
||||
}
|
||||
|
||||
let collector = tracing_subscriber::registry().with(fmt::layer().with_writer(std::io::stdout));
|
||||
|
||||
tracing::subscriber::set_global_default(collector).expect("Unable to set a global collector");
|
||||
}
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
/target
|
||||
target
|
||||
.DS_Store
|
||||
@ -0,0 +1,3 @@
|
||||
/target
|
||||
target
|
||||
.DS_Store
|
||||
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,32 @@
|
||||
[package]
|
||||
name = "ps-gui"
|
||||
version = "0.2.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
eyre = "0.6.5"
|
||||
thiserror = "1.0.29"
|
||||
tracing = "0.1.29"
|
||||
tracing-subscriber = "0.3.0"
|
||||
once_cell = "1.8.0"
|
||||
crossbeam-channel = "0.5.1"
|
||||
eframe = "0.15.0"
|
||||
num-format = "0.4.0"
|
||||
tinyfiledialogs = "3.0"
|
||||
rand = "0.8.4"
|
||||
image = { version = "0.23", default-features = false, features = ["png"] }
|
||||
chrono = "0.4.19"
|
||||
shellexpand = "2.1.0"
|
||||
ps-core = { path = "../ps-core" }
|
||||
|
||||
[target."cfg(target_os = \"macos\")".dependencies.cocoa]
|
||||
version = "0.24"
|
||||
[target."cfg(target_os = \"macos\")".dependencies.objc]
|
||||
version = "0.2.7"
|
||||
|
||||
[target."cfg(not(target_arch = \"wasm32\"))".dependencies]
|
||||
ps-importer = { path = "../ps-importer" }
|
||||
# Do I need this?
|
||||
ps-database = { path = "../ps-database" }
|
||||
@ -0,0 +1,42 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
|
||||
# ./setup_web.sh # <- call this first!
|
||||
|
||||
FOLDER_NAME=${PWD##*/}
|
||||
CRATE_NAME=$FOLDER_NAME # assume crate name is the same as the folder name
|
||||
CRATE_NAME_SNAKE_CASE="${CRATE_NAME//-/_}" # for those who name crates with-kebab-case
|
||||
|
||||
# This is required to enable the web_sys clipboard API which egui_web uses
|
||||
# https://rustwasm.github.io/wasm-bindgen/api/web_sys/struct.Clipboard.html
|
||||
# https://rustwasm.github.io/docs/wasm-bindgen/web-sys/unstable-apis.html
|
||||
export RUSTFLAGS=--cfg=web_sys_unstable_apis
|
||||
|
||||
# Clear output from old stuff:
|
||||
rm -f web_demo/${CRATE_NAME_SNAKE_CASE}_bg.wasm
|
||||
|
||||
echo "Building rust…"
|
||||
BUILD=release
|
||||
cargo build --release -p ${CRATE_NAME} --lib --target wasm32-unknown-unknown
|
||||
|
||||
echo "Generating JS bindings for wasm…"
|
||||
TARGET_NAME="${CRATE_NAME_SNAKE_CASE}.wasm"
|
||||
wasm-bindgen "target/wasm32-unknown-unknown/${BUILD}/${TARGET_NAME}" \
|
||||
--out-dir web_demo --no-modules --no-typescript
|
||||
|
||||
# to get wasm-opt: apt/brew/dnf install binaryen
|
||||
# echo "Optimizing wasm…"
|
||||
# wasm-opt web_demo/${CRATE_NAME_SNAKE_CASE}_bg.wasm -O2 --fast-math -o web_demo/${CRATE_NAME_SNAKE_CASE}_bg.wasm # add -g to get debug symbols
|
||||
|
||||
echo "Finished: web_demo/${CRATE_NAME_SNAKE_CASE}.wasm"
|
||||
|
||||
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
|
||||
# Linux, ex: Fedora
|
||||
xdg-open http://localhost:8080/index.html
|
||||
elif [[ "$OSTYPE" == "msys" ]]; then
|
||||
# Windows
|
||||
start http://localhost:8080/index.html
|
||||
else
|
||||
# Darwin/MacOS, or something else
|
||||
open http://localhost:8080/index.html
|
||||
fi
|
||||
@ -0,0 +1,10 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
|
||||
# Pre-requisites:
|
||||
rustup target add wasm32-unknown-unknown
|
||||
cargo install -f wasm-bindgen-cli
|
||||
cargo update -p wasm-bindgen
|
||||
|
||||
# For local tests with `./start_server`:
|
||||
cargo install basic-http-server
|
||||
@ -1,9 +1,7 @@
|
||||
use crate::database::query::Field;
|
||||
use crate::database::query_result::QueryRow;
|
||||
use crate::model::{items, Engine};
|
||||
use chrono::prelude::*;
|
||||
use eframe::egui::{self, Widget};
|
||||
use eyre::Report;
|
||||
use ps_core::{model::items, model::Engine, Field, QueryRow};
|
||||
|
||||
use super::widgets::Table;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
use crate::model::Engine;
|
||||
use eframe::egui::{self, Color32, Label, Widget};
|
||||
use eyre::Report;
|
||||
use num_format::{Locale, ToFormattedString};
|
||||
use ps_core::model::Engine;
|
||||
|
||||
use super::app_state::UIState;
|
||||
use super::platform::navigation_button;
|
||||
|
Before Width: | Height: | Size: 515 KiB After Width: | Height: | Size: 515 KiB |
@ -1,6 +1,6 @@
|
||||
use crate::model::{segmentations, Engine};
|
||||
use eframe::egui::{self, Widget};
|
||||
use eyre::Report;
|
||||
use ps_core::model::{segmentations, Engine};
|
||||
|
||||
pub struct SegmentationBar<'a> {
|
||||
engine: &'a mut Engine,
|
||||
@ -0,0 +1,3 @@
|
||||
/target
|
||||
target
|
||||
.DS_Store
|
||||
@ -1,61 +0,0 @@
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
use super::formats::shared;
|
||||
use ps_core::{
|
||||
crossbeam_channel::{self, unbounded},
|
||||
Config, DatabaseLike, Importerlike, Message, MessageReceiver,
|
||||
};
|
||||
|
||||
use super::formats::ImporterFormat;
|
||||
|
||||
use eyre::Result;
|
||||
|
||||
pub struct Importer<Format: ImporterFormat> {
|
||||
config: Config,
|
||||
format: Format,
|
||||
}
|
||||
|
||||
impl<Format: ImporterFormat + 'static> Importer<Format> {
|
||||
pub fn new(config: Config, format: Format) -> Self {
|
||||
Self { config, format }
|
||||
}
|
||||
}
|
||||
|
||||
impl<Format: ImporterFormat + 'static> Importerlike for Importer<Format> {
|
||||
fn import<Database: DatabaseLike + 'static>(
|
||||
self,
|
||||
database: Database,
|
||||
) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
||||
let Importer { format, .. } = self;
|
||||
let (sender, receiver) = unbounded();
|
||||
|
||||
let config = self.config;
|
||||
let handle: JoinHandle<Result<()>> = std::thread::spawn(move || {
|
||||
let outer_sender = sender.clone();
|
||||
let processed = move || {
|
||||
let emails = format.emails(&config, sender.clone())?;
|
||||
let processed =
|
||||
shared::database::into_database(&config, emails, sender.clone(), database)?;
|
||||
|
||||
Ok(processed)
|
||||
};
|
||||
let result = processed();
|
||||
|
||||
// Send the error away and map it to a crossbeam channel error
|
||||
match result {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => match outer_sender.send(Message::Error(e)) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(eyre::Report::new(e)),
|
||||
},
|
||||
}
|
||||
});
|
||||
Ok((receiver, handle))
|
||||
}
|
||||
}
|
||||
|
||||
// impl<T: Importerlike + Sized> Importerlike for Box<T> {
|
||||
// fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
||||
// (*self).import()
|
||||
// }
|
||||
// }
|
||||
Binary file not shown.
@ -1,87 +0,0 @@
|
||||
use eyre::Result;
|
||||
|
||||
use std::{
|
||||
io::{stdout, Write},
|
||||
thread::sleep,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use postsack::{
|
||||
self,
|
||||
importer::{Adapter, State},
|
||||
types::FormatType,
|
||||
};
|
||||
|
||||
fn main() -> Result<()> {
|
||||
postsack::setup_tracing();
|
||||
|
||||
let config = postsack::make_config();
|
||||
|
||||
let adapter = postsack::importer::Adapter::new();
|
||||
|
||||
// Could not figure out how to build this properly
|
||||
// with dynamic dispatch. (to abstract away the match)
|
||||
// Will try again when I'm online.
|
||||
let handle = match config.format {
|
||||
FormatType::AppleMail => {
|
||||
let importer = postsack::importer::applemail_importer(config);
|
||||
adapter.process(importer)?
|
||||
}
|
||||
FormatType::GmailVault => {
|
||||
let importer = postsack::importer::gmail_importer(config);
|
||||
adapter.process(importer)?
|
||||
}
|
||||
FormatType::Mbox => {
|
||||
let importer = postsack::importer::mbox_importer(config);
|
||||
adapter.process(importer)?
|
||||
}
|
||||
};
|
||||
|
||||
let mut stdout = stdout();
|
||||
|
||||
loop {
|
||||
match handle_adapter(&adapter) {
|
||||
Ok(true) => break,
|
||||
Ok(false) => (),
|
||||
Err(e) => {
|
||||
println!("Execution Error:\n{:?}", &e);
|
||||
panic!();
|
||||
}
|
||||
}
|
||||
stdout.flush().unwrap();
|
||||
}
|
||||
|
||||
match handle.join() {
|
||||
Err(e) => println!("Error: {:?}", e),
|
||||
Ok(Err(e)) => println!("Error: {:?}", e),
|
||||
_ => (),
|
||||
}
|
||||
println!("\rDone");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn handle_adapter(adapter: &Adapter) -> Result<bool> {
|
||||
let State {
|
||||
done, finishing, ..
|
||||
} = adapter.finished()?;
|
||||
if done {
|
||||
return Ok(true);
|
||||
}
|
||||
if finishing {
|
||||
print!("\rFinishing up...");
|
||||
} else {
|
||||
let write = adapter.write_count()?;
|
||||
if write.count > 0 {
|
||||
print!("\rWriting emails to DB {}/{}...", write.count, write.total);
|
||||
} else {
|
||||
let read = adapter.read_count()?;
|
||||
print!(
|
||||
"\rReading Emails {}%...",
|
||||
(read.count as f32 / read.total as f32) * 100.0
|
||||
);
|
||||
}
|
||||
}
|
||||
sleep(Duration::from_millis(50));
|
||||
Ok(false)
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
#[cfg(feature = "gui")]
|
||||
fn main() {
|
||||
#[cfg(debug_assertions)]
|
||||
postsack::setup_tracing();
|
||||
|
||||
postsack::gui::run_gui();
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "gui"))]
|
||||
fn main() {
|
||||
println!("Gui not selected")
|
||||
}
|
||||
@ -1,145 +0,0 @@
|
||||
use std::collections::HashMap;
|
||||
use std::convert::TryInto;
|
||||
use std::str::FromStr;
|
||||
|
||||
use chrono::prelude::*;
|
||||
use eyre::{bail, eyre, Result};
|
||||
use rusqlite::{self, types, Row};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::query::{Field, ValueField, AMOUNT_FIELD_NAME};
|
||||
use super::query_result::QueryResult;
|
||||
use crate::importer::{EmailEntry, EmailMeta};
|
||||
|
||||
/// rusqlite does offer Serde to Value conversion, but it
|
||||
/// converts everything to strings!
|
||||
pub fn json_to_value(input: &Value) -> Result<types::Value> {
|
||||
let ok = match input {
|
||||
Value::Number(n) if n.is_i64() => {
|
||||
types::Value::Integer(n.as_i64().ok_or_else(|| eyre!("Invalid Number {:?}", n))?)
|
||||
}
|
||||
Value::Number(n) if n.is_u64() => {
|
||||
let value = n.as_u64().ok_or_else(|| eyre!("Invalid Number {:?}", n))?;
|
||||
let converted: i64 = value.try_into()?;
|
||||
types::Value::Integer(converted)
|
||||
}
|
||||
Value::Number(n) if n.is_f64() => {
|
||||
types::Value::Real(n.as_f64().ok_or_else(|| eyre!("Invalid Number {:?}", n))?)
|
||||
}
|
||||
Value::Bool(n) => types::Value::Integer(*n as i64),
|
||||
Value::String(n) => types::Value::Text(n.clone()),
|
||||
_ => bail!("Invalid type: {}", &input),
|
||||
};
|
||||
Ok(ok)
|
||||
}
|
||||
|
||||
pub trait RowConversion<'a>: Sized {
|
||||
fn grouped_from_row<'stmt>(field: &'a Field, row: &Row<'stmt>) -> Result<Self>;
|
||||
fn from_row<'stmt>(fields: &'a [Field], row: &Row<'stmt>) -> Result<Self>;
|
||||
}
|
||||
|
||||
impl<'a> RowConversion<'a> for QueryResult {
|
||||
fn grouped_from_row<'stmt>(field: &'a Field, row: &Row<'stmt>) -> Result<Self> {
|
||||
let amount: usize = row.get(AMOUNT_FIELD_NAME)?;
|
||||
let values = values_from_fields(&[*field], row)?;
|
||||
|
||||
Ok(QueryResult::Grouped {
|
||||
count: amount,
|
||||
value: values[field].clone(),
|
||||
})
|
||||
}
|
||||
fn from_row<'stmt>(fields: &'a [Field], row: &Row<'stmt>) -> Result<Self> {
|
||||
let values = values_from_fields(fields, row)?;
|
||||
Ok(QueryResult::Normal(values))
|
||||
}
|
||||
}
|
||||
|
||||
fn values_from_fields<'stmt>(
|
||||
fields: &[Field],
|
||||
row: &Row<'stmt>,
|
||||
) -> Result<HashMap<Field, ValueField>> {
|
||||
let mut values: HashMap<Field, ValueField> = HashMap::default();
|
||||
for field in fields {
|
||||
values.insert(*field, value_from_field(field, row)?);
|
||||
}
|
||||
Ok(values)
|
||||
}
|
||||
|
||||
pub fn value_from_field<'stmt>(field: &Field, row: &Row<'stmt>) -> Result<ValueField> {
|
||||
use Field::*;
|
||||
// Use type safety when unpacking
|
||||
match field {
|
||||
Path | SenderDomain | SenderLocalPart | SenderName | ToGroup | ToName | ToAddress
|
||||
| Subject => {
|
||||
let string: String = row.get::<&str, String>(field.as_str())?;
|
||||
Ok(ValueField::string(field, &string))
|
||||
}
|
||||
Year | Month | Day | Timestamp => {
|
||||
return Ok(ValueField::usize(
|
||||
field,
|
||||
row.get::<&str, usize>(field.as_str())?,
|
||||
));
|
||||
}
|
||||
MetaTags => {
|
||||
let tag_string = row.get::<&str, String>(field.as_str())?;
|
||||
let tags =
|
||||
crate::importer::formats::shared::email::EmailMeta::tags_from_string(&tag_string);
|
||||
Ok(ValueField::array(
|
||||
field,
|
||||
tags.into_iter().map(Value::String).collect(),
|
||||
))
|
||||
}
|
||||
IsReply | IsSend | MetaIsSeen => {
|
||||
return Ok(ValueField::bool(
|
||||
field,
|
||||
row.get::<&str, bool>(field.as_str())?,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl EmailEntry {
|
||||
#[allow(unused)]
|
||||
fn from_row(row: &Row<'_>) -> Result<Self> {
|
||||
let path: String = row.get("path")?;
|
||||
let path = std::path::PathBuf::from_str(&path)?;
|
||||
let sender_domain: String = row.get("sender_domain")?;
|
||||
let sender_local_part: String = row.get("sender_local_part")?;
|
||||
let sender_name: String = row.get("sender_name")?;
|
||||
let timestamp: i64 = row.get("timestamp")?;
|
||||
let datetime = Utc.timestamp(timestamp, 0);
|
||||
let subject: String = row.get("subject")?;
|
||||
let to_count: usize = row.get("to_count")?;
|
||||
let to_group: Option<String> = row.get("to_group")?;
|
||||
let to_name: Option<String> = row.get("to_name")?;
|
||||
let to_address: Option<String> = row.get("to_address")?;
|
||||
|
||||
let to_first = to_address.map(|a| (to_name.unwrap_or_default(), a));
|
||||
|
||||
let is_reply: bool = row.get("is_reply")?;
|
||||
let is_send: bool = row.get("is_send")?;
|
||||
|
||||
// Parse EmailMeta
|
||||
let meta_tags: Option<String> = row.get("meta_tags")?;
|
||||
let meta_is_seen: Option<bool> = row.get("meta_is_seen")?;
|
||||
let meta = match (meta_tags, meta_is_seen) {
|
||||
(Some(a), Some(b)) => Some(EmailMeta::from(b, &a)),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
Ok(EmailEntry {
|
||||
path,
|
||||
sender_domain,
|
||||
sender_local_part,
|
||||
sender_name,
|
||||
datetime,
|
||||
subject,
|
||||
to_count,
|
||||
to_group,
|
||||
to_first,
|
||||
is_reply,
|
||||
is_send,
|
||||
meta,
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,255 +0,0 @@
|
||||
use chrono::Datelike;
|
||||
use crossbeam_channel::{unbounded, Sender};
|
||||
use eyre::{bail, Report, Result};
|
||||
use rusqlite::{self, params, Connection, Statement};
|
||||
|
||||
use core::panic;
|
||||
use std::{collections::HashMap, path::Path, thread::JoinHandle};
|
||||
|
||||
use super::{query::Query, query_result::QueryResult, sql::*, DBMessage};
|
||||
use crate::database::query::OtherQuery;
|
||||
use crate::types::Config;
|
||||
use crate::{
|
||||
database::{value_from_field, RowConversion},
|
||||
importer::EmailEntry,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Database {
|
||||
connection: Option<Connection>,
|
||||
}
|
||||
|
||||
impl Database {
|
||||
/// Open a database and try to retrieve a config from the information stored in there
|
||||
pub fn config<P: AsRef<Path>>(path: P) -> Result<Config> {
|
||||
let database = Self::new(path.as_ref())?;
|
||||
let fields = database.select_config_fields()?;
|
||||
Config::from_fields(path.as_ref(), fields)
|
||||
}
|
||||
|
||||
/// Open database at path `Path`.
|
||||
pub fn new<P: AsRef<Path>>(path: P) -> Result<Self> {
|
||||
#[allow(unused_mut)]
|
||||
let mut connection = Connection::open(path.as_ref())?;
|
||||
|
||||
// Improve the insertion performance.
|
||||
connection.pragma_update(None, "journal_mode", &"memory")?;
|
||||
connection.pragma_update(None, "synchronous", &"OFF")?;
|
||||
|
||||
Self::create_tables(&connection)?;
|
||||
|
||||
#[cfg(feature = "trace-sql")]
|
||||
connection.trace(Some(|query| {
|
||||
tracing::trace!("SQL: {}", &query);
|
||||
}));
|
||||
|
||||
Ok(Database {
|
||||
connection: Some(connection),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn total_mails(&self) -> Result<usize> {
|
||||
let connection = match &self.connection {
|
||||
Some(n) => n,
|
||||
None => bail!("No connection to database available in query"),
|
||||
};
|
||||
let mut stmt = connection.prepare(QUERY_COUNT_MAILS)?;
|
||||
let count: usize = stmt.query_row([], |q| q.get(0))?;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
pub fn save_config(&self, config: Config) -> Result<()> {
|
||||
let fields = config
|
||||
.into_fields()
|
||||
.ok_or_else(|| eyre::eyre!("Could not create fields from config"))?;
|
||||
self.insert_config_fields(fields)
|
||||
}
|
||||
|
||||
pub fn query(&self, query: &super::query::Query) -> Result<Vec<QueryResult>> {
|
||||
use rusqlite::params_from_iter;
|
||||
let c = match &self.connection {
|
||||
Some(n) => n,
|
||||
None => bail!("No connection to database available in query"),
|
||||
};
|
||||
let (sql, values) = query.to_sql();
|
||||
let mut stmt = c.prepare(&sql)?;
|
||||
let mut query_results = Vec::new();
|
||||
let mut converted = Vec::new();
|
||||
for value in values {
|
||||
converted.push(super::conversion::json_to_value(&value)?);
|
||||
}
|
||||
|
||||
let p = params_from_iter(converted.iter());
|
||||
|
||||
let mut rows = stmt.query(p)?;
|
||||
while let Some(row) = rows.next()? {
|
||||
match query {
|
||||
Query::Grouped { group_by, .. } => {
|
||||
let result = QueryResult::grouped_from_row(group_by, row)?;
|
||||
query_results.push(result);
|
||||
}
|
||||
Query::Normal { fields, .. } => {
|
||||
let result = QueryResult::from_row(fields, row)?;
|
||||
query_results.push(result);
|
||||
}
|
||||
Query::Other {
|
||||
query: OtherQuery::All(field),
|
||||
} => query_results.push(QueryResult::Other(value_from_field(field, row)?)),
|
||||
}
|
||||
}
|
||||
Ok(query_results)
|
||||
}
|
||||
|
||||
/// Begin the data import.
|
||||
/// This will consume the `Database`. A new one has to be opened
|
||||
/// afterwards in order to support multi-threading.
|
||||
/// Returns an input `Sender` and a `JoinHandle`.
|
||||
/// The `Sender` is used to submit work to the database via `DBMessage`
|
||||
/// cases. The `JoinHandle` is used to wait for database completion.
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// ``` ignore
|
||||
/// let db = Database::new("db.sqlite").unwrap();
|
||||
/// let (sender, handle) = db.import();
|
||||
/// sender.send(DBMessage::Mail(m1)).unwrap();
|
||||
/// sender.send(DBMessage::Mail(m2)).unwrap();
|
||||
/// handle.join().unwrap();
|
||||
/// ```
|
||||
pub fn import(mut self) -> (Sender<DBMessage>, JoinHandle<Result<usize>>) {
|
||||
let (sender, receiver) = unbounded();
|
||||
|
||||
// Import can only be called *once* on a database created with `new`.
|
||||
// Therefore there should always be a value to unwrap;
|
||||
let mut connection = self.connection.take().unwrap();
|
||||
let handle = std::thread::spawn(move || {
|
||||
let mut counter = 0;
|
||||
{
|
||||
let transaction = connection.transaction()?;
|
||||
{
|
||||
let mut mail_prepared = transaction.prepare(QUERY_EMAILS)?;
|
||||
let mut error_prepared = transaction.prepare(QUERY_ERRORS)?;
|
||||
loop {
|
||||
let next = match receiver.recv() {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
println!("Receiver error: {:?}", &e);
|
||||
panic!("should not happen");
|
||||
}
|
||||
};
|
||||
match next {
|
||||
DBMessage::Mail(mail) => {
|
||||
counter += 1;
|
||||
insert_mail(&mut mail_prepared, &mail)
|
||||
}
|
||||
DBMessage::Error(report) => insert_error(&mut error_prepared, &report),
|
||||
DBMessage::Done => {
|
||||
tracing::trace!("Received DBMessage::Done");
|
||||
break;
|
||||
}
|
||||
}?;
|
||||
}
|
||||
}
|
||||
if let Err(e) = transaction.commit() {
|
||||
return Err(eyre::eyre!("Transaction Error: {:?}", &e));
|
||||
}
|
||||
}
|
||||
// In case closing the database fails, we try again until we succeed
|
||||
let mut c = connection;
|
||||
loop {
|
||||
tracing::trace!("Attempting close");
|
||||
match c.close() {
|
||||
Ok(_n) => break,
|
||||
Err((a, _b)) => c = a,
|
||||
}
|
||||
}
|
||||
tracing::trace!("Finished SQLITE: {}", &counter);
|
||||
Ok(counter)
|
||||
});
|
||||
(sender, handle)
|
||||
}
|
||||
|
||||
fn create_tables(connection: &Connection) -> Result<()> {
|
||||
connection.execute(TBL_EMAILS, params![])?;
|
||||
connection.execute(TBL_ERRORS, params![])?;
|
||||
connection.execute(TBL_META, params![])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn select_config_fields(&self) -> Result<HashMap<String, serde_json::Value>> {
|
||||
let connection = match &self.connection {
|
||||
Some(n) => n,
|
||||
None => bail!("No connection to database available in query"),
|
||||
};
|
||||
let mut stmt = connection.prepare(QUERY_SELECT_META)?;
|
||||
let mut query_results = HashMap::new();
|
||||
let mut rows = stmt.query([])?;
|
||||
while let Some(row) = rows.next()? {
|
||||
let (k, v) = match (
|
||||
row.get::<_, String>("key"),
|
||||
row.get::<_, serde_json::Value>("value"),
|
||||
) {
|
||||
(Ok(k), Ok(v)) => (k, v),
|
||||
(a, b) => {
|
||||
tracing::error!("Invalid row data. Missing fields key and or value:\nkey: {:?}\nvalue: {:?}\n", a, b);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
query_results.insert(k, v);
|
||||
}
|
||||
Ok(query_results)
|
||||
}
|
||||
|
||||
fn insert_config_fields(&self, fields: HashMap<String, serde_json::Value>) -> Result<()> {
|
||||
let connection = match &self.connection {
|
||||
Some(n) => n,
|
||||
None => bail!("No connection to database available in query"),
|
||||
};
|
||||
let mut stmt = connection.prepare(QUERY_INSERT_META)?;
|
||||
for (key, value) in fields {
|
||||
stmt.execute(params![key, value])?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_mail(statement: &mut Statement, entry: &EmailEntry) -> Result<()> {
|
||||
let path = entry.path.display().to_string();
|
||||
let year = entry.datetime.date().year();
|
||||
let month = entry.datetime.date().month();
|
||||
let day = entry.datetime.date().day();
|
||||
let timestamp = entry.datetime.timestamp();
|
||||
let e = entry;
|
||||
let to_name = e.to_first.as_ref().map(|e| &e.0);
|
||||
let to_address = e.to_first.as_ref().map(|e| &e.1);
|
||||
let meta_tags = e.meta.as_ref().map(|e| e.tags_string());
|
||||
let meta_is_seen = e.meta.as_ref().map(|e| e.is_seen);
|
||||
let p = params![
|
||||
path,
|
||||
e.sender_domain,
|
||||
e.sender_local_part,
|
||||
e.sender_name,
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
timestamp,
|
||||
e.subject,
|
||||
e.to_count,
|
||||
e.to_group,
|
||||
to_name,
|
||||
to_address,
|
||||
e.is_reply,
|
||||
e.is_send,
|
||||
meta_tags,
|
||||
meta_is_seen
|
||||
];
|
||||
statement.execute(p)?;
|
||||
tracing::trace!("Insert Mail {}", &path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn insert_error(statement: &mut Statement, message: &Report) -> Result<()> {
|
||||
statement.execute(params![message.to_string()])?;
|
||||
tracing::trace!("Insert Error {}", message);
|
||||
Ok(())
|
||||
}
|
||||
@ -1,14 +0,0 @@
|
||||
use eyre::Report;
|
||||
|
||||
use crate::importer::EmailEntry;
|
||||
|
||||
/// Parameter for sending work to the database during `import`.
|
||||
pub enum DBMessage {
|
||||
/// Send for a successfuly parsed mail
|
||||
Mail(Box<EmailEntry>),
|
||||
/// Send for any kind of error during reading / parsing
|
||||
Error(Report),
|
||||
/// Send once all parsing is done.
|
||||
/// This is used to break out of the receiving loop
|
||||
Done,
|
||||
}
|
||||
@ -1,10 +0,0 @@
|
||||
mod conversion;
|
||||
mod db;
|
||||
mod db_message;
|
||||
pub mod query;
|
||||
pub mod query_result;
|
||||
mod sql;
|
||||
|
||||
pub use conversion::{value_from_field, RowConversion};
|
||||
pub use db::Database;
|
||||
pub use db_message::DBMessage;
|
||||
@ -1,242 +0,0 @@
|
||||
use rsql_builder;
|
||||
use serde_json;
|
||||
pub use serde_json::Value;
|
||||
use strum::{self, IntoEnumIterator};
|
||||
use strum_macros::{EnumIter, IntoStaticStr};
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
pub const AMOUNT_FIELD_NAME: &str = "amount";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Filter {
|
||||
/// A database Like Operation
|
||||
Like(ValueField),
|
||||
NotLike(ValueField),
|
||||
/// A extended like that implies:
|
||||
/// - wildcards on both sides (like '%test%')
|
||||
/// - case in-sensitive comparison
|
||||
/// - Trying to handle values as strings
|
||||
Contains(ValueField),
|
||||
Is(ValueField),
|
||||
}
|
||||
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, IntoStaticStr, EnumIter)]
|
||||
#[strum(serialize_all = "snake_case")]
|
||||
pub enum Field {
|
||||
Path,
|
||||
SenderDomain,
|
||||
SenderLocalPart,
|
||||
SenderName,
|
||||
Year,
|
||||
Month,
|
||||
Day,
|
||||
Timestamp,
|
||||
ToGroup,
|
||||
ToName,
|
||||
ToAddress,
|
||||
IsReply,
|
||||
IsSend,
|
||||
Subject,
|
||||
MetaIsSeen,
|
||||
MetaTags,
|
||||
}
|
||||
|
||||
const INVALID_FIELDS: &[Field] = &[
|
||||
Field::Path,
|
||||
Field::Subject,
|
||||
Field::Timestamp,
|
||||
Field::IsReply,
|
||||
Field::IsSend,
|
||||
Field::MetaIsSeen,
|
||||
Field::MetaTags,
|
||||
];
|
||||
|
||||
impl Field {
|
||||
pub fn all_cases() -> impl Iterator<Item = Field> {
|
||||
Field::iter().filter(|f| !INVALID_FIELDS.contains(f))
|
||||
}
|
||||
|
||||
/// Just a wrapper to offer `into` without the type ambiguity
|
||||
/// that sometimes arises
|
||||
pub fn as_str(&self) -> &'static str {
|
||||
self.into()
|
||||
}
|
||||
|
||||
/// A human readable name
|
||||
pub fn name(&self) -> &str {
|
||||
use Field::*;
|
||||
match self {
|
||||
SenderDomain => "Domain",
|
||||
SenderLocalPart => "Address",
|
||||
SenderName => "Name",
|
||||
ToGroup => "Group",
|
||||
ToName => "To name",
|
||||
ToAddress => "To address",
|
||||
Year => "Year",
|
||||
Month => "Month",
|
||||
Day => "Day",
|
||||
Subject => "Subject",
|
||||
_ => self.as_str(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Field {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
write!(f, "{}", self.name())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
pub struct ValueField {
|
||||
field: Field,
|
||||
value: Value,
|
||||
}
|
||||
|
||||
impl ValueField {
|
||||
pub fn string<S: AsRef<str>>(field: &Field, value: S) -> ValueField {
|
||||
ValueField {
|
||||
field: *field,
|
||||
value: Value::String(value.as_ref().to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bool(field: &Field, value: bool) -> ValueField {
|
||||
ValueField {
|
||||
field: *field,
|
||||
value: Value::Bool(value),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn usize(field: &Field, value: usize) -> ValueField {
|
||||
ValueField {
|
||||
field: *field,
|
||||
value: Value::Number(value.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn array(field: &Field, value: Vec<Value>) -> ValueField {
|
||||
ValueField {
|
||||
field: *field,
|
||||
value: Value::Array(value),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn field(&self) -> &Field {
|
||||
&self.field
|
||||
}
|
||||
|
||||
pub fn value(&self) -> &Value {
|
||||
&self.value
|
||||
}
|
||||
|
||||
#[allow(clippy::inherent_to_string)]
|
||||
pub fn to_string(&self) -> String {
|
||||
match &self.value {
|
||||
Value::String(s) => s.clone(),
|
||||
_ => format!("{}", &self.value),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum OtherQuery {
|
||||
/// Get all contents of a specific field
|
||||
All(Field),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub enum Query {
|
||||
Grouped {
|
||||
filters: Vec<Filter>,
|
||||
group_by: Field,
|
||||
},
|
||||
Normal {
|
||||
fields: Vec<Field>,
|
||||
filters: Vec<Filter>,
|
||||
range: Range<usize>,
|
||||
},
|
||||
Other {
|
||||
query: OtherQuery,
|
||||
},
|
||||
}
|
||||
|
||||
impl Query {
|
||||
fn filters(&self) -> &[Filter] {
|
||||
match self {
|
||||
Query::Grouped { ref filters, .. } => filters,
|
||||
Query::Normal { ref filters, .. } => filters,
|
||||
Query::Other { .. } => &[],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Query {
|
||||
pub fn to_sql(&self) -> (String, Vec<serde_json::Value>) {
|
||||
let mut conditions = {
|
||||
let mut whr = rsql_builder::B::new_where();
|
||||
for filter in self.filters() {
|
||||
match filter {
|
||||
Filter::Like(f) => whr.like(f.field.into(), f.value()),
|
||||
Filter::NotLike(f) => whr.not_like(f.field.into(), f.value()),
|
||||
Filter::Contains(f) => whr.like(
|
||||
f.field.into(),
|
||||
&format!("%{}%", f.to_string().to_lowercase()),
|
||||
),
|
||||
Filter::Is(f) => whr.eq(f.field.into(), f.value()),
|
||||
};
|
||||
}
|
||||
whr
|
||||
};
|
||||
|
||||
let (header, group_by) = match self {
|
||||
Query::Grouped { group_by, .. } => (
|
||||
format!(
|
||||
"SELECT count(path) as {}, {} FROM emails",
|
||||
AMOUNT_FIELD_NAME,
|
||||
group_by.as_str()
|
||||
),
|
||||
format!("GROUP BY {}", group_by.as_str()),
|
||||
),
|
||||
Query::Normal { fields, range, .. } => {
|
||||
let fields: Vec<&str> = fields.iter().map(|e| e.into()).collect();
|
||||
(
|
||||
format!("SELECT {} FROM emails", fields.join(", ")),
|
||||
format!("LIMIT {}, {}", range.start, range.end - range.start),
|
||||
)
|
||||
}
|
||||
Query::Other {
|
||||
query: OtherQuery::All(field),
|
||||
} => (
|
||||
format!("SELECT {} FROM emails", field.as_str()),
|
||||
format!(""),
|
||||
),
|
||||
};
|
||||
|
||||
let (sql, values) = rsql_builder::B::prepare(
|
||||
rsql_builder::B::new_sql(&header)
|
||||
.push_build(&mut conditions)
|
||||
.push_sql(&group_by),
|
||||
);
|
||||
|
||||
(sql, values)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_test() {
|
||||
let query = Query::Grouped {
|
||||
filters: vec![
|
||||
Filter::Like(ValueField::string(&Field::SenderDomain, "gmail.com")),
|
||||
Filter::Is(ValueField::usize(&Field::Year, 2021)),
|
||||
],
|
||||
group_by: Field::Month,
|
||||
};
|
||||
dbg!(&query.to_sql());
|
||||
}
|
||||
}
|
||||
@ -1,17 +0,0 @@
|
||||
use super::query::{Field, ValueField};
|
||||
use std::collections::HashMap;
|
||||
|
||||
pub type QueryRow = HashMap<Field, ValueField>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum QueryResult {
|
||||
Grouped {
|
||||
/// How many items did we find?
|
||||
count: usize,
|
||||
/// All the itmes that we grouped by including their values.
|
||||
/// So that we can use each of them to limit the next query.
|
||||
value: ValueField,
|
||||
},
|
||||
Normal(QueryRow),
|
||||
Other(ValueField),
|
||||
}
|
||||
@ -1,71 +0,0 @@
|
||||
pub const TBL_EMAILS: &str = r#"
|
||||
CREATE TABLE IF NOT EXISTS emails (
|
||||
path TEXT NOT NULL,
|
||||
sender_domain TEXT NOT NULL,
|
||||
sender_local_part TEXT NOT NULL,
|
||||
sender_name TEXT NOT NULL,
|
||||
year INTEGER NOT NULL,
|
||||
month INTEGER NOT NULL,
|
||||
day INTEGER NOT NULL,
|
||||
timestamp INTEGER NOT NULL,
|
||||
subject TEXT NOT NULL,
|
||||
to_count INTEGER NOT NULL,
|
||||
to_group TEXT NULL,
|
||||
to_name TEXT NULL,
|
||||
to_address TEXT NULL,
|
||||
is_reply BOOL,
|
||||
is_send BOOL,
|
||||
meta_tags TEXT NULL,
|
||||
meta_is_seen BOOL NULL
|
||||
);"#;
|
||||
|
||||
pub const QUERY_EMAILS: &str = r#"
|
||||
INSERT INTO emails
|
||||
(
|
||||
path, sender_domain, sender_local_part, sender_name,
|
||||
year, month, day, timestamp, subject,
|
||||
to_count, to_group, to_name, to_address,
|
||||
is_reply, is_send,
|
||||
meta_tags, meta_is_seen
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
?, ?, ?, ?,
|
||||
?, ?, ?, ?, ?,
|
||||
?, ?, ?, ?,
|
||||
?, ?,
|
||||
?, ?
|
||||
)
|
||||
"#;
|
||||
|
||||
pub const TBL_ERRORS: &str = r#"
|
||||
CREATE TABLE IF NOT EXISTS errors (
|
||||
message TEXT NOT NULL
|
||||
);"#;
|
||||
|
||||
pub const QUERY_ERRORS: &str = r#"
|
||||
INSERT INTO errors
|
||||
(message)
|
||||
VALUES
|
||||
(?)
|
||||
"#;
|
||||
|
||||
pub const TBL_META: &str = r#"
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT NOT NULL,
|
||||
value TEXT NOT NULL
|
||||
);"#;
|
||||
|
||||
pub const QUERY_INSERT_META: &str = r#"
|
||||
INSERT INTO meta
|
||||
(key, value)
|
||||
VALUES
|
||||
(?, ?)
|
||||
"#;
|
||||
|
||||
pub const QUERY_SELECT_META: &str = r#"
|
||||
SELECT key, value FROM meta"#;
|
||||
|
||||
pub const QUERY_COUNT_MAILS: &str = r#"
|
||||
SELECT count(path) FROM emails
|
||||
"#;
|
||||
Binary file not shown.
@ -1,88 +0,0 @@
|
||||
//! We use a stubbornly stupid algorithm where we just
|
||||
//! recursively drill down into the appropriate folder
|
||||
//! until we find `emlx` files and return those.
|
||||
|
||||
use eyre::{eyre, Result};
|
||||
use rayon::prelude::*;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use super::super::shared::filesystem::emails_in;
|
||||
use super::super::{Message, MessageSender};
|
||||
use crate::types::Config;
|
||||
|
||||
use super::mail::Mail;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub fn read_emails(config: &Config, sender: MessageSender) -> Result<Vec<Mail>> {
|
||||
// on macOS, we might need permission for the `Library` folder...
|
||||
match std::fs::read_dir(&config.emails_folder_path) {
|
||||
Ok(_) => (),
|
||||
Err(e) => match e.kind() {
|
||||
#[cfg(target_os = "macos")]
|
||||
std::io::ErrorKind::PermissionDenied => {
|
||||
tracing::info!("Could not read folder: {}", e);
|
||||
if let Err(e) = sender.send(Message::MissingPermissions) {
|
||||
tracing::error!("Error sending: {}", e);
|
||||
}
|
||||
// We should return early now, otherwise the code below will send a different
|
||||
// error
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
_ => {
|
||||
if let Err(e) = sender.send(Message::Error(eyre!("Error: {:?}", &e))) {
|
||||
tracing::error!("Error sending: {}", e);
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
// As `walkdir` does not support `par_iter` (see https://www.reddit.com/r/rust/comments/6eif7r/walkdir_users_we_need_you/)
|
||||
// - -we first collect all folders,
|
||||
// then all sub-folders in those ending in mboxending in .mbox and then iterate over them in paralell
|
||||
let folders: Vec<PathBuf> = WalkDir::new(&config.emails_folder_path)
|
||||
.into_iter()
|
||||
.filter_map(|e| match e {
|
||||
Ok(n)
|
||||
if n.path().is_dir()
|
||||
&& n.path()
|
||||
.to_str()
|
||||
.map(|e| e.contains(".mbox"))
|
||||
.unwrap_or(false) =>
|
||||
{
|
||||
tracing::trace!("Found folder {}", n.path().display());
|
||||
Some(n.path().to_path_buf())
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::info!("Could not read folder: {}", e);
|
||||
if let Err(e) = sender.send(Message::Error(eyre!("Could not read folder: {:?}", e)))
|
||||
{
|
||||
tracing::error!("Error sending error {}", e);
|
||||
}
|
||||
None
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
sender.send(Message::ReadTotal(folders.len()))?;
|
||||
let mails: Vec<Mail> = folders
|
||||
.into_par_iter()
|
||||
.filter_map(
|
||||
|path| match emails_in(path.clone(), sender.clone(), Mail::new) {
|
||||
Ok(n) => Some(n),
|
||||
Err(e) => {
|
||||
tracing::error!("{} {:?}", path.display(), &e);
|
||||
if let Err(e) = sender.send(Message::Error(eyre!(
|
||||
"Could read mails in {}: {:?}",
|
||||
path.display(),
|
||||
e
|
||||
))) {
|
||||
tracing::error!("Error sending error {}", e);
|
||||
}
|
||||
None
|
||||
}
|
||||
},
|
||||
)
|
||||
.flatten()
|
||||
.collect();
|
||||
Ok(mails)
|
||||
}
|
||||
@ -1,69 +0,0 @@
|
||||
use emlx::parse_emlx;
|
||||
use eyre::Result;
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::super::shared::email::EmailMeta;
|
||||
use super::super::shared::parse::ParseableEmail;
|
||||
|
||||
pub struct Mail {
|
||||
path: PathBuf,
|
||||
// This is parsed out of the `emlx` as it is parsed
|
||||
is_seen: bool,
|
||||
// This is parsed out of the `path`
|
||||
label: Option<String>,
|
||||
// Maildata
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Mail {
|
||||
pub fn new<P: AsRef<Path>>(path: P) -> Option<Self> {
|
||||
let path = path.as_ref();
|
||||
let name = path.file_name()?.to_str()?;
|
||||
if !name.ends_with(".emlx") {
|
||||
return None;
|
||||
}
|
||||
// find the folder ending with `.mbox` in the path
|
||||
let ext = ".mbox";
|
||||
let label = path
|
||||
.iter()
|
||||
.map(|e| e.to_str())
|
||||
.flatten()
|
||||
.find(|s| s.ends_with(ext))
|
||||
.map(|s| s.replace(ext, ""));
|
||||
Some(Self {
|
||||
path: path.to_path_buf(),
|
||||
is_seen: false,
|
||||
label,
|
||||
data: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ParseableEmail for Mail {
|
||||
fn prepare(&mut self) -> Result<()> {
|
||||
let data = std::fs::read(self.path.as_path())?;
|
||||
let parsed = parse_emlx(&data)?;
|
||||
self.is_seen = !parsed.flags.is_read;
|
||||
self.data = parsed.message.to_vec();
|
||||
Ok(())
|
||||
}
|
||||
fn message(&self) -> Result<Cow<'_, [u8]>> {
|
||||
Ok(Cow::Borrowed(self.data.as_slice()))
|
||||
}
|
||||
fn path(&self) -> &Path {
|
||||
self.path.as_path()
|
||||
}
|
||||
fn meta(&self) -> Result<Option<EmailMeta>> {
|
||||
let tags = match self.label {
|
||||
Some(ref n) => vec![n.clone()],
|
||||
None => vec![],
|
||||
};
|
||||
let meta = EmailMeta {
|
||||
tags,
|
||||
is_seen: self.is_seen,
|
||||
};
|
||||
Ok(Some(meta))
|
||||
}
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
mod filesystem;
|
||||
mod mail;
|
||||
|
||||
use shellexpand;
|
||||
use std::{path::PathBuf, str::FromStr};
|
||||
|
||||
use super::{Config, ImporterFormat, MessageSender, Result};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct AppleMail {}
|
||||
|
||||
impl ImporterFormat for AppleMail {
|
||||
type Item = mail::Mail;
|
||||
|
||||
fn default_path() -> Option<PathBuf> {
|
||||
let path = shellexpand::tilde("~/Library/Mail");
|
||||
Some(PathBuf::from_str(&path.to_string()).unwrap())
|
||||
}
|
||||
|
||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
|
||||
filesystem::read_emails(config, sender)
|
||||
}
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
use chrono::prelude::*;
|
||||
|
||||
use eyre::{bail, Result};
|
||||
use serde::Deserialize;
|
||||
use serde_json;
|
||||
|
||||
use super::super::shared::email::EmailMeta;
|
||||
use super::raw_email::RawEmailEntry;
|
||||
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct Meta {
|
||||
pub msg_id: String,
|
||||
pub subject: String,
|
||||
pub labels: Vec<String>,
|
||||
pub flags: Vec<String>,
|
||||
internal_date: i64,
|
||||
|
||||
#[serde(skip, default = "Utc::now")]
|
||||
pub created: DateTime<Utc>,
|
||||
}
|
||||
|
||||
impl Meta {
|
||||
pub fn is_seen(&self) -> bool {
|
||||
self.labels.contains(&"\\seen".to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Meta> for EmailMeta {
|
||||
fn from(meta: Meta) -> Self {
|
||||
let is_seen = meta.is_seen();
|
||||
EmailMeta {
|
||||
tags: meta.labels,
|
||||
is_seen,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_meta(raw_entry: &RawEmailEntry) -> Result<Meta> {
|
||||
let content = match raw_entry.read_gmail_meta() {
|
||||
None => bail!("No Gmail Meta Information Available"),
|
||||
Some(content) => content?,
|
||||
};
|
||||
let mut meta: Meta = serde_json::from_slice(&content)?;
|
||||
meta.created = Utc.timestamp(meta.internal_date, 0);
|
||||
Ok(meta)
|
||||
}
|
||||
@ -1,23 +0,0 @@
|
||||
mod meta;
|
||||
mod raw_email;
|
||||
|
||||
use super::shared::filesystem::{emails_in, folders_in};
|
||||
use super::{Config, ImporterFormat, MessageSender, Result};
|
||||
use raw_email::RawEmailEntry;
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Gmail {}
|
||||
|
||||
impl ImporterFormat for Gmail {
|
||||
type Item = raw_email::RawEmailEntry;
|
||||
|
||||
fn default_path() -> Option<std::path::PathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
|
||||
folders_in(&config.emails_folder_path, sender, |path, sender| {
|
||||
emails_in(path, sender, RawEmailEntry::new)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,118 +0,0 @@
|
||||
use eyre::{eyre, Result};
|
||||
use flate2::read::GzDecoder;
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::super::shared::email::EmailMeta;
|
||||
use super::super::shared::parse::ParseableEmail;
|
||||
|
||||
/// Raw representation of an email.
|
||||
/// Contains the paths to the relevant files as well
|
||||
/// as the name of the folder the email was in.
|
||||
#[derive(Debug)]
|
||||
pub struct RawEmailEntry {
|
||||
#[allow(unused)]
|
||||
folder_name: String,
|
||||
eml_path: PathBuf,
|
||||
gmail_meta_path: Option<PathBuf>,
|
||||
is_compressed: bool,
|
||||
#[allow(unused)]
|
||||
size: u64,
|
||||
}
|
||||
|
||||
impl RawEmailEntry {
|
||||
pub fn path(&self) -> &Path {
|
||||
self.eml_path.as_path()
|
||||
}
|
||||
|
||||
pub fn read(&self) -> Result<Vec<u8>> {
|
||||
if self.is_compressed {
|
||||
let reader = std::fs::File::open(&self.eml_path)?;
|
||||
let mut decoder = GzDecoder::new(reader);
|
||||
let mut buffer = Vec::new();
|
||||
decoder.read_to_end(&mut buffer)?;
|
||||
Ok(buffer)
|
||||
} else {
|
||||
std::fs::read(&self.eml_path).map_err(|e| eyre!("IO Error: {}", &e))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_gmail_meta(&self) -> bool {
|
||||
self.gmail_meta_path.is_some()
|
||||
}
|
||||
|
||||
pub fn read_gmail_meta(&self) -> Option<Result<Vec<u8>>> {
|
||||
// Just using map here returns a `&Option` whereas we want `Option`
|
||||
#[allow(clippy::manual_map)]
|
||||
match &self.gmail_meta_path {
|
||||
Some(p) => Some(std::fs::read(p).map_err(|e| eyre!("IO Error: {}", &e))),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl RawEmailEntry {
|
||||
pub(super) fn new<P: AsRef<std::path::Path>>(path: P) -> Option<RawEmailEntry> {
|
||||
let path = path.as_ref();
|
||||
let stem = path.file_stem()?.to_str()?;
|
||||
let name = path.file_name()?.to_str()?;
|
||||
let is_eml_gz = name.ends_with(".eml.gz");
|
||||
let is_eml = name.ends_with(".eml");
|
||||
if !is_eml_gz && !is_eml {
|
||||
return None;
|
||||
}
|
||||
let is_compressed = is_eml_gz;
|
||||
let folder_name = path.parent()?.file_name()?.to_str()?.to_owned();
|
||||
let eml_path = path.to_path_buf();
|
||||
|
||||
let file_metadata = path.metadata().ok()?;
|
||||
|
||||
// Build a meta path
|
||||
let meta_path = path
|
||||
.parent()?
|
||||
.join(format!("{}.meta", stem.replace(".eml", "")));
|
||||
|
||||
// Only embed it, if it exists
|
||||
let gmail_meta_path = if meta_path.exists() {
|
||||
Some(meta_path)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
tracing::trace!(
|
||||
"Email [c?: {}] {} {:?}",
|
||||
is_compressed,
|
||||
eml_path.display(),
|
||||
gmail_meta_path
|
||||
);
|
||||
Some(RawEmailEntry {
|
||||
folder_name,
|
||||
eml_path,
|
||||
gmail_meta_path,
|
||||
is_compressed,
|
||||
size: file_metadata.len(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ParseableEmail for RawEmailEntry {
|
||||
fn prepare(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn message(&self) -> Result<Cow<'_, [u8]>> {
|
||||
Ok(Cow::Owned(self.read()?))
|
||||
}
|
||||
|
||||
fn path(&self) -> &Path {
|
||||
self.eml_path.as_path()
|
||||
}
|
||||
|
||||
fn meta(&self) -> Result<Option<EmailMeta>> {
|
||||
if self.has_gmail_meta() {
|
||||
Ok(Some(super::meta::parse_meta(self)?.into()))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,133 +0,0 @@
|
||||
//! FIXME: Implement our own Mailbox reader that better implements the spec.
|
||||
//! use jetsci for efficient searching:
|
||||
//! https://github.com/shepmaster/jetscii
|
||||
//! (or aho corasick)
|
||||
//! MBox parsing is also not particularly fast as it currently doesn't use parallelism
|
||||
|
||||
use eyre::eyre;
|
||||
use mbox_reader;
|
||||
use rayon::prelude::*;
|
||||
use tracing;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use super::{Config, ImporterFormat, Message, MessageSender, Result};
|
||||
|
||||
use super::shared::email::EmailMeta;
|
||||
use super::shared::parse::ParseableEmail;
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct Mail {
|
||||
path: PathBuf,
|
||||
/// For now, we go with a very simple implementation:
|
||||
/// Each mal will have a heap-allocated vec of the corresponding
|
||||
/// bytes in the mbox.
|
||||
/// This wastes a lot of allocations and shows the limits of our current abstraction.
|
||||
/// It would be better to just save the headers and ignore the rest.
|
||||
content: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Mbox;
|
||||
|
||||
/// The inner parsing code
|
||||
fn inner_emails(config: &Config, sender: MessageSender) -> Result<Vec<Mail>> {
|
||||
// find all files ending in .mbox
|
||||
let mboxes: Vec<PathBuf> = WalkDir::new(&config.emails_folder_path)
|
||||
.into_iter()
|
||||
.filter_map(|e| match e {
|
||||
Ok(n)
|
||||
if n.path().is_file()
|
||||
&& n.path()
|
||||
.to_str()
|
||||
.map(|e| e.contains(".mbox"))
|
||||
.unwrap_or(false) =>
|
||||
{
|
||||
tracing::trace!("Found mbox file {}", n.path().display());
|
||||
Some(n.path().to_path_buf())
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::info!("Could not read folder: {}", e);
|
||||
if let Err(e) = sender.send(Message::Error(eyre!("Could not read folder: {:?}", e)))
|
||||
{
|
||||
tracing::error!("Error sending error {}", e);
|
||||
}
|
||||
None
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mails: Vec<Mail> = mboxes
|
||||
.into_par_iter()
|
||||
.filter_map(|mbox_file| {
|
||||
let mbox = match mbox_reader::MboxFile::from_file(&mbox_file) {
|
||||
Ok(n) => n,
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
"Could not open mbox file at {}: {}",
|
||||
&mbox_file.display(),
|
||||
e
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let inner_mails: Vec<Mail> = mbox
|
||||
.iter()
|
||||
.filter_map(|e| {
|
||||
let content = match e.message() {
|
||||
Some(n) => n,
|
||||
None => {
|
||||
tracing::error!("Could not parse mail at offset {}", e.offset());
|
||||
return None;
|
||||
}
|
||||
};
|
||||
Some(Mail {
|
||||
path: mbox_file.clone(),
|
||||
content: content.to_owned(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
Some(inner_mails)
|
||||
})
|
||||
.flatten()
|
||||
.collect();
|
||||
Ok(mails)
|
||||
}
|
||||
|
||||
impl ImporterFormat for Mbox {
|
||||
type Item = Mail;
|
||||
|
||||
fn default_path() -> Option<std::path::PathBuf> {
|
||||
None
|
||||
}
|
||||
|
||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>> {
|
||||
inner_emails(config, sender)
|
||||
}
|
||||
}
|
||||
|
||||
impl ParseableEmail for Mail {
|
||||
fn prepare(&mut self) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
fn message(&self) -> Result<Cow<'_, [u8]>> {
|
||||
Ok(self.content.as_slice().into())
|
||||
}
|
||||
fn path(&self) -> &Path {
|
||||
self.path.as_path()
|
||||
}
|
||||
fn meta(&self) -> Result<Option<EmailMeta>> {
|
||||
// The filename is a tag, e.g. `INBOX.mbox`, `WORK.mbox`
|
||||
if let Some(prefix) = self.path.file_stem() {
|
||||
if let Some(s) = prefix.to_str().map(|s| s.to_owned()) {
|
||||
return Ok(Some(EmailMeta {
|
||||
tags: vec![s],
|
||||
is_seen: false,
|
||||
}));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
@ -1,31 +0,0 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub use eyre::Result;
|
||||
|
||||
mod apple_mail;
|
||||
mod gmailbackup;
|
||||
mod mbox;
|
||||
pub mod shared;
|
||||
|
||||
pub use apple_mail::AppleMail;
|
||||
pub use gmailbackup::Gmail;
|
||||
pub use mbox::Mbox;
|
||||
|
||||
pub use crate::types::Config;
|
||||
use shared::parse::ParseableEmail;
|
||||
|
||||
pub use super::{Message, MessageReceiver, MessageSender};
|
||||
|
||||
/// This is implemented by the various formats
|
||||
/// to define how they return email data.
|
||||
pub trait ImporterFormat: Send + Sync {
|
||||
type Item: ParseableEmail;
|
||||
|
||||
/// The default location path where the data for this format resides
|
||||
/// on system. If there is none (such as for mbox) return `None`
|
||||
fn default_path() -> Option<PathBuf>;
|
||||
|
||||
/// Return all the emails in this format.
|
||||
/// Use the sneder to give progress updates via the `ReadProgress` case.
|
||||
fn emails(&self, config: &Config, sender: MessageSender) -> Result<Vec<Self::Item>>;
|
||||
}
|
||||
@ -1,83 +0,0 @@
|
||||
use super::parse::{parse_email, ParseableEmail};
|
||||
use crate::database::{DBMessage, Database};
|
||||
use crate::types::Config;
|
||||
|
||||
use super::super::{Message, MessageSender};
|
||||
|
||||
use eyre::{bail, Result};
|
||||
use rayon::prelude::*;
|
||||
|
||||
pub fn into_database<Mail: ParseableEmail + 'static>(
|
||||
config: &Config,
|
||||
mut emails: Vec<Mail>,
|
||||
tx: MessageSender,
|
||||
) -> Result<usize> {
|
||||
let total = emails.len();
|
||||
tracing::info!("Loaded {} emails", &total);
|
||||
|
||||
// First, communicate the total amount of mails received
|
||||
if let Err(e) = tx.send(Message::WriteTotal(total)) {
|
||||
bail!("Channel Failure {:?}", &e);
|
||||
}
|
||||
|
||||
// Create a new database connection, just for writing
|
||||
let database = Database::new(config.database_path.clone()).unwrap();
|
||||
|
||||
// Save the config into the database
|
||||
if let Err(e) = database.save_config(config.clone()) {
|
||||
bail!("Could not save config to database {:?}", &e);
|
||||
}
|
||||
|
||||
// Consume the connection to begin the import. It will return the `handle` to use for
|
||||
// waiting for the database to finish importing, and the `sender` to submit work.
|
||||
let (sender, handle) = database.import();
|
||||
|
||||
// Iterate over the mails..
|
||||
emails
|
||||
// in paralell..
|
||||
.par_iter_mut()
|
||||
// parsing them
|
||||
.map(|raw_mail| parse_email(raw_mail, &config.sender_emails))
|
||||
// and inserting them into SQLite
|
||||
.for_each(|entry| {
|
||||
// Try to write the message into the database
|
||||
if let Err(e) = match entry {
|
||||
Ok(mail) => sender.send(DBMessage::Mail(Box::new(mail))),
|
||||
Err(e) => sender.send(DBMessage::Error(e)),
|
||||
} {
|
||||
tracing::error!("Error Inserting into Database: {:?}", &e);
|
||||
}
|
||||
// Signal the write
|
||||
if let Err(e) = tx.send(Message::WriteOne) {
|
||||
tracing::error!("Channel Failure: {:?}", &e);
|
||||
}
|
||||
});
|
||||
|
||||
// Tell SQLite there's no more work coming. This will exit the listening loop
|
||||
if let Err(e) = sender.send(DBMessage::Done) {
|
||||
bail!("Channel Failure {:?}", &e);
|
||||
}
|
||||
|
||||
// Wait for SQLite to finish parsing
|
||||
tracing::info!("Waiting for SQLite to finish");
|
||||
|
||||
if let Err(e) = tx.send(Message::FinishingUp) {
|
||||
bail!("Channel Failure {:?}", &e);
|
||||
}
|
||||
|
||||
tracing::trace!("Waiting for database handle...");
|
||||
let output = match handle.join() {
|
||||
Ok(Ok(count)) => Ok(count),
|
||||
Ok(Err(e)) => Err(e),
|
||||
Err(e) => Err(eyre::eyre!("Join Error: {:?}", &e)),
|
||||
};
|
||||
|
||||
// Tell the caller that we're done processing. This will allow leaving the
|
||||
// display loop
|
||||
tracing::trace!("Messaging Done");
|
||||
if let Err(e) = tx.send(Message::Done) {
|
||||
bail!("Channel Failure {:?}", &e);
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
@ -1,50 +0,0 @@
|
||||
use chrono::prelude::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub type Tag = String;
|
||||
|
||||
/// This is based on additional information in some systems such as
|
||||
/// Gmail labels or Apple Mail tags or Apple XML
|
||||
#[derive(Debug, Default)]
|
||||
pub struct EmailMeta {
|
||||
pub tags: Vec<Tag>,
|
||||
pub is_seen: bool,
|
||||
}
|
||||
|
||||
const TAG_SEP: &str = ":|:";
|
||||
|
||||
impl EmailMeta {
|
||||
pub fn tags_from_string(tag_string: &str) -> Vec<String> {
|
||||
tag_string.split(TAG_SEP).map(|e| e.to_string()).collect()
|
||||
}
|
||||
|
||||
pub fn from(is_seen: bool, tag_string: &str) -> Self {
|
||||
let tags = EmailMeta::tags_from_string(tag_string);
|
||||
EmailMeta { tags, is_seen }
|
||||
}
|
||||
|
||||
pub fn tags_string(&self) -> String {
|
||||
self.tags.join(TAG_SEP)
|
||||
}
|
||||
}
|
||||
|
||||
/// Representation of an email
|
||||
#[derive(Debug)]
|
||||
pub struct EmailEntry {
|
||||
pub path: PathBuf,
|
||||
pub sender_domain: String,
|
||||
pub sender_local_part: String,
|
||||
pub sender_name: String,
|
||||
pub datetime: chrono::DateTime<Utc>,
|
||||
pub subject: String,
|
||||
/// The amount of `to:` adresses
|
||||
pub to_count: usize,
|
||||
/// When this email was send to a group, the group name
|
||||
pub to_group: Option<String>,
|
||||
/// The first address and name in `To`, if any
|
||||
pub to_first: Option<(String, String)>,
|
||||
pub is_reply: bool,
|
||||
/// Was this email send from the account we're importing?
|
||||
pub is_send: bool,
|
||||
pub meta: Option<EmailMeta>,
|
||||
}
|
||||
@ -1,76 +0,0 @@
|
||||
use eyre::{bail, Result};
|
||||
use rayon::prelude::*;
|
||||
use tracing::trace;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use super::super::{Message, MessageSender};
|
||||
|
||||
/// Call `FolderAction` on all files in all sub folders in
|
||||
/// folder `folder`.
|
||||
pub fn folders_in<FolderAction, ActionResult, P>(
|
||||
folder: P,
|
||||
sender: MessageSender,
|
||||
action: FolderAction,
|
||||
) -> Result<Vec<ActionResult>>
|
||||
where
|
||||
P: AsRef<Path>,
|
||||
FolderAction: Fn(PathBuf, MessageSender) -> Result<Vec<ActionResult>> + Send + Sync,
|
||||
ActionResult: Send,
|
||||
{
|
||||
let folder = folder.as_ref();
|
||||
if !folder.exists() {
|
||||
bail!("Folder {} does not exist", &folder.display());
|
||||
}
|
||||
// For progress reporting, we collect the iterator in order to
|
||||
// know how many items there are.
|
||||
let items: Vec<_> = std::fs::read_dir(&folder)?.collect();
|
||||
let total = items.len();
|
||||
sender.send(Message::ReadTotal(total))?;
|
||||
Ok(items
|
||||
.into_iter()
|
||||
.par_bridge()
|
||||
.filter_map(|entry| {
|
||||
let path = entry
|
||||
.map_err(|e| tracing::error!("{} {:?}", &folder.display(), &e))
|
||||
.ok()?
|
||||
.path();
|
||||
if !path.is_dir() {
|
||||
return None;
|
||||
}
|
||||
let sender = sender.clone();
|
||||
trace!("Reading folder {}", path.display());
|
||||
action(path.clone(), sender)
|
||||
.map_err(|e| tracing::error!("{} {:?}", path.display(), &e))
|
||||
.ok()
|
||||
})
|
||||
.flatten()
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub fn emails_in<O, F, P: AsRef<Path>>(path: P, sender: MessageSender, make: F) -> Result<Vec<O>>
|
||||
where
|
||||
F: Fn(PathBuf) -> Option<O>,
|
||||
F: Send + Sync + 'static,
|
||||
O: Send + Sync,
|
||||
{
|
||||
let path = path.as_ref();
|
||||
let result = Ok(std::fs::read_dir(path)?
|
||||
.into_iter()
|
||||
.par_bridge()
|
||||
.filter_map(|entry| {
|
||||
let path = entry
|
||||
.map_err(|e| tracing::error!("{} {:?}", &path.display(), &e))
|
||||
.ok()?
|
||||
.path();
|
||||
if path.is_dir() {
|
||||
return None;
|
||||
}
|
||||
trace!("Reading {}", &path.display());
|
||||
make(path)
|
||||
})
|
||||
.collect());
|
||||
// We're done reading the folder
|
||||
sender.send(Message::ReadOne).unwrap();
|
||||
result
|
||||
}
|
||||
@ -1,4 +0,0 @@
|
||||
pub mod database;
|
||||
pub mod email;
|
||||
pub mod filesystem;
|
||||
pub mod parse;
|
||||
@ -1,155 +0,0 @@
|
||||
use chrono::prelude::*;
|
||||
use email_parser::address::{Address, EmailAddress, Mailbox};
|
||||
use eyre::{eyre, Result};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
use super::email::{EmailEntry, EmailMeta};
|
||||
|
||||
/// Different `importer`s can implement this trait to provide the necessary
|
||||
/// data to parse their data into a `EmailEntry`.
|
||||
pub trait ParseableEmail: Send + Sized + Sync {
|
||||
/// This will be called once before `message`, `path` and `meta`
|
||||
/// are called. It can be used to perform parsing operations
|
||||
fn prepare(&mut self) -> Result<()>;
|
||||
/// The message content as bytes
|
||||
fn message(&self) -> Result<Cow<'_, [u8]>>;
|
||||
/// The original path of the email in the filesystem
|
||||
fn path(&self) -> &Path;
|
||||
/// Optional meta information if they're available.
|
||||
/// (Depending on the `importer` capabilities and system)
|
||||
fn meta(&self) -> Result<Option<EmailMeta>>;
|
||||
}
|
||||
|
||||
pub fn parse_email<Entry: ParseableEmail>(
|
||||
entry: &mut Entry,
|
||||
sender_emails: &HashSet<String>,
|
||||
) -> Result<EmailEntry> {
|
||||
if let Err(e) = entry.prepare() {
|
||||
tracing::error!("Prepare Error: {:?}", e);
|
||||
return Err(e);
|
||||
}
|
||||
let content = entry.message()?;
|
||||
match email_parser::email::Email::parse(&content) {
|
||||
Ok(email) => {
|
||||
let path = entry.path();
|
||||
tracing::trace!("Parsing {}", path.display());
|
||||
let (sender_name, _, sender_local_part, sender_domain) =
|
||||
mailbox_to_string(&email.sender);
|
||||
|
||||
let datetime = emaildatetime_to_chrono(&email.date);
|
||||
let subject = email.subject.map(|e| e.to_string()).unwrap_or_default();
|
||||
|
||||
let to_count = match email.to.as_ref() {
|
||||
Some(n) => n.len(),
|
||||
None => 0,
|
||||
};
|
||||
let to = match email.to.as_ref().map(|v| v.first()).flatten() {
|
||||
Some(n) => address_to_name_string(n),
|
||||
None => None,
|
||||
};
|
||||
let to_group = to.as_ref().map(|e| e.0.clone()).flatten();
|
||||
let to_first = to.as_ref().map(|e| (e.1.clone(), e.2.clone()));
|
||||
|
||||
let is_reply = email.in_reply_to.map(|v| !v.is_empty()).unwrap_or(false);
|
||||
|
||||
let meta = entry.meta()?;
|
||||
|
||||
// In order to determine the sender, we have to
|
||||
// build up the address again :-(
|
||||
let is_send = {
|
||||
let email = format!("{}@{}", sender_local_part, sender_domain);
|
||||
sender_emails.contains(&email)
|
||||
};
|
||||
|
||||
Ok(EmailEntry {
|
||||
path: path.to_path_buf(),
|
||||
sender_domain,
|
||||
sender_local_part,
|
||||
sender_name,
|
||||
datetime,
|
||||
subject,
|
||||
meta,
|
||||
is_reply,
|
||||
to_count,
|
||||
to_group,
|
||||
to_first,
|
||||
is_send,
|
||||
})
|
||||
}
|
||||
Err(error) => {
|
||||
let error = eyre!(
|
||||
"Could not parse email (trace to see contents): {:?} [{}]",
|
||||
&error,
|
||||
entry.path().display()
|
||||
);
|
||||
tracing::error!("{:?}", &error);
|
||||
if let Ok(content_string) = String::from_utf8(content.into_owned()) {
|
||||
tracing::trace!("Contents:\n{}\n---\n", content_string);
|
||||
} else {
|
||||
tracing::trace!("Contents:\nInvalid UTF8\n---\n");
|
||||
}
|
||||
Err(error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a conversion from address to the fields we care about:
|
||||
/// ([group name], display name, email address)
|
||||
fn address_to_name_string(address: &Address) -> Option<(Option<String>, String, String)> {
|
||||
match address {
|
||||
Address::Group((names, boxes)) => match (names.first(), boxes.first()) {
|
||||
(group_name, Some(mailbox)) => {
|
||||
let group = group_name.map(|e| e.to_string());
|
||||
let (display_name, address, _, _) = mailbox_to_string(mailbox);
|
||||
Some((group, display_name, address))
|
||||
}
|
||||
_ => None,
|
||||
},
|
||||
Address::Mailbox(mailbox) => {
|
||||
let (display_name, address, _, _) = mailbox_to_string(mailbox);
|
||||
Some((None, display_name, address))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns (display name, email address, local part, domain)
|
||||
fn mailbox_to_string(mailbox: &Mailbox) -> (String, String, String, String) {
|
||||
let names = match mailbox.name.as_ref() {
|
||||
Some(n) => n
|
||||
.iter()
|
||||
.map(|e| e.as_ref())
|
||||
.collect::<Vec<&str>>()
|
||||
.join(" "),
|
||||
None => "".to_owned(),
|
||||
};
|
||||
(
|
||||
names,
|
||||
emailaddress_to_string(&mailbox.address),
|
||||
mailbox.address.local_part.to_string(),
|
||||
mailbox.address.domain.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
fn emailaddress_to_string(address: &EmailAddress) -> String {
|
||||
format!(
|
||||
"{}@{}",
|
||||
address.local_part.to_string(),
|
||||
address.domain.to_string()
|
||||
)
|
||||
}
|
||||
|
||||
fn emaildatetime_to_chrono(dt: &email_parser::time::DateTime) -> chrono::DateTime<Utc> {
|
||||
Utc.ymd(
|
||||
dt.date.year as i32,
|
||||
dt.date.month_number() as u32,
|
||||
dt.date.day as u32,
|
||||
)
|
||||
.and_hms(
|
||||
dt.time.time.hour as u32,
|
||||
dt.time.time.minute as u32,
|
||||
dt.time.time.second as u32,
|
||||
)
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
use super::formats::shared;
|
||||
use super::{Config, ImporterFormat};
|
||||
|
||||
use super::{Message, MessageReceiver};
|
||||
|
||||
use crossbeam_channel::{self, unbounded};
|
||||
use eyre::Result;
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
pub trait Importerlike {
|
||||
fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)>;
|
||||
}
|
||||
|
||||
pub struct Importer<Format: ImporterFormat> {
|
||||
config: Config,
|
||||
format: Format,
|
||||
}
|
||||
|
||||
impl<Format: ImporterFormat + 'static> Importer<Format> {
|
||||
pub fn new(config: Config, format: Format) -> Self {
|
||||
Self { config, format }
|
||||
}
|
||||
}
|
||||
|
||||
impl<Format: ImporterFormat + 'static> Importerlike for Importer<Format> {
|
||||
fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
||||
let Importer { format, .. } = self;
|
||||
let (sender, receiver) = unbounded();
|
||||
|
||||
let config = self.config;
|
||||
let handle: JoinHandle<Result<()>> = std::thread::spawn(move || {
|
||||
let outer_sender = sender.clone();
|
||||
let processed = move || {
|
||||
let emails = format.emails(&config, sender.clone())?;
|
||||
let processed = shared::database::into_database(&config, emails, sender.clone())?;
|
||||
|
||||
Ok(processed)
|
||||
};
|
||||
let result = processed();
|
||||
|
||||
// Send the error away and map it to a crossbeam channel error
|
||||
match result {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => match outer_sender.send(Message::Error(e)) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(eyre::Report::new(e)),
|
||||
},
|
||||
}
|
||||
});
|
||||
Ok((receiver, handle))
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Importerlike + Sized> Importerlike for Box<T> {
|
||||
fn import(self) -> Result<(MessageReceiver, JoinHandle<Result<()>>)> {
|
||||
(*self).import()
|
||||
}
|
||||
}
|
||||
@ -1,151 +0,0 @@
|
||||
use eyre::{bail, eyre, Report, Result};
|
||||
|
||||
use std::sync::{Arc, RwLock};
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
use super::formats::ImporterFormat;
|
||||
use super::importer::Importerlike;
|
||||
use super::Message;
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Data {
|
||||
total_read: usize,
|
||||
read: usize,
|
||||
total_write: usize,
|
||||
write: usize,
|
||||
finishing: bool,
|
||||
done: bool,
|
||||
error: Option<Report>,
|
||||
#[cfg(target_os = "macos")]
|
||||
missing_permissions: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Copy)]
|
||||
pub struct Progress {
|
||||
pub total: usize,
|
||||
pub count: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Copy)]
|
||||
pub struct State {
|
||||
pub finishing: bool,
|
||||
pub done: bool,
|
||||
pub written: usize,
|
||||
#[cfg(target_os = "macos")]
|
||||
pub missing_permissions: bool,
|
||||
}
|
||||
|
||||
/// This can be initialized with a [`MessageSender`] and it will
|
||||
/// automatically tally up the information into a thread-safe
|
||||
/// datastructure
|
||||
pub struct Adapter {
|
||||
producer_lock: Arc<RwLock<Data>>,
|
||||
consumer_lock: Arc<RwLock<Data>>,
|
||||
}
|
||||
|
||||
impl Adapter {
|
||||
#[allow(clippy::new_without_default)]
|
||||
pub fn new() -> Self {
|
||||
let rw_lock = Arc::new(RwLock::default());
|
||||
// FIXME: Look up this warning. It looks like the clones are necessary?
|
||||
#[allow(clippy::redundant_clone)]
|
||||
let producer_lock = rw_lock.clone();
|
||||
#[allow(clippy::redundant_clone)]
|
||||
let consumer_lock = rw_lock.clone();
|
||||
Self {
|
||||
producer_lock,
|
||||
consumer_lock,
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts up a thread that handles the `MessageReceiver` messages
|
||||
/// into state that can be accessed via [`read_count`], [`write_count`] and [`finished`]
|
||||
pub fn process<Format: ImporterFormat + 'static>(
|
||||
&self,
|
||||
importer: super::importer::Importer<Format>,
|
||||
) -> Result<JoinHandle<Result<()>>> {
|
||||
let (receiver, handle) = importer.import()?;
|
||||
let lock = self.producer_lock.clone();
|
||||
let handle = std::thread::spawn(move || {
|
||||
'outer: loop {
|
||||
let mut write_guard = match lock.write() {
|
||||
Ok(n) => n,
|
||||
Err(e) => bail!("RwLock Error: {:?}", e),
|
||||
};
|
||||
for entry in receiver.try_iter() {
|
||||
match entry {
|
||||
Message::ReadTotal(n) => write_guard.total_read = n,
|
||||
Message::ReadOne => {
|
||||
write_guard.read += 1;
|
||||
// Depending on the implementation, we may receive read calls before
|
||||
// the total size is known. We prevent division by zero by
|
||||
// always setting the total to read + 1 in these cases
|
||||
if write_guard.total_read <= write_guard.read {
|
||||
write_guard.total_read = write_guard.read + 1;
|
||||
}
|
||||
}
|
||||
Message::WriteTotal(n) => write_guard.total_write = n,
|
||||
Message::WriteOne => write_guard.write += 1,
|
||||
Message::FinishingUp => write_guard.finishing = true,
|
||||
Message::Done => {
|
||||
write_guard.done = true;
|
||||
break 'outer;
|
||||
}
|
||||
Message::Error(e) => {
|
||||
write_guard.error = Some(e);
|
||||
}
|
||||
#[cfg(target_os = "macos")]
|
||||
Message::MissingPermissions => {
|
||||
write_guard.missing_permissions = true;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
let _ = handle.join().map_err(|op| eyre::eyre!("{:?}", &op))??;
|
||||
|
||||
Ok(())
|
||||
});
|
||||
Ok(handle)
|
||||
}
|
||||
|
||||
pub fn read_count(&self) -> Result<Progress> {
|
||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
||||
Ok(Progress {
|
||||
total: item.total_read,
|
||||
count: item.read,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn write_count(&self) -> Result<Progress> {
|
||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
||||
Ok(Progress {
|
||||
total: item.total_write,
|
||||
count: item.write,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn finished(&self) -> Result<State> {
|
||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
||||
Ok(State {
|
||||
finishing: item.finishing,
|
||||
done: item.done,
|
||||
written: item.write,
|
||||
#[cfg(target_os = "macos")]
|
||||
missing_permissions: item.missing_permissions,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn error(&self) -> Result<Option<Report>> {
|
||||
// We take the error of out of the write lock only if there is an error.
|
||||
let item = self.consumer_lock.read().map_err(|e| eyre!("{:?}", &e))?;
|
||||
let is_error = item.error.is_some();
|
||||
drop(item);
|
||||
if is_error {
|
||||
let mut item = self.producer_lock.write().map_err(|e| eyre!("{:?}", &e))?;
|
||||
Ok(item.error.take())
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
use crossbeam_channel;
|
||||
|
||||
pub(crate) mod formats;
|
||||
#[allow(clippy::module_inception)]
|
||||
mod importer;
|
||||
mod message_adapter;
|
||||
|
||||
use crate::types::Config;
|
||||
pub use formats::shared::email::{EmailEntry, EmailMeta};
|
||||
pub use importer::Importerlike;
|
||||
pub use message_adapter::*;
|
||||
|
||||
use formats::ImporterFormat;
|
||||
|
||||
/// The message that informs of the importers progress
|
||||
#[derive(Debug)]
|
||||
pub enum Message {
|
||||
/// How much progress are we making on reading the contents
|
||||
/// of the emails.
|
||||
/// The `usize` parameter marks the total amount of items to read - if it is known.
|
||||
/// The values here can vary wildly based on the type of Importer `Format` in use.
|
||||
/// A Gmail backup will list the folders and how many of them
|
||||
/// are already read. A mbox format will list other things as there
|
||||
/// no folders.
|
||||
ReadTotal(usize),
|
||||
/// Whenever an item out of the total is read, this message will be emitted
|
||||
ReadOne,
|
||||
/// Similar to [`ReadTotal`]
|
||||
WriteTotal(usize),
|
||||
/// Similar to `ReadOne`
|
||||
WriteOne,
|
||||
/// Once everything has been written, we need to wait for the database
|
||||
/// to sync
|
||||
FinishingUp,
|
||||
/// Finally, this indicates that we're done.
|
||||
Done,
|
||||
/// An error happened during processing
|
||||
Error(eyre::Report),
|
||||
/// A special case for macOS, where a permission error means we have to grant this app
|
||||
/// the right to see the mail folder
|
||||
#[cfg(target_os = "macos")]
|
||||
MissingPermissions,
|
||||
}
|
||||
|
||||
pub type MessageSender = crossbeam_channel::Sender<Message>;
|
||||
pub type MessageReceiver = crossbeam_channel::Receiver<Message>;
|
||||
|
||||
pub fn importer(config: &Config) -> Box<dyn importer::Importerlike> {
|
||||
use crate::types::FormatType::*;
|
||||
match config.format {
|
||||
AppleMail => Box::new(applemail_importer(config.clone())),
|
||||
GmailVault => Box::new(gmail_importer(config.clone())),
|
||||
Mbox => Box::new(gmail_importer(config.clone())),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn gmail_importer(config: Config) -> importer::Importer<formats::Gmail> {
|
||||
importer::Importer::new(config, formats::Gmail::default())
|
||||
}
|
||||
|
||||
pub fn applemail_importer(config: Config) -> importer::Importer<formats::AppleMail> {
|
||||
importer::Importer::new(config, formats::AppleMail::default())
|
||||
}
|
||||
|
||||
pub fn mbox_importer(config: Config) -> importer::Importer<formats::Mbox> {
|
||||
importer::Importer::new(config, formats::Mbox::default())
|
||||
}
|
||||
@ -1,70 +0,0 @@
|
||||
#[cfg(target_os = "macos")]
|
||||
#[macro_use]
|
||||
extern crate objc;
|
||||
|
||||
use tracing_subscriber::fmt;
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
pub mod database;
|
||||
#[cfg(feature = "gui")]
|
||||
pub mod gui;
|
||||
pub mod importer;
|
||||
pub mod model;
|
||||
pub mod types;
|
||||
|
||||
pub fn setup_tracing() {
|
||||
if std::env::var("RUST_LOG").is_err() {
|
||||
std::env::set_var("RUST_LOG", "error")
|
||||
}
|
||||
|
||||
let collector = tracing_subscriber::registry().with(fmt::layer().with_writer(std::io::stdout));
|
||||
|
||||
tracing::subscriber::set_global_default(collector).expect("Unable to set a global collector");
|
||||
}
|
||||
|
||||
/// Create a config for the `cli` and validate the input
|
||||
pub fn make_config() -> types::Config {
|
||||
use std::path::Path;
|
||||
use types::FormatType;
|
||||
let arguments: Vec<String> = std::env::args().collect();
|
||||
let folder = arguments
|
||||
.get(1)
|
||||
.unwrap_or_else(|| usage("Missing email folder argument"));
|
||||
let database = arguments
|
||||
.get(2)
|
||||
.unwrap_or_else(|| usage("Missing database path argument"));
|
||||
let sender = arguments
|
||||
.get(3)
|
||||
.unwrap_or_else(|| usage("Missing sender email address argument"));
|
||||
let format: FormatType = arguments
|
||||
.get(4)
|
||||
.unwrap_or_else(|| usage("Missing sender email address argument"))
|
||||
.into();
|
||||
|
||||
let database_path = Path::new(database);
|
||||
if database_path.is_dir() {
|
||||
panic!(
|
||||
"Database Path can't be a directory: {}",
|
||||
&database_path.display()
|
||||
);
|
||||
}
|
||||
let emails_folder_path = Path::new(folder);
|
||||
// For non-mbox files, we make sure we have a directory
|
||||
if !emails_folder_path.is_dir() {
|
||||
panic!(
|
||||
"Emails Folder Path is not a directory: {}",
|
||||
&emails_folder_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
match crate::types::Config::new(Some(database), folder, vec![sender.to_string()], format) {
|
||||
Ok(n) => n,
|
||||
Err(r) => panic!("Error: {:?}", &r),
|
||||
}
|
||||
}
|
||||
|
||||
fn usage(error: &'static str) -> ! {
|
||||
println!("Usage: cli [email-folder] [database-path] [sender-email-address] [format]");
|
||||
println!("\tExample: cli ~/Library/Mails/V9/ ./db.sqlite my-address@gmail.com apple");
|
||||
panic!("{}", error);
|
||||
}
|
||||
@ -1,243 +0,0 @@
|
||||
//! The `Engine` is the entry point to the data that should be
|
||||
//! displayed in Segmentations.
|
||||
//! See [`Engine`] for more information.
|
||||
//! See also:
|
||||
//! - [`segmentations::`]
|
||||
//! - [`items::`]
|
||||
use eyre::{bail, Result};
|
||||
use lru::LruCache;
|
||||
|
||||
use crate::database::query::{Field, Filter, OtherQuery, Query, ValueField};
|
||||
use crate::model::link::Response;
|
||||
use crate::types::Config;
|
||||
|
||||
use super::link::Link;
|
||||
use super::segmentations;
|
||||
use super::types::{LoadingState, Segment, Segmentation};
|
||||
|
||||
/// This signifies the action we're currently evaluating
|
||||
/// It is used for sending requests and receiving responses
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
|
||||
pub(super) enum Action {
|
||||
/// Recalculate the current `Segmentation` based on a changed aggregation
|
||||
RecalculateSegmentation,
|
||||
/// Push a new `Segmentation`
|
||||
PushSegmentation,
|
||||
/// Load the mails for the current `Segmentation`
|
||||
LoadItems,
|
||||
/// Load all tags
|
||||
AllTags,
|
||||
}
|
||||
|
||||
/// Interact with the `Database`, operate on `Segmentations`, `Segments`, and `Items`.
|
||||
/// `Engine` is used as the input for almost all operations in the
|
||||
/// `items::` and `segmentation::` modules.
|
||||
pub struct Engine {
|
||||
pub(super) search_stack: Vec<ValueField>,
|
||||
pub(super) group_by_stack: Vec<Field>,
|
||||
pub(super) link: Link<Action>,
|
||||
pub(super) segmentations: Vec<Segmentation>,
|
||||
/// Additional filters. See [`segmentations::set_filters`]
|
||||
pub(super) filters: Vec<Filter>,
|
||||
/// This is a very simple cache from ranges to rows.
|
||||
/// It doesn't account for overlapping ranges.
|
||||
/// There's a lot of room for improvement here.
|
||||
pub(super) item_cache: LruCache<usize, LoadingState>,
|
||||
pub(super) known_tags: Vec<String>,
|
||||
}
|
||||
|
||||
impl Engine {
|
||||
pub fn new(config: &Config) -> Result<Self> {
|
||||
let link = super::link::run(config)?;
|
||||
let engine = Engine {
|
||||
link,
|
||||
search_stack: Vec::new(),
|
||||
group_by_stack: vec![default_group_by_stack(0).unwrap()],
|
||||
segmentations: Vec::new(),
|
||||
filters: Vec::new(),
|
||||
item_cache: LruCache::new(10000),
|
||||
known_tags: Vec::new(),
|
||||
};
|
||||
Ok(engine)
|
||||
}
|
||||
|
||||
/// Start the `Engine`. This will create a thread to
|
||||
/// asynchronously communicate with the underlying backend
|
||||
/// in a non-blocking manner.
|
||||
pub fn start(&mut self) -> Result<()> {
|
||||
// The initial segmentation
|
||||
self.link
|
||||
.request(&segmentations::make_query(self)?, Action::PushSegmentation)?;
|
||||
// Get all tags
|
||||
self.link.request(
|
||||
&Query::Other {
|
||||
query: OtherQuery::All(Field::MetaTags),
|
||||
},
|
||||
Action::AllTags,
|
||||
)
|
||||
}
|
||||
|
||||
/// Information on the underlying `Format`. Does it have tags
|
||||
pub fn format_has_tags(&self) -> bool {
|
||||
!self.known_tags.is_empty()
|
||||
}
|
||||
|
||||
/// Information on the underlying `Format`. Does it have `seen` information
|
||||
pub fn format_has_seen(&self) -> bool {
|
||||
// FIXME: The current implementation just assumes that the existance of meta tags also implies is_seen
|
||||
!self.known_tags.is_empty()
|
||||
}
|
||||
|
||||
/// All the known tags in the current emails
|
||||
pub fn known_tags(&self) -> &[String] {
|
||||
&self.known_tags
|
||||
}
|
||||
|
||||
/// Return the current stack of `Segmentations`
|
||||
pub fn segmentations(&self) -> &[Segmentation] {
|
||||
&self.segmentations
|
||||
}
|
||||
|
||||
/// Push a new `Segment` to select a more specific `Segmentation`.
|
||||
///
|
||||
/// Pushing will create an additional `Aggregation` based on the selected
|
||||
/// `Segment`, retrieve the data from the backend, and add it to the
|
||||
/// current stack of `Segmentations`.
|
||||
/// It allows to **drill down** into the data.
|
||||
pub fn push(&mut self, segment: Segment) -> Result<()> {
|
||||
// Assign the segmentation
|
||||
let current = match self.segmentations.last_mut() {
|
||||
Some(n) => n,
|
||||
None => return Ok(()),
|
||||
};
|
||||
current.selected = Some(segment);
|
||||
|
||||
// Create the new search stack
|
||||
self.search_stack = self
|
||||
.segmentations
|
||||
.iter()
|
||||
.filter_map(|e| e.selected.as_ref())
|
||||
.map(|p| p.field.clone())
|
||||
.collect();
|
||||
|
||||
// Add the next group by
|
||||
let index = self.group_by_stack.len();
|
||||
let next = default_group_by_stack(index)
|
||||
.ok_or_else(|| eyre::eyre!("default group by stack out of bounds"))?;
|
||||
self.group_by_stack.push(next);
|
||||
|
||||
// Block UI & Wait for updates
|
||||
self.link
|
||||
.request(&segmentations::make_query(self)?, Action::PushSegmentation)
|
||||
}
|
||||
|
||||
/// Pop the current `Segmentation` from the stack.
|
||||
/// The opposite of [`engine::push`]
|
||||
pub fn pop(&mut self) {
|
||||
if self.group_by_stack.is_empty()
|
||||
|| self.segmentations.is_empty()
|
||||
|| self.search_stack.is_empty()
|
||||
{
|
||||
tracing::error!(
|
||||
"Invalid state. Not everything has the same length: {:?}, {:?}, {:?}",
|
||||
&self.group_by_stack,
|
||||
self.segmentations,
|
||||
self.search_stack
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Remove the last entry of everything
|
||||
self.group_by_stack.remove(self.group_by_stack.len() - 1);
|
||||
self.segmentations.remove(self.segmentations.len() - 1);
|
||||
self.search_stack.remove(self.search_stack.len() - 1);
|
||||
|
||||
// Remove the selection in the last segmentation
|
||||
if let Some(e) = self.segmentations.last_mut() {
|
||||
e.selected = None
|
||||
}
|
||||
|
||||
// Remove any rows that were cached for this segmentation
|
||||
self.item_cache.clear();
|
||||
}
|
||||
|
||||
/// Call this continously to retrieve calculation results and apply them.
|
||||
/// Any mutating function on [`Engine`], such as [`Engine::push`] or [`items::items`]
|
||||
/// require calling this method to apply there results once they're
|
||||
/// available from the asynchronous backend.
|
||||
/// This method is specifically non-blocking for usage in
|
||||
/// `Eventloop` based UI frameworks such as `egui`.
|
||||
pub fn process(&mut self) -> Result<()> {
|
||||
let response = match self.link.receive()? {
|
||||
Some(n) => n,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
match response {
|
||||
Response::Grouped(_, Action::PushSegmentation, p) => {
|
||||
self.segmentations.push(p);
|
||||
// Remove any rows that were cached for this segmentation
|
||||
self.item_cache.clear();
|
||||
}
|
||||
Response::Grouped(_, Action::RecalculateSegmentation, p) => {
|
||||
let len = self.segmentations.len();
|
||||
self.segmentations[len - 1] = p;
|
||||
// Remove any rows that were cached for this segmentation
|
||||
self.item_cache.clear();
|
||||
}
|
||||
Response::Normal(Query::Normal { range, .. }, Action::LoadItems, r) => {
|
||||
for (index, row) in range.zip(r) {
|
||||
let entry = LoadingState::Loaded(row.clone());
|
||||
self.item_cache.put(index, entry);
|
||||
}
|
||||
}
|
||||
Response::Other(Query::Other { .. }, Action::AllTags, r) => {
|
||||
self.known_tags = r;
|
||||
}
|
||||
_ => bail!("Invalid Query / Response combination"),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns true if there're currently calculations open and `process`
|
||||
/// needs to be called. This can be used in `Eventloop` based frameworks
|
||||
/// such as `egui` to know when to continue calling `process` in the `loop`
|
||||
/// ```ignore
|
||||
/// loop {
|
||||
/// self.engine.process().unwrap();
|
||||
/// if self.engine.is_busy() {
|
||||
/// // Call the library function to run the event-loop again.
|
||||
/// ctx.request_repaint();
|
||||
/// }
|
||||
/// }
|
||||
/// ```
|
||||
pub fn is_busy(&self) -> bool {
|
||||
self.link.is_processing() || self.segmentations.is_empty()
|
||||
}
|
||||
|
||||
/// Blocking waiting until the current operation is done
|
||||
/// This is useful for usage on a commandline or in unit tests
|
||||
#[allow(unused)]
|
||||
pub fn wait(&mut self) -> Result<()> {
|
||||
loop {
|
||||
self.process()?;
|
||||
if !self.link.is_processing() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the default aggregation fields for each segmentation stack level
|
||||
pub fn default_group_by_stack(index: usize) -> Option<Field> {
|
||||
match index {
|
||||
0 => Some(Field::Year),
|
||||
1 => Some(Field::SenderDomain),
|
||||
2 => Some(Field::SenderLocalPart),
|
||||
3 => Some(Field::Month),
|
||||
4 => Some(Field::Day),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@ -1,93 +0,0 @@
|
||||
//! Operations related to retrieving `items` from the current `Segmentation`
|
||||
//!
|
||||
//! A `Segmentation` is a aggregation of items into many `Segments`.
|
||||
//! These operations allow retreiving the individual items for all
|
||||
//! segments in the `Segmentation.
|
||||
|
||||
use eyre::Result;
|
||||
|
||||
use super::types::LoadingState;
|
||||
use super::{engine::Action, Engine};
|
||||
use crate::database::{
|
||||
query::{Field, Filter, Query},
|
||||
query_result::QueryRow,
|
||||
};
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
/// Return the `items` in the current `Segmentation`
|
||||
///
|
||||
/// If the items don't exist in the cache, they will be queried
|
||||
/// asynchronously from the database. The return value distinguishes
|
||||
/// between `Loaded` and `Loading` items.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
/// * `range` - The range of items to retrieve. If `None` then all items will be retrieved
|
||||
pub fn items(engine: &mut Engine, range: Option<Range<usize>>) -> Result<Vec<Option<QueryRow>>> {
|
||||
// build an array with either empty values or values from our cache.
|
||||
let mut rows = Vec::new();
|
||||
|
||||
// The given range or all items
|
||||
let range = range.unwrap_or_else(|| Range {
|
||||
start: 0,
|
||||
end: count(engine),
|
||||
});
|
||||
|
||||
let mut missing_data = false;
|
||||
for index in range.clone() {
|
||||
let entry = engine.item_cache.get(&index);
|
||||
let entry = match entry {
|
||||
Some(LoadingState::Loaded(n)) => Some((*n).clone()),
|
||||
Some(LoadingState::Loading) => None,
|
||||
None => {
|
||||
// for simplicity, we keep the "something is missing" state separate
|
||||
missing_data = true;
|
||||
|
||||
// Mark the row as being loaded
|
||||
engine.item_cache.put(index, LoadingState::Loading);
|
||||
None
|
||||
}
|
||||
};
|
||||
rows.push(entry);
|
||||
}
|
||||
// Only if at least some data is missing do we perform the request
|
||||
if missing_data && !range.is_empty() {
|
||||
let request = make_query(engine, range);
|
||||
engine.link.request(&request, Action::LoadItems)?;
|
||||
}
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// The total amount of elements in the current `Segmentation`
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
pub fn count(engine: &Engine) -> usize {
|
||||
let segmentation = match engine.segmentations.last() {
|
||||
Some(n) => n,
|
||||
None => return 0,
|
||||
};
|
||||
segmentation.element_count()
|
||||
}
|
||||
|
||||
/// Make the query for retrieving items
|
||||
fn make_query(engine: &Engine, range: Range<usize>) -> Query {
|
||||
let mut filters = Vec::new();
|
||||
for entry in &engine.search_stack {
|
||||
filters.push(Filter::Like(entry.clone()));
|
||||
}
|
||||
Query::Normal {
|
||||
filters,
|
||||
fields: vec![
|
||||
Field::SenderDomain,
|
||||
Field::SenderLocalPart,
|
||||
Field::Subject,
|
||||
Field::Path,
|
||||
Field::Timestamp,
|
||||
],
|
||||
range,
|
||||
}
|
||||
}
|
||||
@ -1,163 +0,0 @@
|
||||
//! Abstraction to perform asynchronous calculations & queries without blocking UI
|
||||
//!
|
||||
//! This opens a `crossbeam` `channel` to communicate with a backend.
|
||||
//! Each backend operation is send and retrieved in a loop on a thread.
|
||||
//! This allows sending operations into `Link` and retrieving the contents
|
||||
//! asynchronously without blocking the UI.
|
||||
|
||||
use std::sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
};
|
||||
use std::{collections::HashSet, convert::TryInto};
|
||||
|
||||
use crossbeam_channel::{unbounded, Receiver, Sender};
|
||||
use eyre::Result;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::database::{
|
||||
query::Query,
|
||||
query_result::{QueryResult, QueryRow},
|
||||
Database,
|
||||
};
|
||||
use crate::types::Config;
|
||||
|
||||
use super::types::Segmentation;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Response<Context: Send + 'static> {
|
||||
Grouped(Query, Context, Segmentation),
|
||||
Normal(Query, Context, Vec<QueryRow>),
|
||||
/// FIXME: OtherQuery results are currently limited to strings as that's enough right now.
|
||||
Other(Query, Context, Vec<String>),
|
||||
}
|
||||
|
||||
pub(super) type InputSender<Context> = Sender<(Query, Context)>;
|
||||
pub(super) type OutputReciever<Context> = Receiver<Result<Response<Context>>>;
|
||||
|
||||
pub(super) struct Link<Context: Send + 'static> {
|
||||
pub input_sender: InputSender<Context>,
|
||||
pub output_receiver: OutputReciever<Context>,
|
||||
// We need to account for the brief moment where the processing channel is empty
|
||||
// but we're applying the results. If there is a UI update in this window,
|
||||
// the UI will not update again after the changes were applied because an empty
|
||||
// channel indicates completed processing.
|
||||
// There's also a delay between a request taken out of the input channel and being
|
||||
// put into the output channel. In order to account for all of this, we employ a
|
||||
// request counter to know how many requests are currently in the pipeline
|
||||
request_counter: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl<Context: Send + Sync + 'static> Link<Context> {
|
||||
pub fn request(&mut self, query: &Query, context: Context) -> Result<()> {
|
||||
self.request_counter.fetch_add(1, Ordering::Relaxed);
|
||||
self.input_sender.send((query.clone(), context))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn receive(&mut self) -> Result<Option<Response<Context>>> {
|
||||
match self.output_receiver.try_recv() {
|
||||
// We received something
|
||||
Ok(Ok(response)) => {
|
||||
// Only subtract if we successfuly received a value
|
||||
self.request_counter.fetch_sub(1, Ordering::Relaxed);
|
||||
Ok(Some(response))
|
||||
}
|
||||
// We received nothing
|
||||
Err(_) => Ok(None),
|
||||
// There was an error, we forward it
|
||||
Ok(Err(e)) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_processing(&self) -> bool {
|
||||
self.request_counter.load(Ordering::Relaxed) > 0
|
||||
}
|
||||
|
||||
/// This can be used to track the `link` from a different thread.
|
||||
#[allow(unused)]
|
||||
pub fn request_counter(&self) -> Arc<AtomicUsize> {
|
||||
self.request_counter.clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn run<Context: Send + Sync + 'static>(config: &Config) -> Result<Link<Context>> {
|
||||
// Create a new database connection, just for reading
|
||||
let database = Database::new(&config.database_path)?;
|
||||
let (input_sender, input_receiver) = unbounded();
|
||||
let (output_sender, output_receiver) = unbounded();
|
||||
let _ = std::thread::spawn(move || inner_loop(database, input_receiver, output_sender));
|
||||
Ok(Link {
|
||||
input_sender,
|
||||
output_receiver,
|
||||
request_counter: Arc::new(AtomicUsize::new(0)),
|
||||
})
|
||||
}
|
||||
|
||||
fn inner_loop<Context: Send + Sync + 'static>(
|
||||
database: Database,
|
||||
input_receiver: Receiver<(Query, Context)>,
|
||||
output_sender: Sender<Result<Response<Context>>>,
|
||||
) -> Result<()> {
|
||||
loop {
|
||||
let (query, context) = input_receiver.recv()?;
|
||||
let result = database.query(&query)?;
|
||||
let response = match query {
|
||||
Query::Grouped { .. } => {
|
||||
let segmentations = calculate_segmentations(&result)?;
|
||||
Response::Grouped(query, context, segmentations)
|
||||
}
|
||||
Query::Normal { .. } => {
|
||||
let converted = calculate_rows(&result)?;
|
||||
Response::Normal(query, context, converted)
|
||||
}
|
||||
Query::Other { .. } => {
|
||||
let mut results = HashSet::new();
|
||||
for entry in result {
|
||||
match entry {
|
||||
QueryResult::Other(field) => match field.value() {
|
||||
Value::Array(s) => {
|
||||
for n in s {
|
||||
if let Value::String(s) = n {
|
||||
if !results.contains(s) {
|
||||
results.insert(s.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => panic!("Should not end up here"),
|
||||
},
|
||||
_ => panic!("Should not end up here"),
|
||||
}
|
||||
}
|
||||
Response::Other(query, context, results.into_iter().collect())
|
||||
}
|
||||
};
|
||||
output_sender.send(Ok(response))?;
|
||||
}
|
||||
}
|
||||
|
||||
fn calculate_segmentations(result: &[QueryResult]) -> Result<Segmentation> {
|
||||
let mut segmentations = Vec::new();
|
||||
for r in result.iter() {
|
||||
let segmentation = r.try_into()?;
|
||||
segmentations.push(segmentation);
|
||||
}
|
||||
|
||||
Ok(Segmentation::new(segmentations))
|
||||
}
|
||||
|
||||
fn calculate_rows(result: &[QueryResult]) -> Result<Vec<QueryRow>> {
|
||||
Ok(result
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let values = match r {
|
||||
QueryResult::Normal(values) => values,
|
||||
_ => {
|
||||
panic!("Invalid result type, expected `Normal`")
|
||||
}
|
||||
};
|
||||
values.clone()
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
mod engine;
|
||||
pub mod items;
|
||||
mod link;
|
||||
pub mod segmentations;
|
||||
mod types;
|
||||
|
||||
pub use engine::Engine;
|
||||
pub use types::Segment;
|
||||
@ -1,206 +0,0 @@
|
||||
//! Operations on `Segmentations`
|
||||
//!
|
||||
//! `Segmentations` are collections of `Segments` based on an aggregation of `Items`.
|
||||
//!
|
||||
//! A `Segmentation` can be changed to be aggregated on a different `Field.
|
||||
//! - [`aggregations`]
|
||||
//! - [`aggregated_by`]
|
||||
//! - [`set_aggregation`]
|
||||
//! A `Segmentation` can be changed to only return a `Range` of segments.
|
||||
//! - [`current_range`]
|
||||
//! - [`set_current_range`]
|
||||
//! A `Segmentation` has multiple `Segments` which each can be layouted
|
||||
//! to fit into a rectangle.
|
||||
//! - [`layouted_segments]
|
||||
|
||||
use eyre::{eyre, Result};
|
||||
|
||||
use super::engine::Action;
|
||||
use super::{
|
||||
types::{Aggregation, Segment},
|
||||
Engine,
|
||||
};
|
||||
use crate::database::query::{Field, Filter, Query};
|
||||
use std::ops::RangeInclusive;
|
||||
|
||||
/// Filter the `Range` of segments of the current `Segmentation`
|
||||
///
|
||||
/// Returns the `Range` and the total number of segments.
|
||||
/// If no custom range has been set with [`set_segments_range`], returns
|
||||
/// the full range of items, otherwise the custom range.
|
||||
///
|
||||
/// Returns `None` if no current `Segmentation` exists.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
/// * `aggregation` - The aggregation to return the fields for. Required to also return the current aggregation field.
|
||||
pub fn segments_range(engine: &Engine) -> Option<(RangeInclusive<usize>, usize)> {
|
||||
let segmentation = engine.segmentations.last()?;
|
||||
let len = segmentation.len();
|
||||
Some(match &segmentation.range {
|
||||
Some(n) => (0..=len, *n.end()),
|
||||
None => (0..=len, len),
|
||||
})
|
||||
}
|
||||
|
||||
/// Set the `Range` of segments of the current `Segmentation`
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for setting data
|
||||
/// * `range` - The range to apply. `None` to reset it to all `Segments`
|
||||
pub fn set_segments_range(engine: &mut Engine, range: Option<RangeInclusive<usize>>) {
|
||||
if let Some(n) = engine.segmentations.last_mut() {
|
||||
// Make sure the range does not go beyond the current semgents count
|
||||
if let Some(r) = range {
|
||||
let len = n.len();
|
||||
if len > *r.start() && *r.end() < len {
|
||||
n.range = Some(r);
|
||||
}
|
||||
} else {
|
||||
n.range = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Additional filters to use in the query
|
||||
///
|
||||
/// These filters will be evaluated in addition to the `segmentation` conditions
|
||||
/// in the query.
|
||||
/// Setting this value will recalculate the current segmentations.
|
||||
pub fn set_filters(engine: &mut Engine, filters: &[Filter]) -> Result<()> {
|
||||
engine.filters = filters.to_vec();
|
||||
|
||||
// Remove any rows that were cached for this Segmentation
|
||||
engine.item_cache.clear();
|
||||
engine
|
||||
.link
|
||||
.request(&make_query(engine)?, Action::RecalculateSegmentation)
|
||||
}
|
||||
|
||||
/// The fields available for the given aggregation
|
||||
///
|
||||
/// As the user `pushes` Segmentations and dives into the data,
|
||||
/// less fields become available to aggregate by. It is inconsequential
|
||||
/// to aggregate, say, by year, then by month, and then again by year.
|
||||
/// This method returns the possible fields still available for aggregation.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
/// * `aggregation` - The aggregation to return the fields for. Required to also return the current aggregation field.
|
||||
pub fn aggregation_fields(engine: &Engine, aggregation: &Aggregation) -> Vec<Field> {
|
||||
#[allow(clippy::unnecessary_filter_map)]
|
||||
Field::all_cases()
|
||||
.filter_map(|f| {
|
||||
if f == aggregation.field {
|
||||
return Some(f);
|
||||
}
|
||||
if engine.group_by_stack.contains(&f) {
|
||||
None
|
||||
} else {
|
||||
Some(f)
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Return all `Aggregation`s applied for the current `Segmentation`
|
||||
///
|
||||
/// E.g. if we're first aggregating by Year, and then by Month, this
|
||||
/// will return a `Vec` of `[Year, Month]`.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
pub fn aggregated_by(engine: &Engine) -> Vec<Aggregation> {
|
||||
let mut result = Vec::new();
|
||||
// for everything in the current stack
|
||||
let len = engine.group_by_stack.len();
|
||||
for (index, field) in engine.group_by_stack.iter().enumerate() {
|
||||
let value = match (
|
||||
len,
|
||||
engine.segmentations.get(index).map(|e| e.selected.as_ref()),
|
||||
) {
|
||||
(n, Some(Some(segment))) if len == n => Some(segment.field.clone()),
|
||||
_ => None,
|
||||
};
|
||||
result.push(Aggregation {
|
||||
value,
|
||||
field: *field,
|
||||
index,
|
||||
});
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Change the `Field` in the given `Aggregation` to the new one.
|
||||
///
|
||||
/// The `Aggregation` will identify the `Segmentation` to use. So this function
|
||||
/// can be used to change the way a `Segmentation` is the aggregated.
|
||||
///
|
||||
/// Retrieve the available aggregations with [`segmentation::aggregated_by`].
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
/// * `aggregation` - The aggregation to change
|
||||
/// * `field` - The field to aggregate the `aggregation` by.
|
||||
pub fn set_aggregation(
|
||||
engine: &mut Engine,
|
||||
aggregation: &Aggregation,
|
||||
field: &Field,
|
||||
) -> Result<()> {
|
||||
if let Some(e) = engine.group_by_stack.get_mut(aggregation.index) {
|
||||
*e = *field;
|
||||
}
|
||||
// Remove any rows that were cached for this Segmentation
|
||||
engine.item_cache.clear();
|
||||
engine
|
||||
.link
|
||||
.request(&make_query(engine)?, Action::RecalculateSegmentation)
|
||||
}
|
||||
|
||||
/// Return the `Segment`s in the current `Segmentation`. Apply layout based on `Rect`.
|
||||
///
|
||||
/// It will perform the calculations so that all segments fit into bounds.
|
||||
/// The results will be applied to each `Segment`.
|
||||
///
|
||||
/// Returns the layouted segments.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `engine` - The engine to use for retrieving data
|
||||
/// * `Rect` - The bounds into which the segments have to fit.
|
||||
pub fn layouted_segments(engine: &mut Engine, bounds: eframe::egui::Rect) -> Option<&[Segment]> {
|
||||
let segmentation = engine.segmentations.last_mut()?;
|
||||
segmentation.update_layout(bounds);
|
||||
Some(segmentation.items())
|
||||
}
|
||||
|
||||
/// Can another level of aggregation be performed? Based on
|
||||
/// [`Engine::default_group_by_stack`]
|
||||
pub fn can_aggregate_more(engine: &Engine) -> bool {
|
||||
let index = engine.group_by_stack.len();
|
||||
super::engine::default_group_by_stack(index).is_some()
|
||||
}
|
||||
|
||||
/// Perform the query that returns an aggregated `Segmentation`
|
||||
pub(super) fn make_query(engine: &Engine) -> Result<Query> {
|
||||
let mut filters = Vec::new();
|
||||
for entry in &engine.search_stack {
|
||||
filters.push(Filter::Like(entry.clone()));
|
||||
}
|
||||
for entry in &engine.filters {
|
||||
filters.push(entry.clone());
|
||||
}
|
||||
let last = engine
|
||||
.group_by_stack
|
||||
.last()
|
||||
.ok_or_else(|| eyre!("Invalid Segmentation state"))?;
|
||||
Ok(Query::Grouped {
|
||||
filters,
|
||||
group_by: *last,
|
||||
})
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
use crate::database::query::{Field, ValueField};
|
||||
|
||||
/// A aggregation field.
|
||||
/// Contains the `Field` to aggregate by, the `Value` used for aggregation
|
||||
/// As well as the index in the stack of Segmentations that this relates to.
|
||||
pub struct Aggregation {
|
||||
pub(in super::super) value: Option<ValueField>,
|
||||
pub(in super::super) field: Field,
|
||||
pub(in super::super) index: usize,
|
||||
}
|
||||
|
||||
impl Aggregation {
|
||||
/// Return the value in this aggregation as a string
|
||||
pub fn value(&self) -> Option<String> {
|
||||
self.value.as_ref().map(|e| e.value().to_string())
|
||||
}
|
||||
|
||||
/// The name of the field as a `String`
|
||||
pub fn name(&self) -> &str {
|
||||
self.field.name()
|
||||
}
|
||||
|
||||
/// The indes of the field within the given fields
|
||||
pub fn index(&self, in_fields: &[Field]) -> Option<usize> {
|
||||
in_fields.iter().position(|p| p == &self.field)
|
||||
}
|
||||
}
|
||||
@ -1,8 +0,0 @@
|
||||
use crate::database::query_result::QueryRow;
|
||||
|
||||
/// Is a individual row/item being loaded or already loaded.
|
||||
/// Used in a cache to improve the loading of data for the UI.
|
||||
pub enum LoadingState {
|
||||
Loaded(QueryRow),
|
||||
Loading,
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
mod aggregation;
|
||||
mod loading_state;
|
||||
mod segment;
|
||||
mod segmentation;
|
||||
|
||||
pub use aggregation::Aggregation;
|
||||
pub use loading_state::LoadingState;
|
||||
pub use segment::*;
|
||||
pub use segmentation::*;
|
||||
@ -1,61 +0,0 @@
|
||||
use std::convert::TryFrom;
|
||||
|
||||
use eframe::egui::Rect as EguiRect;
|
||||
use eyre::{Report, Result};
|
||||
use treemap::{Mappable, Rect};
|
||||
|
||||
use crate::database::{query::ValueField, query_result::QueryResult};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Segment {
|
||||
pub field: ValueField,
|
||||
pub count: usize,
|
||||
/// A TreeMap Rect
|
||||
pub rect: Rect,
|
||||
}
|
||||
|
||||
impl Segment {
|
||||
/// Perform rect conversion from TreeMap to Egui
|
||||
pub fn layout_rect(&self) -> EguiRect {
|
||||
use eframe::egui::pos2;
|
||||
EguiRect {
|
||||
min: pos2(self.rect.x as f32, self.rect.y as f32),
|
||||
max: pos2(
|
||||
self.rect.x as f32 + self.rect.w as f32,
|
||||
self.rect.y as f32 + self.rect.h as f32,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Mappable for Segment {
|
||||
fn size(&self) -> f64 {
|
||||
self.count as f64
|
||||
}
|
||||
|
||||
fn bounds(&self) -> &Rect {
|
||||
&self.rect
|
||||
}
|
||||
|
||||
fn set_bounds(&mut self, bounds: Rect) {
|
||||
self.rect = bounds;
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> TryFrom<&'a QueryResult> for Segment {
|
||||
type Error = Report;
|
||||
fn try_from(result: &'a QueryResult) -> Result<Self> {
|
||||
let (count, field) = match result {
|
||||
QueryResult::Grouped { count, value } => (count, value),
|
||||
_ => return Err(eyre::eyre!("Invalid result type, expected `Grouped`")),
|
||||
};
|
||||
// so far we can only support one group by at a time.
|
||||
// at least in here. The queries support it
|
||||
|
||||
Ok(Segment {
|
||||
field: field.clone(),
|
||||
count: *count,
|
||||
rect: Rect::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
use eframe::egui::Rect as EguiRect;
|
||||
use treemap::{Rect, TreemapLayout};
|
||||
|
||||
use super::segment::Segment;
|
||||
|
||||
/// A small NewType so that we can keep all the `TreeMap` code in here and don't
|
||||
/// have to do the layout calculation in a widget.
|
||||
#[derive(Debug)]
|
||||
pub struct Segmentation {
|
||||
items: Vec<Segment>,
|
||||
pub selected: Option<Segment>,
|
||||
pub range: Option<std::ops::RangeInclusive<usize>>,
|
||||
}
|
||||
|
||||
impl Segmentation {
|
||||
pub fn new(items: Vec<Segment>) -> Self {
|
||||
Self {
|
||||
items,
|
||||
selected: None,
|
||||
range: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.items.len()
|
||||
}
|
||||
|
||||
/// Update the layout information in the Segments
|
||||
/// based on the current size
|
||||
pub fn update_layout(&mut self, rect: EguiRect) {
|
||||
let layout = TreemapLayout::new();
|
||||
let bounds = Rect::from_points(
|
||||
rect.left() as f64,
|
||||
rect.top() as f64,
|
||||
rect.width() as f64,
|
||||
rect.height() as f64,
|
||||
);
|
||||
layout.layout_items(self.items(), bounds);
|
||||
}
|
||||
|
||||
/// The total amount of items in all the `Segments`.
|
||||
/// E.g. the sum of the count of the `Segments`
|
||||
pub fn element_count(&self) -> usize {
|
||||
self.items.iter().map(|e| e.count).sum::<usize>()
|
||||
}
|
||||
|
||||
/// The items in this `Segmentation`, with range applied
|
||||
pub fn items(&mut self) -> &mut [Segment] {
|
||||
match &self.range {
|
||||
Some(n) => {
|
||||
// we reverse the range
|
||||
let reversed_range = (self.len() - n.end())..=(self.len() - 1);
|
||||
&mut self.items[reversed_range]
|
||||
}
|
||||
None => self.items.as_mut_slice(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,200 +0,0 @@
|
||||
use eyre::{eyre, Result};
|
||||
use rand::Rng;
|
||||
use serde_json::Value;
|
||||
use strum::{self, IntoEnumIterator};
|
||||
use strum_macros::{EnumIter, IntoStaticStr};
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::iter::FromIterator;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::str::FromStr;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, IntoStaticStr, EnumIter)]
|
||||
pub enum FormatType {
|
||||
AppleMail,
|
||||
GmailVault,
|
||||
Mbox,
|
||||
}
|
||||
|
||||
impl FormatType {
|
||||
pub fn all_cases() -> impl Iterator<Item = FormatType> {
|
||||
FormatType::iter()
|
||||
}
|
||||
|
||||
pub fn name(&self) -> &'static str {
|
||||
match self {
|
||||
FormatType::AppleMail => "Apple Mail",
|
||||
FormatType::GmailVault => "Gmail Vault Download",
|
||||
FormatType::Mbox => "Mbox",
|
||||
}
|
||||
}
|
||||
|
||||
/// Forward the importer format location
|
||||
pub fn default_path(&self) -> Option<PathBuf> {
|
||||
use crate::importer::formats::{self, ImporterFormat};
|
||||
match self {
|
||||
FormatType::AppleMail => formats::AppleMail::default_path(),
|
||||
FormatType::GmailVault => formats::Gmail::default_path(),
|
||||
FormatType::Mbox => formats::Mbox::default_path(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FormatType {
|
||||
/// We return a different default, based on the platform we're on
|
||||
/// FIXME: We don't have support for Outlook yet, so on windows we go with Mbox as well
|
||||
fn default() -> Self {
|
||||
#[cfg(target_os = "macos")]
|
||||
return FormatType::AppleMail;
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
return FormatType::Mbox;
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&String> for FormatType {
|
||||
fn from(format: &String) -> Self {
|
||||
FormatType::from(format.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for FormatType {
|
||||
fn from(format: &str) -> Self {
|
||||
match format {
|
||||
"apple" => FormatType::AppleMail,
|
||||
"gmailvault" => FormatType::GmailVault,
|
||||
"mbox" => FormatType::Mbox,
|
||||
_ => panic!("Unknown format: {}", &format),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<FormatType> for String {
|
||||
fn from(format: FormatType) -> Self {
|
||||
match format {
|
||||
FormatType::AppleMail => "apple".to_owned(),
|
||||
FormatType::GmailVault => "gmailvault".to_owned(),
|
||||
FormatType::Mbox => "mbox".to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
/// The path to where the database should be stored
|
||||
pub database_path: PathBuf,
|
||||
/// The path where the emails are
|
||||
pub emails_folder_path: PathBuf,
|
||||
/// The addresses used to send emails
|
||||
pub sender_emails: HashSet<String>,
|
||||
/// The importer format we're using
|
||||
pub format: FormatType,
|
||||
/// Did the user intend to keep the database
|
||||
/// (e.g. is the database path temporary?)
|
||||
pub persistent: bool,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
/// Construct a config from a hashmap of field values.
|
||||
/// For missing fields, take a reasonable default value,
|
||||
/// in order to be somewhat backwards compatible.
|
||||
pub fn from_fields<P: AsRef<Path>>(path: P, fields: HashMap<String, Value>) -> Result<Config> {
|
||||
// The following fields are of version 1.0, so they should aways exist
|
||||
let emails_folder_path_str = fields
|
||||
.get("emails_folder_path")
|
||||
.ok_or_else(|| eyre!("Missing config field emails_folder_path"))?
|
||||
.as_str()
|
||||
.ok_or_else(|| eyre!("Invalid field type for emails_folder_path"))?;
|
||||
let emails_folder_path = PathBuf::from_str(emails_folder_path_str).map_err(|e| {
|
||||
eyre!(
|
||||
"Invalid emails_folder_path: {}: {}",
|
||||
&emails_folder_path_str,
|
||||
e
|
||||
)
|
||||
})?;
|
||||
#[allow(clippy::needless_collect)]
|
||||
let sender_emails: Vec<String> = fields
|
||||
.get("sender_emails")
|
||||
.map(|v| v.as_str().map(|e| e.to_string()))
|
||||
.flatten()
|
||||
.ok_or_else(|| eyre!("Missing config field sender_emails"))?
|
||||
.split(',')
|
||||
.map(|e| e.trim().to_owned())
|
||||
.collect();
|
||||
let format = fields
|
||||
.get("format")
|
||||
.map(|e| e.as_str())
|
||||
.flatten()
|
||||
.map(FormatType::from)
|
||||
.ok_or_else(|| eyre!("Missing config field format_type"))?;
|
||||
let persistent = fields
|
||||
.get("persistent")
|
||||
.map(|e| e.as_bool())
|
||||
.flatten()
|
||||
.ok_or_else(|| eyre!("Missing config field persistent"))?;
|
||||
Ok(Config {
|
||||
database_path: path.as_ref().to_path_buf(),
|
||||
emails_folder_path,
|
||||
sender_emails: HashSet::from_iter(sender_emails.into_iter()),
|
||||
format,
|
||||
persistent,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new<A: AsRef<Path>>(
|
||||
db: Option<A>,
|
||||
mails: A,
|
||||
sender_emails: Vec<String>,
|
||||
format: FormatType,
|
||||
) -> eyre::Result<Self> {
|
||||
// If we don't have a database path, we use a temporary folder.
|
||||
let persistent = db.is_some();
|
||||
let database_path = match db {
|
||||
Some(n) => n.as_ref().to_path_buf(),
|
||||
None => {
|
||||
let number: u32 = rand::thread_rng().gen();
|
||||
let folder = "postsack";
|
||||
let filename = format!("{}.sqlite", number);
|
||||
let mut temp_dir = std::env::temp_dir();
|
||||
temp_dir.push(folder);
|
||||
// the folder has to be created
|
||||
std::fs::create_dir_all(&temp_dir)?;
|
||||
temp_dir.push(filename);
|
||||
temp_dir
|
||||
}
|
||||
};
|
||||
Ok(Config {
|
||||
database_path,
|
||||
emails_folder_path: mails.as_ref().to_path_buf(),
|
||||
sender_emails: HashSet::from_iter(sender_emails.into_iter()),
|
||||
format,
|
||||
persistent,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn into_fields(&self) -> Option<HashMap<String, Value>> {
|
||||
let mut new = HashMap::new();
|
||||
new.insert(
|
||||
"database_path".to_owned(),
|
||||
self.database_path.to_str()?.into(),
|
||||
);
|
||||
new.insert(
|
||||
"emails_folder_path".to_owned(),
|
||||
self.emails_folder_path.to_str()?.into(),
|
||||
);
|
||||
new.insert("persistent".to_owned(), self.persistent.into());
|
||||
new.insert(
|
||||
"sender_emails".to_owned(),
|
||||
self.sender_emails
|
||||
.iter()
|
||||
.cloned()
|
||||
.collect::<Vec<String>>()
|
||||
.join(",")
|
||||
.into(),
|
||||
);
|
||||
let format: String = self.format.into();
|
||||
new.insert("format".to_owned(), format.into());
|
||||
|
||||
Some(new)
|
||||
}
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
mod config;
|
||||
|
||||
pub use config::{Config, FormatType};
|
||||
Loading…
Reference in New Issue