search: add Message-ID, and other header search support

Add support for searching by Message-ID, In-Reply-To, References, or any
header with the following keywords:

- "message-id:term", "msg-id:term"
- "in-reply-to:term"
- "references:term"
- "header:title,value"

Signed-off-by: Manos Pitsidianakis <manos@pitsidianak.is>

meli/docs/meli.1

@@ -287,10 +287,12 @@ Sqlite3 on the contrary at reasonable mailbox sizes should have a non noticeable
 .HorizontalRule
 .Bl -dash -compact
 .It
-.Li query = \&"(\&" query \&")\&" | from | to | cc | bcc | alladdresses | subject | flags | has_attachments | query \&"or\&" query | query \&"and\&" query | not query
+.Li query = \&"(\&" query \&")\&" | from | to | cc | bcc | message_id | in_reply_to | references | header | all_addresses | subject | flags | has_attachment | query \&"or\&" query | query \&"and\&" query | not query
 .It
 .Li not = \&"not\&" | \&"!\&"
 .It
+.Li has_attachment = \&"has:attachment\&" | \&"has:attachments\&"
+.It
 .Li quoted = ALPHA / SP *(ALPHA / DIGIT / SP)
 .It
 .Li term = ALPHA *(ALPHA / DIGIT) | DQUOTE quoted DQUOTE
@@ -301,6 +303,8 @@ Sqlite3 on the contrary at reasonable mailbox sizes should have a non noticeable
 .It
 .Li flagterm = flagval | flagval \&",\&" flagterm
 .It
+.Li flags = \&"flag:\&" flag | \&"flags:\&" flag | \&"tag:\&" flag | \&"tags:\&" flag | \&"is:\&" flag
+.It
 .Li from = \&"from:\&" term
 .It
 .Li to = \&"to:\&" term
@@ -309,11 +313,21 @@ Sqlite3 on the contrary at reasonable mailbox sizes should have a non noticeable
 .It
 .Li bcc = \&"bcc:\&" term
 .It
-.Li alladdresses = \&"alladdresses:\&" term
+.Li message_id = \&"message-id:\&" term | \&"msg-id:\&" term
 .It
-.Li subject = \&"subject:\&" term
+.Li in_reply_to = \&"in-reply-to:\&" term
+.It
+.Li references = \&"references:\&" term
+.It
+.Li header = \&"header:\&" field_name \&",\&" field_value
 .It
-.Li flags = \&"flags:\&" flag | \&"tags:\&" flag | \&"is:\&" flag
+.Li field_name = term
+.It
+.Li field_value = term
+.It
+.Li all_addresses = \&"all-addresses:\&" term
+.It
+.Li subject = \&"subject:\&" term
 .El
 .Sh FLAGS
 .Nm

meli/src/sqlite3.rs

@@ -496,7 +496,7 @@ pub fn query_to_sql(q: &Query) -> String {
                 s.extend(escape_double_quote(t).chars());
                 s.push_str("%\" ");
             }
-            AllText(t) => {
+            Body(t) | AllText(t) => {
                 s.push_str("body_text LIKE \"%");
                 s.extend(escape_double_quote(t).chars());
                 s.push_str("%\" ");

melib/src/imap/search.rs

@@ -197,6 +197,14 @@ impl ToImapSearch for Query {
                     s.extend(escape_double_quote(t).chars());
                     s.push('"');
                 }
+                Q(Header(t, v)) => {
+                    space_pad!(s);
+                    s.push_str(r#"HEADER ""#);
+                    s.push_str(t.as_str());
+                    s.push_str(r#"" ""#);
+                    s.extend(escape_double_quote(v).chars());
+                    s.push('"');
+                }
                 Q(AllAddresses(t)) => {
                     let is_empty = space_pad!(s);
                     if !is_empty {
@@ -261,7 +269,7 @@ mod tests {
     #[test]
     fn test_imap_query_search() {
         let (_, q) = query().parse_complete("subject: test and i").unwrap();
-        assert_eq!(&q.to_imap_search(), r#"SUBJECT "test" TEXT "i""#);
+        assert_eq!(&q.to_imap_search(), r#"SUBJECT "test" BODY "i""#);
 
         let (_, q) = query().parse_complete("is:unseen").unwrap();
         assert_eq!(&q.to_imap_search(), r#"UNSEEN"#);

melib/src/jmap/email.rs

@@ -679,6 +679,13 @@ impl From<crate::search::Query> for Filter<EmailFilterCondition, EmailObject> {
                             .into(),
                     );
                 }
+                Header(ref t, ref v) => {
+                    *f = Filter::Condition(
+                        EmailFilterCondition::new()
+                            .header(vec![t.as_str().into(), v.to_string().into()])
+                            .into(),
+                    );
+                }
                 AllAddresses(_) => {
                     // [ref:TODO]: implement AllAddresses query for jmap
                 }

melib/src/notmuch/mod.rs

@@ -978,7 +978,7 @@ impl MailBackend for NotmuchDb {
             } else {
                 String::new()
             };
-            melib_query.query_to_string(&mut query_s);
+            melib_query.query_to_string(&mut query_s)?;
             let query: Query = Query::new(&database, &query_s)?;
             let iter = query.search()?;
             for message in iter {

melib/src/notmuch/query.rs

@@ -22,6 +22,7 @@
 use std::{borrow::Cow, ffi::CString, ptr::NonNull, sync::Arc};
 
 use crate::{
+    email::HeaderName,
     error::{Error, ErrorKind, Result},
     notmuch::{
         ffi::{
@@ -118,11 +119,11 @@ impl Drop for Query<'_> {
 }
 
 pub trait MelibQueryToNotmuchQuery {
-    fn query_to_string(&self, ret: &mut String);
+    fn query_to_string(&self, ret: &mut String) -> Result<()>;
 }
 
 impl MelibQueryToNotmuchQuery for crate::search::Query {
-    fn query_to_string(&self, ret: &mut String) {
+    fn query_to_string(&self, ret: &mut String) -> Result<()> {
         use crate::search::Query::*;
         match self {
             Before(timestamp) => {
@@ -167,8 +168,59 @@ impl MelibQueryToNotmuchQuery for crate::search::Query {
                 }
                 ret.push('"');
             }
-            InReplyTo(_s) | References(_s) | AllAddresses(_s) => {}
-            /* * * * */
+            AllAddresses(s) => {
+                return <Self as MelibQueryToNotmuchQuery>::query_to_string(
+                    &Or(
+                        Box::new(From(s.to_string())),
+                        Box::new(Or(
+                            Box::new(Cc(s.to_string())),
+                            Box::new(Bcc(s.to_string())),
+                        )),
+                    ),
+                    ret,
+                );
+            }
+            Header(t, v) if t == HeaderName::MESSAGE_ID => {
+                ret.push_str("id:\"");
+                for c in v.chars() {
+                    if c == '"' {
+                        ret.push_str("\\\"");
+                    } else {
+                        ret.push(c);
+                    }
+                }
+                ret.push('"');
+            }
+            InReplyTo(s) => {
+                return <Self as MelibQueryToNotmuchQuery>::query_to_string(
+                    &Header(HeaderName::IN_REPLY_TO, s.to_string()),
+                    ret,
+                )
+            }
+            References(s) => {
+                return <Self as MelibQueryToNotmuchQuery>::query_to_string(
+                    &Header(HeaderName::REFERENCES, s.to_string()),
+                    ret,
+                )
+            }
+            Header(t, v) => {
+                // See: <https://stackoverflow.com/questions/37480617/search-for-custom-header-value-in-notmuch>
+                for c in t.as_str().chars() {
+                    if c == '-' {
+                        continue;
+                    }
+                    ret.push(c);
+                }
+                ret.push_str(":\"");
+                for c in v.chars() {
+                    if c == '"' {
+                        ret.push_str("\\\"");
+                    } else {
+                        ret.push(c);
+                    }
+                }
+                ret.push('"');
+            }
             Body(s) => {
                 ret.push_str("body:\"");
                 for c in s.chars() {
@@ -224,27 +276,30 @@ impl MelibQueryToNotmuchQuery for crate::search::Query {
             }
             And(q1, q2) => {
                 ret.push('(');
-                q1.query_to_string(ret);
+                q1.query_to_string(ret)?;
                 ret.push_str(") AND (");
-                q2.query_to_string(ret);
+                q2.query_to_string(ret)?;
                 ret.push(')');
             }
             Or(q1, q2) => {
                 ret.push('(');
-                q1.query_to_string(ret);
+                q1.query_to_string(ret)?;
                 ret.push_str(") OR (");
-                q2.query_to_string(ret);
+                q2.query_to_string(ret)?;
                 ret.push(')');
             }
             Not(q) => {
                 ret.push_str("(NOT (");
-                q.query_to_string(ret);
+                q.query_to_string(ret)?;
                 ret.push_str("))");
             }
-            Answered => todo!(),
-            AnsweredBy { .. } => todo!(),
-            Larger { .. } => todo!(),
-            Smaller { .. } => todo!(),
+            Answered | AnsweredBy { .. } | Larger { .. } | Smaller { .. } => {
+                return Err(
+                    Error::new(format!("{:?} query is not implemented for notmuch", self))
+                        .set_kind(ErrorKind::NotImplemented),
+                );
+            }
         }
+        Ok(())
     }
 }

melib/src/search.rs

@@ -24,9 +24,12 @@ use std::{borrow::Cow, convert::TryFrom};
 pub use query_parser::query;
 use Query::*;
 
-use crate::utils::{
-    datetime::{formats, UnixTimestamp},
-    parsec::*,
+use crate::{
+    email::headers::HeaderName,
+    utils::{
+        datetime::{formats, UnixTimestamp},
+        parsec::*,
+    },
 };
 
 #[derive(Clone, Debug, PartialEq, Serialize)]
@@ -35,7 +38,7 @@ pub enum Query {
     After(UnixTimestamp),
     Between(UnixTimestamp, UnixTimestamp),
     On(UnixTimestamp),
-    /* * * * */
+    Header(HeaderName, String),
     From(String),
     To(String),
     Cc(String),
@@ -43,11 +46,9 @@ pub enum Query {
     InReplyTo(String),
     References(String),
     AllAddresses(String),
-    /* * * * */
     Body(String),
     Subject(String),
     AllText(String),
-    /* * * * */
     Flags(Vec<String>),
     HasAttachment,
     And(Box<Query>, Box<Query>),
@@ -84,10 +85,15 @@ impl QueryTrait for crate::Envelope {
                 self.date() > timestamp.saturating_sub(60 * 60 * 24)
                     && self.date() < *timestamp + 60 * 60 * 24
             }
-            From(s) => self.other_headers()["From"].contains(s),
-            To(s) => self.other_headers()["To"].contains(s),
-            Cc(s) => self.other_headers()["Cc"].contains(s),
-            Bcc(s) => self.other_headers()["Bcc"].contains(s),
+            Header(name, needle) => self
+                .other_headers()
+                .get(name)
+                .map(|h| h.contains(needle.as_str()))
+                .unwrap_or(false),
+            From(s) => self.other_headers()[HeaderName::FROM].contains(s),
+            To(s) => self.other_headers()[HeaderName::TO].contains(s),
+            Cc(s) => self.other_headers()[HeaderName::CC].contains(s),
+            Bcc(s) => self.other_headers()[HeaderName::BCC].contains(s),
             AllAddresses(s) => {
                 self.is_match(&From(s.clone()))
                     || self.is_match(&To(s.clone()))
@@ -95,19 +101,19 @@ impl QueryTrait for crate::Envelope {
                     || self.is_match(&Bcc(s.clone()))
             }
             Flags(v) => v.iter().any(|s| self.flags() == s.as_str()),
-            Subject(s) => self.other_headers()["Subject"].contains(s),
+            Subject(s) => self.other_headers()[HeaderName::SUBJECT].contains(s),
             HasAttachment => self.has_attachments(),
             And(q_a, q_b) => self.is_match(q_a) && self.is_match(q_b),
             Or(q_a, q_b) => self.is_match(q_a) || self.is_match(q_b),
             Not(q) => !self.is_match(q),
-            InReplyTo(_) => {
-                log::warn!("Filtering with InReplyTo is unimplemented.");
-                false
-            }
-            References(_) => {
-                log::warn!("Filtering with References is unimplemented.");
-                false
-            }
+            InReplyTo(s) => <Self as QueryTrait>::is_match(
+                self,
+                &Query::Header(HeaderName::IN_REPLY_TO, s.to_string()),
+            ),
+            References(s) => <Self as QueryTrait>::is_match(
+                self,
+                &Query::Header(HeaderName::REFERENCES, s.to_string()),
+            ),
             AllText(_) => {
                 log::warn!("Filtering with AllText is unimplemented.");
                 false
@@ -271,6 +277,58 @@ pub mod query_parser {
         .map(Query::Bcc)
     }
 
+    fn message_id<'a>() -> impl Parser<'a, Query> {
+        prefix(
+            whitespace_wrap(either(
+                match_literal("message-id:"),
+                match_literal("msg-id:"),
+            )),
+            whitespace_wrap(literal()),
+        )
+        .map(|s| Query::Header(HeaderName::MESSAGE_ID, s))
+    }
+
+    fn in_reply_to<'a>() -> impl Parser<'a, Query> {
+        prefix(
+            whitespace_wrap(match_literal("in-reply-to:")),
+            whitespace_wrap(literal()),
+        )
+        .map(|s| Query::Header(HeaderName::IN_REPLY_TO, s))
+    }
+
+    fn references<'a>() -> impl Parser<'a, Query> {
+        prefix(
+            whitespace_wrap(match_literal("references:")),
+            whitespace_wrap(literal()),
+        )
+        .map(|s| Query::Header(HeaderName::REFERENCES, s))
+    }
+
+    fn header<'a>() -> impl Parser<'a, Query> {
+        prefix(
+            whitespace_wrap(match_literal("header:")),
+            pair(
+                suffix(
+                    whitespace_wrap(move |input| {
+                        is_not(b",").parse(input).and_then(|(last_input, _)| {
+                            {
+                                |s| {
+                                    <HeaderName as std::str::FromStr>::from_str(s)
+                                        .map(|res| ("", res))
+                                        .map_err(|_| s)
+                                }
+                            }
+                            .parse(last_input)
+                        })
+                    }),
+                    whitespace_wrap(match_literal(",")),
+                ),
+                whitespace_wrap(literal()),
+            ),
+        )
+        .map(|(t1, t2)| Query::Header(t1, t2))
+    }
+
     fn all_addresses<'a>() -> impl Parser<'a, Query> {
         prefix(
             whitespace_wrap(match_literal("all-addresses:")),
@@ -283,7 +341,7 @@ pub mod query_parser {
         move |input| {
             whitespace_wrap(match_literal_anycase("or"))
                 .parse(input)
-                .and_then(|(last_input, _)| query().parse(debug!(last_input)))
+                .and_then(|(last_input, _)| query().parse(last_input))
         }
     }
 
@@ -308,9 +366,12 @@ pub mod query_parser {
 
     fn has_attachment<'a>() -> impl Parser<'a, Query> {
         move |input| {
-            whitespace_wrap(match_literal_anycase("has:attachment"))
-                .map(|()| Query::HasAttachment)
-                .parse(input)
+            whitespace_wrap(either(
+                match_literal_anycase("has:attachment"),
+                match_literal_anycase("has:attachments"),
+            ))
+            .map(|()| Query::HasAttachment)
+            .parse(input)
         }
     }
 
@@ -332,11 +393,17 @@ pub mod query_parser {
     fn flags<'a>() -> impl Parser<'a, Query> {
         move |input| {
             whitespace_wrap(either(
+                match_literal_anycase("is:"),
                 either(
-                    match_literal_anycase("flags:"),
-                    match_literal_anycase("tags:"),
+                    either(
+                        match_literal_anycase("flag:"),
+                        match_literal_anycase("flags:"),
+                    ),
+                    either(
+                        match_literal_anycase("tag:"),
+                        match_literal_anycase("tags:"),
+                    ),
                 ),
-                match_literal_anycase("is:"),
             ))
             .parse(input)
             .and_then(|(rest, _)| {
@@ -375,7 +442,7 @@ pub mod query_parser {
     ///
     /// let input = "test";
     /// let query = query().parse(input);
-    /// assert_eq!(Ok(("", Query::AllText("test".to_string()))), query);
+    /// assert_eq!(Ok(("", Query::Body("test".to_string()))), query);
     /// ```
     pub fn query<'a>() -> impl Parser<'a, Query> {
         move |input| {
@@ -385,6 +452,10 @@ pub mod query_parser {
                 .or_else(|_| to().parse(input))
                 .or_else(|_| cc().parse(input))
                 .or_else(|_| bcc().parse(input))
+                .or_else(|_| message_id().parse(input))
+                .or_else(|_| in_reply_to().parse(input))
+                .or_else(|_| references().parse(input))
+                .or_else(|_| header().parse(input))
                 .or_else(|_| all_addresses().parse(input))
                 .or_else(|_| subject().parse(input))
                 .or_else(|_| before().parse(input))
@@ -409,7 +480,7 @@ pub mod query_parser {
                         .map(|(_, s)| s != "and" && s != "or" && s != "not")
                         .unwrap_or(false)
                 {
-                    result.map(|(r, s)| (r, AllText(s)))
+                    result.map(|(r, s)| (r, Body(s)))
                 } else {
                     Err("")
                 }
@@ -474,13 +545,13 @@ mod tests {
                 "",
                 And(
                     Box::new(Subject("test".to_string())),
-                    Box::new(AllText("i".to_string()))
+                    Box::new(Body("i".to_string()))
                 )
             )),
             query().parse_complete("subject:test and i")
         );
         assert_eq!(
-            Ok(("", AllText("test".to_string()))),
+            Ok(("", Body("test".to_string()))),
             query().parse_complete("test")
         );
         assert_eq!(
@@ -547,6 +618,28 @@ mod tests {
                 "(from:Manos and (subject:foo or subject:bar) and (from:woo or from:my))"
             )
         );
+        assert_eq!(
+            Ok((
+                "",
+                Or(
+                    Box::new(Header(
+                        HeaderName::MESSAGE_ID,
+                        "123_user@example.org".to_string()
+                    )),
+                    Box::new(And(
+                        Box::new(Header(
+                            HeaderName::MESSAGE_ID,
+                            "1234_user@example.org".to_string()
+                        )),
+                        Box::new(Body("header:List-ID,meli-devel".to_string()))
+                    ))
+                )
+            )),
+            query().parse_complete(
+                "(message-id:123_user@example.org or (msg-id:1234_user@example.org) and \
+                 (header:List-ID,meli-devel))"
+            )
+        );
         assert_eq!(
             Ok(("", Flags(vec!["test".to_string(), "testtest".to_string()]))),
             query().parse_complete("flags:test,testtest")