diff --git a/Cargo.lock b/Cargo.lock index 7c20e254..e9c8dc80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1171,6 +1171,7 @@ dependencies = [ "bitflags", "data-encoding", "encoding", + "encoding_rs", "flate2", "futures", "indexmap", @@ -1182,6 +1183,7 @@ dependencies = [ "nix", "nom", "notify", + "regex", "rusqlite", "serde", "serde_derive", diff --git a/Cargo.toml b/Cargo.toml index 6a958ac0..734b7677 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,10 +26,6 @@ path = "src/lib.rs" name = "managesieve-client" path = "src/managesieve.rs" -#[[bin]] -#name = "async" -#path = "src/async.rs" - [dependencies] async-task = "^4.2.0" bincode = { version = "^1.3.0", default-features = false } diff --git a/docs/meli.conf.5 b/docs/meli.conf.5 index 09046128..fa511369 100644 --- a/docs/meli.conf.5 +++ b/docs/meli.conf.5 @@ -505,6 +505,12 @@ Example: "INBOX/Drafts" = { sort_order = 1 } "INBOX/Lists" = { sort_order = 2 } .Ed +.It Ic encoding Ar String +.Pq Em optional +Override the default utf-8 charset for the mailbox name. +Useful only for mUTF-7 mailboxes. +.\" default value +.Pq Em "utf7", "utf-7", "utf8", "utf-8" .El .Sh COMPOSING Composing specific options diff --git a/melib/Cargo.toml b/melib/Cargo.toml index ed0c64c1..9fd0b4a5 100644 --- a/melib/Cargo.toml +++ b/melib/Cargo.toml @@ -26,6 +26,7 @@ bincode = { version = "^1.3.0", default-features = false } bitflags = "1.0" data-encoding = { version = "2.1.1" } encoding = { version = "0.2.33", default-features = false } +encoding_rs = { version = "^0.8" } flate2 = { version = "1.0.16", optional = true } futures = "0.3.5" @@ -38,6 +39,7 @@ native-tls = { version = "0.2.3", default-features = false, optional = true } nix = "^0.24" nom = { version = "7" } notify = { version = "4.0.15", optional = true } +regex = { version = "1" } rusqlite = { version = "^0.28", default-features = false, optional = true } serde = { version = "1.0.71", features = ["rc", ] } serde_derive = "1.0.71" diff --git a/melib/src/backends.rs b/melib/src/backends.rs index 1952123b..be7e8c87 100644 --- a/melib/src/backends.rs +++ b/melib/src/backends.rs @@ -19,6 +19,7 @@ * along with meli. If not, see . */ +pub mod utf7; use smallvec::SmallVec; #[cfg(feature = "imap_backend")] @@ -555,10 +556,10 @@ impl SpecialUsageMailbox { pub trait BackendMailbox: Debug { fn hash(&self) -> MailboxHash; + /// Final component of `path`. fn name(&self) -> &str; /// Path of mailbox within the mailbox hierarchy, with `/` as separator. fn path(&self) -> &str; - fn change_name(&mut self, new_name: &str); fn clone(&self) -> Mailbox; fn children(&self) -> &[MailboxHash]; fn parent(&self) -> Option; diff --git a/melib/src/backends/imap/mailbox.rs b/melib/src/backends/imap/mailbox.rs index baa918ca..0e8aa7fe 100644 --- a/melib/src/backends/imap/mailbox.rs +++ b/melib/src/backends/imap/mailbox.rs @@ -83,10 +83,6 @@ impl BackendMailbox for ImapMailbox { &self.path } - fn change_name(&mut self, s: &str) { - self.name = s.to_string(); - } - fn children(&self) -> &[MailboxHash] { &self.children } diff --git a/melib/src/backends/jmap/mailbox.rs b/melib/src/backends/jmap/mailbox.rs index abeae0bf..023fb1dc 100644 --- a/melib/src/backends/jmap/mailbox.rs +++ b/melib/src/backends/jmap/mailbox.rs @@ -58,8 +58,6 @@ impl BackendMailbox for JmapMailbox { &self.path } - fn change_name(&mut self, _s: &str) {} - fn clone(&self) -> Mailbox { Box::new(std::clone::Clone::clone(self)) } diff --git a/melib/src/backends/maildir.rs b/melib/src/backends/maildir.rs index 234b6e98..dbfd57fe 100644 --- a/melib/src/backends/maildir.rs +++ b/melib/src/backends/maildir.rs @@ -220,10 +220,6 @@ impl BackendMailbox for MaildirMailbox { self.path.to_str().unwrap_or_else(|| self.name()) } - fn change_name(&mut self, s: &str) { - self.name = s.to_string(); - } - fn children(&self) -> &[MailboxHash] { &self.children } diff --git a/melib/src/backends/mbox.rs b/melib/src/backends/mbox.rs index e95951f6..3f8a1a84 100644 --- a/melib/src/backends/mbox.rs +++ b/melib/src/backends/mbox.rs @@ -214,10 +214,6 @@ impl BackendMailbox for MboxMailbox { self.path.to_str().unwrap() } - fn change_name(&mut self, s: &str) { - self.name = s.to_string(); - } - fn clone(&self) -> Mailbox { Box::new(MboxMailbox { hash: self.hash, diff --git a/melib/src/backends/nntp/mailbox.rs b/melib/src/backends/nntp/mailbox.rs index bb42ab65..67b976ed 100644 --- a/melib/src/backends/nntp/mailbox.rs +++ b/melib/src/backends/nntp/mailbox.rs @@ -18,6 +18,7 @@ * You should have received a copy of the GNU General Public License * along with meli. If not, see . */ + use crate::backends::{ BackendMailbox, LazyCountSet, Mailbox, MailboxHash, MailboxPermissions, SpecialUsageMailbox, }; @@ -58,10 +59,6 @@ impl BackendMailbox for NntpMailbox { &self.nntp_path } - fn change_name(&mut self, s: &str) { - self.nntp_path = s.to_string(); - } - fn children(&self) -> &[MailboxHash] { &[] } diff --git a/melib/src/backends/notmuch.rs b/melib/src/backends/notmuch.rs index c8d69c71..8c7cca0c 100644 --- a/melib/src/backends/notmuch.rs +++ b/melib/src/backends/notmuch.rs @@ -253,8 +253,6 @@ impl BackendMailbox for NotmuchMailbox { self.path.as_str() } - fn change_name(&mut self, _s: &str) {} - fn clone(&self) -> Mailbox { Box::new(std::clone::Clone::clone(self)) } diff --git a/melib/src/backends/utf7.rs b/melib/src/backends/utf7.rs new file mode 100644 index 00000000..06c9ad7f --- /dev/null +++ b/melib/src/backends/utf7.rs @@ -0,0 +1,196 @@ +/* + * MIT License + * + * Copyright (c) 2021 Ilya Medvedev + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Code from */ + +//! A Rust library for encoding and decoding [UTF-7](https://datatracker.ietf.org/doc/html/rfc2152) string as defined by the [IMAP](https://datatracker.ietf.org/doc/html/rfc3501) standard in [RFC 3501 (#5.1.3)](https://datatracker.ietf.org/doc/html/rfc3501#section-5.1.3). +//! +//! Idea is based on Python [mutf7](https://github.com/cheshire-mouse/mutf7) library. + +use encoding_rs::UTF_16BE; +use regex::{Captures, Regex}; + +/// Encode UTF-7 IMAP mailbox name +/// +/// +pub fn encode_utf7_imap(text: String) -> String { + let mut result = "".to_string(); + let text = text.replace('&', "&-"); + let mut text = text.as_str(); + while !text.is_empty() { + result = format!("{}{}", result, get_ascii(text)); + text = remove_ascii(text); + if !text.is_empty() { + let tmp = get_nonascii(text); + result = format!("{}{}", result, encode_modified_utf7(tmp)); + text = remove_nonascii(text); + } + } + result +} +fn is_ascii_custom(c: u8) -> bool { + (0x20..=0x7f).contains(&c) +} + +fn get_ascii(s: &str) -> &str { + let bytes = s.as_bytes(); + for (i, &item) in bytes.iter().enumerate() { + if !is_ascii_custom(item) { + return &s[0..i]; + } + } + s +} + +fn get_nonascii(s: &str) -> &str { + let bytes = s.as_bytes(); + for (i, &item) in bytes.iter().enumerate() { + if is_ascii_custom(item) { + return &s[0..i]; + } + } + s +} + +fn remove_ascii(s: &str) -> &str { + let bytes = s.as_bytes(); + for (i, &item) in bytes.iter().enumerate() { + if !is_ascii_custom(item) { + return &s[i..]; + } + } + "" +} + +fn remove_nonascii(s: &str) -> &str { + let bytes = s.as_bytes(); + for (i, &item) in bytes.iter().enumerate() { + if is_ascii_custom(item) { + return &s[i..]; + } + } + "" +} + +fn encode_modified_utf7(text: &str) -> String { + let capacity = 2 * text.len(); + let mut input = Vec::with_capacity(capacity); + let text_u16 = text.encode_utf16(); + for value in text_u16 { + input.extend_from_slice(&value.to_be_bytes()); + } + let text_u16 = base64::encode(input); + let text_u16 = text_u16.trim_end_matches('='); + let result = text_u16.replace('/', ","); + format!("&{}-", result) +} + +/// Decode UTF-7 IMAP mailbox name +/// +/// +pub fn decode_utf7_imap(text: &str) -> String { + let pattern = Regex::new(r"&([^-]*)-").unwrap(); + pattern.replace_all(&text, expand).to_string() +} + +fn expand(cap: &Captures) -> String { + if cap.get(1).unwrap().as_str() == "" { + "&".to_string() + } else { + decode_utf7_part(cap.get(0).unwrap().as_str()) + } +} + +fn decode_utf7_part(text: &str) -> String { + if text == "&-" { + return String::from("&"); + } + + let text_mb64 = &text[1..text.len() - 1]; + let mut text_b64 = text_mb64.replace(',', "/"); + + while (text_b64.len() % 4) != 0 { + text_b64 += "="; + } + + let text_u16 = base64::decode(text_b64).unwrap(); + let (cow, _encoding_used, _had_errors) = UTF_16BE.decode(&text_u16); + let result = cow.as_ref(); + + String::from(result) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn encode_test() { + assert_eq!( + encode_utf7_imap("Отправленные"), + "&BB4EQgQ,BEAEMAQyBDsENQQ9BD0ESwQ1-" + ); + } + #[test] + fn encode_test_split() { + assert_eq!( + encode_utf7_imap("Šiukšliadėžė"), + "&AWA-iuk&AWE-liad&ARcBfgEX-" + ) + } + + #[test] + fn encode_consecutive_accents() { + assert_eq!(encode_utf7_imap("théâtre"), "th&AOkA4g-tre") + } + + #[test] + fn decode_test() { + assert_eq!( + decode_utf7_imap("&BB4EQgQ,BEAEMAQyBDsENQQ9BD0ESwQ1-"), + "Отправленные" + ); + } + #[test] + fn decode_test_split() { + // input string with utf7 encoded bits being separated by ascii + assert_eq!( + decode_utf7_imap("&AWA-iuk&AWE-liad&ARcBfgEX-"), + "Šiukšliadėžė" + ) + } + + #[test] + fn decode_consecutive_accents() { + assert_eq!(decode_utf7_imap("th&AOkA4g-tre"), "théâtre") + } + + use proptest::prelude::*; + proptest! { + #![proptest_config(ProptestConfig::with_cases(10000))] + #[test] + fn fuzzy_dec_enc_check(s in "\\PC*") { + assert_eq!(decode_utf7_imap(encode_utf7_imap(s.clone())),s) + } + } +} diff --git a/melib/src/conf.rs b/melib/src/conf.rs index af5c178e..9f7c1f43 100644 --- a/melib/src/conf.rs +++ b/melib/src/conf.rs @@ -127,6 +127,8 @@ pub struct MailboxConf { pub usage: Option, #[serde(default = "none")] pub sort_order: Option, + #[serde(default = "none")] + pub encoding: Option, #[serde(flatten)] pub extra: HashMap, } @@ -140,6 +142,7 @@ impl Default for MailboxConf { ignore: ToggleFlag::Unset, usage: None, sort_order: None, + encoding: None, extra: HashMap::default(), } } @@ -166,15 +169,6 @@ pub fn none() -> Option { macro_rules! named_unit_variant { ($variant:ident) => { pub mod $variant { - /* - pub fn serialize(serializer: S) -> Result - where - S: serde::Serializer, - { - serializer.serialize_str(stringify!($variant)) - } - */ - pub fn deserialize<'de, D>(deserializer: D) -> Result<(), D::Error> where D: serde::Deserializer<'de>, diff --git a/src/conf/accounts.rs b/src/conf/accounts.rs index 5efa4231..305f5700 100644 --- a/src/conf/accounts.rs +++ b/src/conf/accounts.rs @@ -101,11 +101,45 @@ impl MailboxStatus { pub struct MailboxEntry { pub status: MailboxStatus, pub name: String, + pub path: String, pub ref_mailbox: Mailbox, pub conf: FileMailboxConf, } impl MailboxEntry { + pub fn new( + status: MailboxStatus, + name: String, + ref_mailbox: Mailbox, + conf: FileMailboxConf, + ) -> Self { + let mut ret = Self { + status, + name, + path: ref_mailbox.path().into(), + ref_mailbox, + conf, + }; + match ret.conf.mailbox_conf.extra.get("encoding") { + None => {} + Some(v) if ["utf-8", "utf8"].iter().any(|e| v.eq_ignore_ascii_case(e)) => {} + Some(v) if ["utf-7", "utf7"].iter().any(|e| v.eq_ignore_ascii_case(e)) => { + ret.name = melib::backends::utf7::decode_utf7_imap(&ret.name); + ret.path = melib::backends::utf7::decode_utf7_imap(&ret.path); + } + Some(other) => { + melib::log( + format!( + "mailbox `{}`: unrecognized mailbox name charset: {}", + &ret.name, other + ), + melib::WARN, + ); + } + } + ret + } + pub fn status(&self) -> String { match self.status { MailboxStatus::Available => format!( @@ -564,12 +598,12 @@ impl Account { } mailbox_entries.insert( f.hash(), - MailboxEntry { - ref_mailbox: f.clone(), - name: f.path().to_string(), - status: MailboxStatus::None, - conf: conf.clone(), - }, + MailboxEntry::new( + MailboxStatus::None, + f.path().to_string(), + f.clone(), + conf.clone(), + ), ); } else { let mut new = FileMailboxConf::default(); @@ -588,12 +622,7 @@ impl Account { mailbox_entries.insert( f.hash(), - MailboxEntry { - ref_mailbox: f.clone(), - name: f.path().to_string(), - status: MailboxStatus::None, - conf: new, - }, + MailboxEntry::new(MailboxStatus::None, f.path().to_string(), f.clone(), new), ); } } @@ -1951,12 +1980,12 @@ impl Account { self.mailbox_entries.insert( mailbox_hash, - MailboxEntry { - name: mailboxes[&mailbox_hash].path().to_string(), + MailboxEntry::new( status, - conf: new, - ref_mailbox: mailboxes.remove(&mailbox_hash).unwrap(), - }, + mailboxes[&mailbox_hash].path().to_string(), + mailboxes.remove(&mailbox_hash).unwrap(), + new, + ), ); self.collection .threads @@ -2370,3 +2399,78 @@ fn build_mailboxes_order( rec(node, mailbox_entries, 0, false); } } + +#[test] +fn test_mailbox_utf7() { + #[derive(Debug)] + struct TestMailbox(String); + + impl melib::BackendMailbox for TestMailbox { + fn hash(&self) -> MailboxHash { + unimplemented!() + } + + fn name(&self) -> &str { + &self.0 + } + + fn path(&self) -> &str { + &self.0 + } + + fn children(&self) -> &[MailboxHash] { + unimplemented!() + } + + fn clone(&self) -> Mailbox { + unimplemented!() + } + + fn special_usage(&self) -> SpecialUsageMailbox { + unimplemented!() + } + + fn parent(&self) -> Option { + unimplemented!() + } + + fn permissions(&self) -> MailboxPermissions { + unimplemented!() + } + + fn is_subscribed(&self) -> bool { + unimplemented!() + } + + fn set_is_subscribed(&mut self, _: bool) -> Result<()> { + unimplemented!() + } + + fn set_special_usage(&mut self, _: SpecialUsageMailbox) -> Result<()> { + unimplemented!() + } + + fn count(&self) -> Result<(usize, usize)> { + unimplemented!() + } + } + for (n, d) in [ + ("~peter/mail/&U,BTFw-/&ZeVnLIqe-", "~peter/mail/台北/日本語"), + ("&BB4EQgQ,BEAEMAQyBDsENQQ9BD0ESwQ1-", "Отправленные"), + ] { + let ref_mbox = TestMailbox(n.to_string()); + let mut conf: melib::MailboxConf = Default::default(); + conf.extra.insert("encoding".to_string(), "utf7".into()); + + let entry = MailboxEntry::new( + MailboxStatus::None, + n.to_string(), + Box::new(ref_mbox), + FileMailboxConf { + mailbox_conf: conf, + ..Default::default() + }, + ); + assert_eq!(&entry.path, d); + } +}