diff --git a/Cargo.lock b/Cargo.lock index 649b1dd..f44e117 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -203,6 +203,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "bitpacking" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3744aff20a3437a99ebc0bb7733e9e60c7bf590478c9b897e95b38d57e5acb68" +dependencies = [ + "crunchy", +] + [[package]] name = "bitvec" version = "0.19.5" @@ -376,6 +385,12 @@ dependencies = [ "jobserver", ] +[[package]] +name = "census" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5927edd8345aef08578bcbb4aea7314f340d80c7f4931f99fbeb40b99d8f5060" + [[package]] name = "cfg-if" version = "0.1.10" @@ -471,6 +486,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "combine" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e" +dependencies = [ + "memchr", +] + [[package]] name = "commoncrypto" version = "0.2.0" @@ -559,6 +583,20 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd01a6eb3daaafa260f6fc94c3a6c36390abc2080e38e3e34ced87393fb77d80" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils 0.8.3", +] + [[package]] name = "crossbeam-channel" version = "0.5.0" @@ -569,6 +607,30 @@ dependencies = [ "crossbeam-utils 0.8.3", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils 0.8.3", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.3", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.1" @@ -889,6 +951,17 @@ dependencies = [ "termcolor", ] +[[package]] +name = "fail" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be3c61c59fdc91f5dbc3ea31ee8623122ce80057058be560654c5d410d181a6" +dependencies = [ + "lazy_static", + "log", + "rand 0.7.3", +] + [[package]] name = "fastrand" version = "1.4.0" @@ -998,6 +1071,16 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e04cda45add94e71c2990de778ae13059897d77b773130a9bc225e2970c413e" +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "funty" version = "1.1.0" @@ -1050,6 +1133,7 @@ dependencies = [ "futures-core", "futures-task", "futures-util", + "num_cpus", ] [[package]] @@ -1450,6 +1534,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "humantime" version = "2.1.0" @@ -1604,7 +1694,7 @@ dependencies = [ "petgraph", "pico-args", "regex", - "regex-syntax", + "regex-syntax 0.6.23", "string_cache", "term", "tiny-keccak", @@ -1632,6 +1722,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "levenshtein_automata" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a" + [[package]] name = "lexical-core" version = "0.7.5" @@ -1730,6 +1826,15 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "lru" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f374d42cdfc1d7dbf3d3dec28afab2eb97ffbf43a3234d795b5986dbf4b90ba" +dependencies = [ + "hashbrown 0.9.1", +] + [[package]] name = "mac-notification-sys" version = "0.3.0" @@ -1780,6 +1885,25 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + [[package]] name = "miniz_oxide" version = "0.4.4" @@ -1813,6 +1937,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "murmurhash32" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a" +dependencies = [ + "byteorder", +] + [[package]] name = "nb-connect" version = "1.0.3" @@ -2034,6 +2167,7 @@ dependencies = [ "shellexpand", "sqlx", "structopt", + "tantivy", "tokio 1.3.0", "tokio-rustls", "tokio-stream", @@ -2418,6 +2552,31 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils 0.8.3", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.1.57" @@ -2462,9 +2621,15 @@ checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ "aho-corasick", "memchr", - "regex-syntax", + "regex-syntax 0.6.23", ] +[[package]] +name = "regex-syntax" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e" + [[package]] name = "regex-syntax" version = "0.6.23" @@ -2516,6 +2681,16 @@ dependencies = [ "crossbeam-utils 0.8.3", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -2756,6 +2931,12 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" +[[package]] +name = "snap" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a" + [[package]] name = "socket2" version = "0.3.19" @@ -2869,6 +3050,12 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "0.3.4" @@ -3001,6 +3188,68 @@ dependencies = [ "unicode-xid 0.0.4", ] +[[package]] +name = "tantivy" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca90bddda472f39fdc74a031d61d52b08b1de97f2a704afae726a8004abb0d" +dependencies = [ + "base64 0.13.0", + "bitpacking", + "byteorder", + "census", + "chrono", + "crc32fast", + "crossbeam", + "downcast-rs", + "fail", + "fnv", + "fs2", + "futures 0.3.13", + "htmlescape", + "levenshtein_automata", + "log", + "lru", + "memmap", + "murmurhash32", + "num_cpus", + "once_cell", + "rayon", + "regex", + "rust-stemmers", + "serde", + "serde_json", + "smallvec", + "snap", + "stable_deref_trait", + "tantivy-fst", + "tantivy-query-grammar", + "tempfile", + "thiserror", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-fst" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb20cdc0d83e9184560bdde9cd60142dbb4af2e0f770e88fce45770495224205" +dependencies = [ + "byteorder", + "regex-syntax 0.4.2", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70864085b31ecd5af8f53a76506440ece1c426d187f3d72f4b722e238d2ce19a" +dependencies = [ + "combine", +] + [[package]] name = "tap" version = "1.0.1" @@ -3313,12 +3562,28 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8-ranges" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba" + [[package]] name = "utf8parse" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d" +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.2", + "serde", +] + [[package]] name = "vcpkg" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 3808d6c..e1a8e2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,6 +46,7 @@ sha2 = "0.9.3" hex = "0.4.3" shellexpand = "2.1.0" mailparse = "0.13.2" +tantivy = "0.14.0" [dependencies.panorama-imap] path = "imap" diff --git a/imap/src/parser/mod.rs b/imap/src/parser/mod.rs index 35f8965..98b86e2 100644 --- a/imap/src/parser/mod.rs +++ b/imap/src/parser/mod.rs @@ -35,8 +35,14 @@ pub fn parse_capability(s: impl AsRef) -> ParseResult { } pub fn parse_streamed_response(s: impl AsRef) -> ParseResult<(Response, usize)> { - // trace!("parsing streamed reponse: {:?}", s.as_ref()); - let mut pairs = Rfc3501::parse(Rule::streamed_response, s.as_ref())?; + let s = s.as_ref(); + let mut pairs = match Rfc3501::parse(Rule::streamed_response, s) { + Ok(v) => v, + Err(e) => { + // error!("stream failed: {}", e); + return Err(e); + } + }; let pair = unwrap1(pairs.next().unwrap()); let span = pair.as_span(); let range = span.end() - span.start(); diff --git a/imap/src/parser/rfc3501.pest b/imap/src/parser/rfc3501.pest index 3763507..e708492 100644 --- a/imap/src/parser/rfc3501.pest +++ b/imap/src/parser/rfc3501.pest @@ -44,7 +44,7 @@ date_month = { "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | "Jul" | "Aug" | " date_time = { dquote_ ~ date_day_fixed ~ "-" ~ date_month ~ "-" ~ date_year ~ sp ~ time ~ sp ~ zone ~ dquote_ } date_year = @{ digit{4} } digit_nz = @{ '\x31'..'\x39' } -env_address1 = { "(" ~ address{1,} ~ ")" } +env_address1 = { "(" ~ address ~ (sp? ~ address)? ~ ")" } env_bcc = { env_address1 | nil } env_cc = { env_address1 | nil } env_date = { nstring } diff --git a/src/lib.rs b/src/lib.rs index dd275d3..4d97a5f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,6 +22,7 @@ extern crate log; pub mod config; pub mod mail; pub mod script; +pub mod search; pub mod ui; /// Consumes any error and dumps it to the logger. diff --git a/src/mail/client.rs b/src/mail/client.rs index ee1b638..eaf7352 100644 --- a/src/mail/client.rs +++ b/src/mail/client.rs @@ -1,7 +1,7 @@ use anyhow::{Context, Result}; use futures::{ - future::FutureExt, - stream::{Stream, StreamExt, TryStreamExt}, + future::{FutureExt, TryFutureExt}, + stream::{self, Stream, StreamExt, TryStreamExt}, }; use notify_rust::{Notification, Timeout}; use panorama_imap::{ @@ -76,9 +76,16 @@ pub async fn sync_main( if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) { if exists < 10 { - let uids = (1..=exists).collect::>(); + let new_uids = stream::iter(1..exists).map(Ok).try_filter_map(|uid| { + mail_store.try_identify_email(&acct_name, &folder, uid, uidvalidity, None) + // invert the option to only select uids that haven't been downloaded + .map_ok(move |o| o.map_or_else(move || Some(uid), |v| None)) + .map_err(|err| err.context("error checking if the email is already downloaded [try_identify_email]")) + }).try_collect::>().await?; + + debug!("fetching uids {:?}", new_uids); let fetched = authed - .uid_fetch(&uids, FetchItems::PanoramaAll) + .uid_fetch(&new_uids, FetchItems::PanoramaAll) .await .context("error fetching uids")?; diff --git a/src/mail/store.rs b/src/mail/store.rs index 73b7b70..e7c7187 100644 --- a/src/mail/store.rs +++ b/src/mail/store.rs @@ -9,7 +9,7 @@ use sha2::{Digest, Sha256}; use sqlx::{ migrate::{MigrateDatabase, Migrator}, sqlite::{Sqlite, SqlitePool}, - Error, + Error as SqlxError, Row, }; use tokio::{fs, sync::broadcast}; @@ -78,9 +78,43 @@ impl MailStore { // self.email_events.subscribe() // } - /// Try to identify an email based on the UID, message-id, and other heuristics - pub async fn try_identify_email() { + /// Given a UID and optional message-id try to identify a particular message + pub async fn try_identify_email( + &self, + acct: impl AsRef, + folder: impl AsRef, + uid: u32, + uidvalidity: u32, + message_id: Option<&str>, + ) -> Result> { + let existing: Option<(u32,)> = into_opt( + sqlx::query_as( + r#" + SELECT rowid FROM "mail" + WHERE account = ? AND folder = ? + AND uid = ? AND uidvalidity = ? + "#, + ) + .bind(acct.as_ref()) + .bind(folder.as_ref()) + .bind(uid) + .bind(uidvalidity) + .fetch_one(&self.pool) + .await, + )?; + if let Some(existing) = existing { + let rowid = existing.0; + debug!( + "folder: {:?} uid: {:?} rowid: {:?}", + folder.as_ref(), + uid, + rowid, + ); + return Ok(Some(rowid)); + } + + Ok(None) } /// Stores the given email @@ -128,27 +162,23 @@ impl MailStore { debug!("message-id: {:?}", message_id); - let existing = sqlx::query( - r#" - SELECT FROM "mail" + let existing = into_opt( + sqlx::query( + r#" + SELECT * FROM "mail" WHERE account = ? AND folder = ? AND uid = ? AND uidvalidity = ? "#, - ) - .bind(acct.as_ref()) - .bind(folder.as_ref()) - .bind(uid) - .bind(uidvalidity) - .fetch_one(&self.pool) - .await; + ) + .bind(acct.as_ref()) + .bind(folder.as_ref()) + .bind(uid) + .bind(uidvalidity) + .fetch_one(&self.pool) + .await, + )?; - let exists = match existing { - Ok(_) => true, - Err(Error::RowNotFound) => true, - _ => false, - }; - - if !exists { + if existing.is_none() { let id = sqlx::query( r#" INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename) @@ -174,3 +204,11 @@ impl MailStore { Ok(()) } } + +fn into_opt(res: Result) -> Result> { + match res { + Ok(v) => Ok(Some(v)), + Err(SqlxError::RowNotFound) => Ok(None), + Err(e) => Err(e.into()), + } +} diff --git a/src/main.rs b/src/main.rs index 58cc5b7..a1257b6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -116,7 +116,6 @@ fn run_ui( } fn setup_logger(log_file: Option>) -> Result<()> { - let now = chrono::Local::now().format("[%Y-%m-%d][%H:%M:%S]"); let colors = ColoredLevelConfig::new() .info(Color::Blue) .debug(Color::BrightBlack) @@ -137,7 +136,7 @@ fn setup_logger(log_file: Option>) -> Result<()> { message )) }) - .level(log::LevelFilter::Debug); + .level(log::LevelFilter::Trace); if let Some(log_file) = log_file { logger = logger.chain(fern::log_file(log_file)?); } diff --git a/src/search.rs b/src/search.rs new file mode 100644 index 0000000..dfc40d5 --- /dev/null +++ b/src/search.rs @@ -0,0 +1,14 @@ +//! Searching + +use crate::config::Config; + +/// A search index manager +/// +/// This is clone-safe: cloning this struct will return references to the same object +#[derive(Clone)] +pub struct SearchIndex {} + +impl SearchIndex { + /// Create a new instance of the search index + pub fn new(config: Config) {} +}