don't fetch mail that's already been fetched

Cargo.lock

@@ -203,6 +203,15 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "bitpacking"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3744aff20a3437a99ebc0bb7733e9e60c7bf590478c9b897e95b38d57e5acb68"
+dependencies = [
+ "crunchy",
+]
+
 [[package]]
 name = "bitvec"
 version = "0.19.5"
@@ -376,6 +385,12 @@ dependencies = [
  "jobserver",
 ]
 
+[[package]]
+name = "census"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5927edd8345aef08578bcbb4aea7314f340d80c7f4931f99fbeb40b99d8f5060"
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -471,6 +486,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "combine"
+version = "4.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "commoncrypto"
 version = "0.2.0"
@@ -559,6 +583,20 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "crossbeam"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd01a6eb3daaafa260f6fc94c3a6c36390abc2080e38e3e34ced87393fb77d80"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-epoch",
+ "crossbeam-queue",
+ "crossbeam-utils 0.8.3",
+]
+
 [[package]]
 name = "crossbeam-channel"
 version = "0.5.0"
@@ -569,6 +607,30 @@ dependencies = [
  "crossbeam-utils 0.8.3",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-epoch",
+ "crossbeam-utils 0.8.3",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils 0.8.3",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
 [[package]]
 name = "crossbeam-queue"
 version = "0.3.1"
@@ -889,6 +951,17 @@ dependencies = [
  "termcolor",
 ]
 
+[[package]]
+name = "fail"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3be3c61c59fdc91f5dbc3ea31ee8623122ce80057058be560654c5d410d181a6"
+dependencies = [
+ "lazy_static",
+ "log",
+ "rand 0.7.3",
+]
+
 [[package]]
 name = "fastrand"
 version = "1.4.0"
@@ -998,6 +1071,16 @@ version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0e04cda45add94e71c2990de778ae13059897d77b773130a9bc225e2970c413e"
 
+[[package]]
+name = "fs2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
 [[package]]
 name = "funty"
 version = "1.1.0"
@@ -1050,6 +1133,7 @@ dependencies = [
  "futures-core",
  "futures-task",
  "futures-util",
+ "num_cpus",
 ]
 
 [[package]]
@@ -1450,6 +1534,12 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "htmlescape"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
+
 [[package]]
 name = "humantime"
 version = "2.1.0"
@@ -1604,7 +1694,7 @@ dependencies = [
  "petgraph",
  "pico-args",
  "regex",
- "regex-syntax",
+ "regex-syntax 0.6.23",
  "string_cache",
  "term",
  "tiny-keccak",
@@ -1632,6 +1722,12 @@ version = "1.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
+[[package]]
+name = "levenshtein_automata"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a"
+
 [[package]]
 name = "lexical-core"
 version = "0.7.5"
@@ -1730,6 +1826,15 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
+[[package]]
+name = "lru"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f374d42cdfc1d7dbf3d3dec28afab2eb97ffbf43a3234d795b5986dbf4b90ba"
+dependencies = [
+ "hashbrown 0.9.1",
+]
+
 [[package]]
 name = "mac-notification-sys"
 version = "0.3.0"
@@ -1780,6 +1885,25 @@ version = "2.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
 
+[[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "memoffset"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "miniz_oxide"
 version = "0.4.4"
@@ -1813,6 +1937,15 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "murmurhash32"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a"
+dependencies = [
+ "byteorder",
+]
+
 [[package]]
 name = "nb-connect"
 version = "1.0.3"
@@ -2034,6 +2167,7 @@ dependencies = [
  "shellexpand",
  "sqlx",
  "structopt",
+ "tantivy",
  "tokio 1.3.0",
  "tokio-rustls",
  "tokio-stream",
@@ -2418,6 +2552,31 @@ dependencies = [
  "rand_core 0.5.1",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils 0.8.3",
+ "lazy_static",
+ "num_cpus",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.57"
@@ -2462,9 +2621,15 @@ checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax",
+ "regex-syntax 0.6.23",
 ]
 
+[[package]]
+name = "regex-syntax"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
+
 [[package]]
 name = "regex-syntax"
 version = "0.6.23"
@@ -2516,6 +2681,16 @@ dependencies = [
  "crossbeam-utils 0.8.3",
 ]
 
+[[package]]
+name = "rust-stemmers"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
+dependencies = [
+ "serde",
+ "serde_derive",
+]
+
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -2756,6 +2931,12 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
 
+[[package]]
+name = "snap"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a"
+
 [[package]]
 name = "socket2"
 version = "0.3.19"
@@ -2869,6 +3050,12 @@ dependencies = [
  "tokio-rustls",
 ]
 
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
 [[package]]
 name = "static_assertions"
 version = "0.3.4"
@@ -3001,6 +3188,68 @@ dependencies = [
  "unicode-xid 0.0.4",
 ]
 
+[[package]]
+name = "tantivy"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "edca90bddda472f39fdc74a031d61d52b08b1de97f2a704afae726a8004abb0d"
+dependencies = [
+ "base64 0.13.0",
+ "bitpacking",
+ "byteorder",
+ "census",
+ "chrono",
+ "crc32fast",
+ "crossbeam",
+ "downcast-rs",
+ "fail",
+ "fnv",
+ "fs2",
+ "futures 0.3.13",
+ "htmlescape",
+ "levenshtein_automata",
+ "log",
+ "lru",
+ "memmap",
+ "murmurhash32",
+ "num_cpus",
+ "once_cell",
+ "rayon",
+ "regex",
+ "rust-stemmers",
+ "serde",
+ "serde_json",
+ "smallvec",
+ "snap",
+ "stable_deref_trait",
+ "tantivy-fst",
+ "tantivy-query-grammar",
+ "tempfile",
+ "thiserror",
+ "uuid",
+ "winapi",
+]
+
+[[package]]
+name = "tantivy-fst"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb20cdc0d83e9184560bdde9cd60142dbb4af2e0f770e88fce45770495224205"
+dependencies = [
+ "byteorder",
+ "regex-syntax 0.4.2",
+ "utf8-ranges",
+]
+
+[[package]]
+name = "tantivy-query-grammar"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70864085b31ecd5af8f53a76506440ece1c426d187f3d72f4b722e238d2ce19a"
+dependencies = [
+ "combine",
+]
+
 [[package]]
 name = "tap"
 version = "1.0.1"
@@ -3313,12 +3562,28 @@ dependencies = [
  "percent-encoding",
 ]
 
+[[package]]
+name = "utf8-ranges"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba"
+
 [[package]]
 name = "utf8parse"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d"
 
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+dependencies = [
+ "getrandom 0.2.2",
+ "serde",
+]
+
 [[package]]
 name = "vcpkg"
 version = "0.2.11"

Cargo.toml

@@ -46,6 +46,7 @@ sha2 = "0.9.3"
 hex = "0.4.3"
 shellexpand = "2.1.0"
 mailparse = "0.13.2"
+tantivy = "0.14.0"
 
 [dependencies.panorama-imap]
 path = "imap"

imap/src/parser/mod.rs

@@ -35,8 +35,14 @@ pub fn parse_capability(s: impl AsRef<str>) -> ParseResult<Capability> {
 }
 
 pub fn parse_streamed_response(s: impl AsRef<str>) -> ParseResult<(Response, usize)> {
-    // trace!("parsing streamed reponse: {:?}", s.as_ref());
+    let s = s.as_ref();
-    let mut pairs = Rfc3501::parse(Rule::streamed_response, s.as_ref())?;
+    let mut pairs = match Rfc3501::parse(Rule::streamed_response, s) {
+        Ok(v) => v,
+        Err(e) => {
+            // error!("stream failed: {}", e);
+            return Err(e);
+        }
+    };
     let pair = unwrap1(pairs.next().unwrap());
     let span = pair.as_span();
     let range = span.end() - span.start();

imap/src/parser/rfc3501.pest

@@ -44,7 +44,7 @@ date_month = { "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | "Jul" | "Aug" | "
 date_time = { dquote_ ~ date_day_fixed ~ "-" ~ date_month ~ "-" ~ date_year ~ sp ~ time ~ sp ~ zone ~ dquote_ }
 date_year = @{ digit{4} }
 digit_nz = @{ '\x31'..'\x39' }
-env_address1 = { "(" ~ address{1,} ~ ")" }
+env_address1 = { "(" ~ address ~ (sp? ~ address)? ~ ")" }
 env_bcc = { env_address1 | nil }
 env_cc = { env_address1 | nil }
 env_date = { nstring }

src/lib.rs

@@ -22,6 +22,7 @@ extern crate log;
 pub mod config;
 pub mod mail;
 pub mod script;
+pub mod search;
 pub mod ui;
 
 /// Consumes any error and dumps it to the logger.

src/mail/client.rs

@@ -1,7 +1,7 @@
 use anyhow::{Context, Result};
 use futures::{
-    future::FutureExt,
+    future::{FutureExt, TryFutureExt},
-    stream::{Stream, StreamExt, TryStreamExt},
+    stream::{self, Stream, StreamExt, TryStreamExt},
 };
 use notify_rust::{Notification, Timeout};
 use panorama_imap::{
@@ -76,9 +76,16 @@ pub async fn sync_main(
 
             if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) {
                 if exists < 10 {
-                    let uids = (1..=exists).collect::<Vec<_>>();
+                    let new_uids = stream::iter(1..exists).map(Ok).try_filter_map(|uid| {
+                        mail_store.try_identify_email(&acct_name, &folder, uid, uidvalidity, None)
+                            // invert the option to only select uids that haven't been downloaded
+                            .map_ok(move |o| o.map_or_else(move || Some(uid), |v| None))
+                            .map_err(|err| err.context("error checking if the email is already downloaded [try_identify_email]"))
+                    }).try_collect::<Vec<_>>().await?;
+
+                    debug!("fetching uids {:?}", new_uids);
                     let fetched = authed
-                        .uid_fetch(&uids, FetchItems::PanoramaAll)
+                        .uid_fetch(&new_uids, FetchItems::PanoramaAll)
                         .await
                         .context("error fetching uids")?;
 

src/mail/store.rs

@@ -9,7 +9,7 @@ use sha2::{Digest, Sha256};
 use sqlx::{
     migrate::{MigrateDatabase, Migrator},
     sqlite::{Sqlite, SqlitePool},
-    Error,
+    Error as SqlxError, Row,
 };
 use tokio::{fs, sync::broadcast};
 
@@ -78,9 +78,43 @@ impl MailStore {
     //     self.email_events.subscribe()
     // }
 
-    /// Try to identify an email based on the UID, message-id, and other heuristics
+    /// Given a UID and optional message-id try to identify a particular message
-    pub async fn try_identify_email() {
+    pub async fn try_identify_email(
+        &self,
+        acct: impl AsRef<str>,
+        folder: impl AsRef<str>,
+        uid: u32,
+        uidvalidity: u32,
+        message_id: Option<&str>,
+    ) -> Result<Option<u32>> {
+        let existing: Option<(u32,)> = into_opt(
+            sqlx::query_as(
+                r#"
+            SELECT rowid FROM "mail"
+            WHERE account = ? AND folder = ?
+                AND uid = ? AND uidvalidity = ?
+            "#,
+            )
+            .bind(acct.as_ref())
+            .bind(folder.as_ref())
+            .bind(uid)
+            .bind(uidvalidity)
+            .fetch_one(&self.pool)
+            .await,
+        )?;
 
+        if let Some(existing) = existing {
+            let rowid = existing.0;
+            debug!(
+                "folder: {:?} uid: {:?} rowid: {:?}",
+                folder.as_ref(),
+                uid,
+                rowid,
+            );
+            return Ok(Some(rowid));
+        }
+
+        Ok(None)
     }
 
     /// Stores the given email
@@ -128,9 +162,10 @@ impl MailStore {
 
         debug!("message-id: {:?}", message_id);
 
-        let existing = sqlx::query(
+        let existing = into_opt(
+            sqlx::query(
                 r#"
-            SELECT FROM "mail"
+            SELECT * FROM "mail"
             WHERE account = ? AND folder = ?
                 AND uid = ? AND uidvalidity = ?
             "#,
@@ -140,15 +175,10 @@ impl MailStore {
             .bind(uid)
             .bind(uidvalidity)
             .fetch_one(&self.pool)
-        .await;
+            .await,
+        )?;
 
-        let exists = match existing {
+        if existing.is_none() {
-            Ok(_) => true,
-            Err(Error::RowNotFound) => true,
-            _ => false,
-        };
-
-        if !exists {
             let id = sqlx::query(
                 r#"
                 INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename)
@@ -174,3 +204,11 @@ impl MailStore {
         Ok(())
     }
 }
+
+fn into_opt<T>(res: Result<T, SqlxError>) -> Result<Option<T>> {
+    match res {
+        Ok(v) => Ok(Some(v)),
+        Err(SqlxError::RowNotFound) => Ok(None),
+        Err(e) => Err(e.into()),
+    }
+}

src/main.rs

@@ -116,7 +116,6 @@ fn run_ui(
 }
 
 fn setup_logger(log_file: Option<impl AsRef<Path>>) -> Result<()> {
-    let now = chrono::Local::now().format("[%Y-%m-%d][%H:%M:%S]");
     let colors = ColoredLevelConfig::new()
         .info(Color::Blue)
         .debug(Color::BrightBlack)
@@ -137,7 +136,7 @@ fn setup_logger(log_file: Option<impl AsRef<Path>>) -> Result<()> {
                 message
             ))
         })
-        .level(log::LevelFilter::Debug);
+        .level(log::LevelFilter::Trace);
     if let Some(log_file) = log_file {
         logger = logger.chain(fern::log_file(log_file)?);
     }

src/search.rs

@@ -0,0 +1,14 @@
+//! Searching
+
+use crate::config::Config;
+
+/// A search index manager
+///
+/// This is clone-safe: cloning this struct will return references to the same object
+#[derive(Clone)]
+pub struct SearchIndex {}
+
+impl SearchIndex {
+    /// Create a new instance of the search index
+    pub fn new(config: Config) {}
+}