michael/panorama
1
0
Fork 0
Code Issues 11 Pull requests Projects 1 Releases Wiki Activity

don't fetch mail that's already been fetched

Browse source
This commit is contained in:
Michael Zhang 2021-03-25 14:51:52 -05:00
parent 7402b83f14
commit dfe6ebb596
Signed by: michael
GPG key ID: BDA47A31A3C8EE6B
9 changed files with 362 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

269
Cargo.lock generated
View file

@ -203,6 +203,15 @@ dependencies = [
"typenum",
]
[[package]]
name = "bitpacking"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3744aff20a3437a99ebc0bb7733e9e60c7bf590478c9b897e95b38d57e5acb68"
dependencies = [
"crunchy",
]
[[package]]
name = "bitvec"
version = "0.19.5"
@ -376,6 +385,12 @@ dependencies = [
"jobserver",
]
[[package]]
name = "census"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5927edd8345aef08578bcbb4aea7314f340d80c7f4931f99fbeb40b99d8f5060"
[[package]]
name = "cfg-if"
version = "0.1.10"
@ -471,6 +486,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "combine"
version = "4.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e"
dependencies = [
"memchr",
]
[[package]]
name = "commoncrypto"
version = "0.2.0"
@ -559,6 +583,20 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd01a6eb3daaafa260f6fc94c3a6c36390abc2080e38e3e34ced87393fb77d80"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils 0.8.3",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
@ -569,6 +607,30 @@ dependencies = [
"crossbeam-utils 0.8.3",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils 0.8.3",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.3",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.1"
@ -889,6 +951,17 @@ dependencies = [
"termcolor",
]
[[package]]
name = "fail"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3be3c61c59fdc91f5dbc3ea31ee8623122ce80057058be560654c5d410d181a6"
dependencies = [
"lazy_static",
"log",
"rand 0.7.3",
]
[[package]]
name = "fastrand"
version = "1.4.0"
@ -998,6 +1071,16 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e04cda45add94e71c2990de778ae13059897d77b773130a9bc225e2970c413e"
[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "funty"
version = "1.1.0"
@ -1050,6 +1133,7 @@ dependencies = [
"futures-core",
"futures-task",
"futures-util",
"num_cpus",
]
[[package]]
@ -1450,6 +1534,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "htmlescape"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
[[package]]
name = "humantime"
version = "2.1.0"
@ -1604,7 +1694,7 @@ dependencies = [
"petgraph",
"pico-args",
"regex",
"regex-syntax",
"regex-syntax 0.6.23",
"string_cache",
"term",
"tiny-keccak",
@ -1632,6 +1722,12 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "levenshtein_automata"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f44db4199cdb049b494a92d105acbfa43c25b3925e33803923ba9580b7bc9e1a"
[[package]]
name = "lexical-core"
version = "0.7.5"
@ -1730,6 +1826,15 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "lru"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f374d42cdfc1d7dbf3d3dec28afab2eb97ffbf43a3234d795b5986dbf4b90ba"
dependencies = [
"hashbrown 0.9.1",
]
[[package]]
name = "mac-notification-sys"
version = "0.3.0"
@ -1780,6 +1885,25 @@ version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi",
]
[[package]]
name = "memoffset"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.4.4"
@ -1813,6 +1937,15 @@ dependencies = [
"winapi",
]
[[package]]
name = "murmurhash32"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d736ff882f0e85fe9689fb23db229616c4c00aee2b3ac282f666d8f20eb25d4a"
dependencies = [
"byteorder",
]
[[package]]
name = "nb-connect"
version = "1.0.3"
@ -2034,6 +2167,7 @@ dependencies = [
"shellexpand",
"sqlx",
"structopt",
"tantivy",
"tokio 1.3.0",
"tokio-rustls",
"tokio-stream",
@ -2418,6 +2552,31 @@ dependencies = [
"rand_core 0.5.1",
]
[[package]]
name = "rayon"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils 0.8.3",
"lazy_static",
"num_cpus",
]
[[package]]
name = "redox_syscall"
version = "0.1.57"
@ -2462,9 +2621,15 @@ checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
"regex-syntax 0.6.23",
]
[[package]]
name = "regex-syntax"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e931c58b93d86f080c734bfd2bce7dd0079ae2331235818133c8be7f422e20e"
[[package]]
name = "regex-syntax"
version = "0.6.23"
@ -2516,6 +2681,16 @@ dependencies = [
"crossbeam-utils 0.8.3",
]
[[package]]
name = "rust-stemmers"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54"
dependencies = [
"serde",
"serde_derive",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -2756,6 +2931,12 @@ version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e"
[[package]]
name = "snap"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc725476a1398f0480d56cd0ad381f6f32acf2642704456f8f59a35df464b59a"
[[package]]
name = "socket2"
version = "0.3.19"
@ -2869,6 +3050,12 @@ dependencies = [
"tokio-rustls",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "static_assertions"
version = "0.3.4"
@ -3001,6 +3188,68 @@ dependencies = [
"unicode-xid 0.0.4",
]
[[package]]
name = "tantivy"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca90bddda472f39fdc74a031d61d52b08b1de97f2a704afae726a8004abb0d"
dependencies = [
"base64 0.13.0",
"bitpacking",
"byteorder",
"census",
"chrono",
"crc32fast",
"crossbeam",
"downcast-rs",
"fail",
"fnv",
"fs2",
"futures 0.3.13",
"htmlescape",
"levenshtein_automata",
"log",
"lru",
"memmap",
"murmurhash32",
"num_cpus",
"once_cell",
"rayon",
"regex",
"rust-stemmers",
"serde",
"serde_json",
"smallvec",
"snap",
"stable_deref_trait",
"tantivy-fst",
"tantivy-query-grammar",
"tempfile",
"thiserror",
"uuid",
"winapi",
]
[[package]]
name = "tantivy-fst"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb20cdc0d83e9184560bdde9cd60142dbb4af2e0f770e88fce45770495224205"
dependencies = [
"byteorder",
"regex-syntax 0.4.2",
"utf8-ranges",
]
[[package]]
name = "tantivy-query-grammar"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70864085b31ecd5af8f53a76506440ece1c426d187f3d72f4b722e238d2ce19a"
dependencies = [
"combine",
]
[[package]]
name = "tap"
version = "1.0.1"
@ -3313,12 +3562,28 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "utf8-ranges"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ae116fef2b7fea257ed6440d3cfcff7f190865f170cdad00bb6465bf18ecba"
[[package]]
name = "utf8parse"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8772a4ccbb4e89959023bc5b7cb8623a795caa7092d99f3aa9501b9484d4557d"
[[package]]
name = "uuid"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
dependencies = [
"getrandom 0.2.2",
"serde",
]
[[package]]
name = "vcpkg"
version = "0.2.11"

1
Cargo.toml
View file

@ -46,6 +46,7 @@ sha2 = "0.9.3"
hex = "0.4.3"
shellexpand = "2.1.0"
mailparse = "0.13.2"
tantivy = "0.14.0"
[dependencies.panorama-imap]
path = "imap"

10
imap/src/parser/mod.rs
View file

@ -35,8 +35,14 @@ pub fn parse_capability(s: impl AsRef<str>) -> ParseResult<Capability> {
}
pub fn parse_streamed_response(s: impl AsRef<str>) -> ParseResult<(Response, usize)> {
// trace!("parsing streamed reponse: {:?}", s.as_ref());
let mut pairs = Rfc3501::parse(Rule::streamed_response, s.as_ref())?;
let s = s.as_ref();
let mut pairs = match Rfc3501::parse(Rule::streamed_response, s) {
Ok(v) => v,
Err(e) => {
// error!("stream failed: {}", e);
return Err(e);
}
};
let pair = unwrap1(pairs.next().unwrap());
let span = pair.as_span();
let range = span.end() - span.start();

2
imap/src/parser/rfc3501.pest
View file

@ -44,7 +44,7 @@ date_month = { "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | "Jul" | "Aug" | "
date_time = { dquote_ ~ date_day_fixed ~ "-" ~ date_month ~ "-" ~ date_year ~ sp ~ time ~ sp ~ zone ~ dquote_ }
date_year = @{ digit{4} }
digit_nz = @{ '\x31'..'\x39' }
env_address1 = { "(" ~ address{1,} ~ ")" }
env_address1 = { "(" ~ address ~ (sp? ~ address)? ~ ")" }
env_bcc = { env_address1 | nil }
env_cc = { env_address1 | nil }
env_date = { nstring }

1
src/lib.rs
View file

@ -22,6 +22,7 @@ extern crate log;
pub mod config;
pub mod mail;
pub mod script;
pub mod search;
pub mod ui;
/// Consumes any error and dumps it to the logger.

15
src/mail/client.rs
View file

@ -1,7 +1,7 @@
use anyhow::{Context, Result};
use futures::{
future::FutureExt,
stream::{Stream, StreamExt, TryStreamExt},
future::{FutureExt, TryFutureExt},
stream::{self, Stream, StreamExt, TryStreamExt},
};
use notify_rust::{Notification, Timeout};
use panorama_imap::{
@ -76,9 +76,16 @@ pub async fn sync_main(
if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) {
if exists < 10 {
let uids = (1..=exists).collect::<Vec<_>>();
let new_uids = stream::iter(1..exists).map(Ok).try_filter_map(|uid| {
mail_store.try_identify_email(&acct_name, &folder, uid, uidvalidity, None)
// invert the option to only select uids that haven't been downloaded
.map_ok(move |o| o.map_or_else(move || Some(uid), |v| None))
.map_err(|err| err.context("error checking if the email is already downloaded [try_identify_email]"))
}).try_collect::<Vec<_>>().await?;
debug!("fetching uids {:?}", new_uids);
let fetched = authed
.uid_fetch(&uids, FetchItems::PanoramaAll)
.uid_fetch(&new_uids, FetchItems::PanoramaAll)
.await
.context("error fetching uids")?;

78
src/mail/store.rs
View file

@ -9,7 +9,7 @@ use sha2::{Digest, Sha256};
use sqlx::{
migrate::{MigrateDatabase, Migrator},
sqlite::{Sqlite, SqlitePool},
Error,
Error as SqlxError, Row,
};
use tokio::{fs, sync::broadcast};
@ -78,9 +78,43 @@ impl MailStore {
// self.email_events.subscribe()
// }
/// Try to identify an email based on the UID, message-id, and other heuristics
pub async fn try_identify_email() {
/// Given a UID and optional message-id try to identify a particular message
pub async fn try_identify_email(
&self,
acct: impl AsRef<str>,
folder: impl AsRef<str>,
uid: u32,
uidvalidity: u32,
message_id: Option<&str>,
) -> Result<Option<u32>> {
let existing: Option<(u32,)> = into_opt(
sqlx::query_as(
r#"
SELECT rowid FROM "mail"
WHERE account = ? AND folder = ?
AND uid = ? AND uidvalidity = ?
"#,
)
.bind(acct.as_ref())
.bind(folder.as_ref())
.bind(uid)
.bind(uidvalidity)
.fetch_one(&self.pool)
.await,
)?;
if let Some(existing) = existing {
let rowid = existing.0;
debug!(
"folder: {:?} uid: {:?} rowid: {:?}",
folder.as_ref(),
uid,
rowid,
);
return Ok(Some(rowid));
}
Ok(None)
}
/// Stores the given email
@ -128,27 +162,23 @@ impl MailStore {
debug!("message-id: {:?}", message_id);
let existing = sqlx::query(
r#"
SELECT FROM "mail"
let existing = into_opt(
sqlx::query(
r#"
SELECT * FROM "mail"
WHERE account = ? AND folder = ?
AND uid = ? AND uidvalidity = ?
"#,
)
.bind(acct.as_ref())
.bind(folder.as_ref())
.bind(uid)
.bind(uidvalidity)
.fetch_one(&self.pool)
.await;
)
.bind(acct.as_ref())
.bind(folder.as_ref())
.bind(uid)
.bind(uidvalidity)
.fetch_one(&self.pool)
.await,
)?;
let exists = match existing {
Ok(_) => true,
Err(Error::RowNotFound) => true,
_ => false,
};
if !exists {
if existing.is_none() {
let id = sqlx::query(
r#"
INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename)
@ -174,3 +204,11 @@ impl MailStore {
Ok(())
}
}
fn into_opt<T>(res: Result<T, SqlxError>) -> Result<Option<T>> {
match res {
Ok(v) => Ok(Some(v)),
Err(SqlxError::RowNotFound) => Ok(None),
Err(e) => Err(e.into()),
}
}

3
src/main.rs
View file

@ -116,7 +116,6 @@ fn run_ui(
}
fn setup_logger(log_file: Option<impl AsRef<Path>>) -> Result<()> {
let now = chrono::Local::now().format("[%Y-%m-%d][%H:%M:%S]");
let colors = ColoredLevelConfig::new()
.info(Color::Blue)
.debug(Color::BrightBlack)
@ -137,7 +136,7 @@ fn setup_logger(log_file: Option<impl AsRef<Path>>) -> Result<()> {
message
))
})
.level(log::LevelFilter::Debug);
.level(log::LevelFilter::Trace);
if let Some(log_file) = log_file {
logger = logger.chain(fern::log_file(log_file)?);
}

14
src/search.rs Normal file
View file

@ -0,0 +1,14 @@
//! Searching
use crate::config::Config;
/// A search index manager
///
/// This is clone-safe: cloning this struct will return references to the same object
#[derive(Clone)]
pub struct SearchIndex {}
impl SearchIndex {
/// Create a new instance of the search index
pub fn new(config: Config) {}
}