From 7402b83f145bbb4246bc9d1a6b3027736436a7cb Mon Sep 17 00:00:00 2001 From: Michael Zhang Date: Thu, 25 Mar 2021 13:27:57 -0500 Subject: [PATCH] config data dir --- Cargo.lock | 50 +++++++++++++++++++++++++-- Cargo.toml | 1 + migrations/1_initial.sql | 1 + src/config.rs | 7 ++-- src/mail/client.rs | 26 ++++++++------ src/mail/mod.rs | 5 ++- src/mail/store.rs | 75 +++++++++++++++++++++++++++++++--------- 7 files changed, 129 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2e54ef..649b1dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,6 +146,21 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" + [[package]] name = "base64" version = "0.13.0" @@ -373,6 +388,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "charset" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f426e64df1c3de26cbf44593c6ffff5dbfd43bbf9de0d075058558126b3fc73" +dependencies = [ + "base64 0.10.1", + "encoding_rs", +] + [[package]] name = "chrono" version = "0.4.19" @@ -821,6 +846,15 @@ dependencies = [ "log", ] +[[package]] +name = "encoding_rs" +version = "0.8.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "enumflags2" version = "0.6.4" @@ -1708,6 +1742,17 @@ dependencies = [ "objc-foundation", ] +[[package]] +name = "mailparse" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31de1f9043c582efde7dbd93de56600df12b6c4488a67eeaefa74ea364019b22" +dependencies = [ + "base64 0.12.3", + "charset", + "quoted_printable", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -1978,6 +2023,7 @@ dependencies = [ "hex 0.4.3", "inotify", "log", + "mailparse", "notify-rust", "panorama-imap", "panorama-smtp", @@ -2464,7 +2510,7 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" dependencies = [ - "base64", + "base64 0.13.0", "blake2b_simd", "constant_time_eq", "crossbeam-utils 0.8.3", @@ -2500,7 +2546,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" dependencies = [ - "base64", + "base64 0.13.0", "log", "ring", "sct", diff --git a/Cargo.toml b/Cargo.toml index 0cba6ef..3808d6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,6 +45,7 @@ sqlx = { version = "0.5.1", features = ["runtime-tokio-rustls", "sqlite"] } sha2 = "0.9.3" hex = "0.4.3" shellexpand = "2.1.0" +mailparse = "0.13.2" [dependencies.panorama-imap] path = "imap" diff --git a/migrations/1_initial.sql b/migrations/1_initial.sql index 13a0d2e..5dc0089 100644 --- a/migrations/1_initial.sql +++ b/migrations/1_initial.sql @@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS "accounts" ( CREATE TABLE IF NOT EXISTS "mail" ( "id" INTEGER PRIMARY KEY, + "message_id" TEXT, "account" TEXT, "folder" TEXT, "uidvalidity" INTEGER, diff --git a/src/config.rs b/src/config.rs index 1c13a53..5ce61f9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -25,11 +25,8 @@ pub struct Config { /// (potentially for migration later?) pub version: String, - /// Directory to store mail in - pub mail_dir: PathBuf, - - /// SQLite database path - pub db_path: PathBuf, + /// Directory to store panorama-related data in + pub data_dir: PathBuf, /// Mail accounts #[serde(rename = "mail")] diff --git a/src/mail/client.rs b/src/mail/client.rs index 2d3fc39..ee1b638 100644 --- a/src/mail/client.rs +++ b/src/mail/client.rs @@ -1,7 +1,7 @@ -use anyhow::Result; +use anyhow::{Context, Result}; use futures::{ future::FutureExt, - stream::{Stream, StreamExt}, + stream::{Stream, StreamExt, TryStreamExt}, }; use notify_rust::{Notification, Timeout}; use panorama_imap::{ @@ -76,15 +76,19 @@ pub async fn sync_main( if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) { if exists < 10 { - let mut fetched = authed - .uid_fetch(&(1..=exists).collect::>(), FetchItems::PanoramaAll) - .await?; - while let Some((uid, attrs)) = fetched.next().await { - debug!("- {} : {:?}", uid, attrs); - mail_store - .store_email(&acct_name, &folder, uid, uidvalidity, attrs) - .await?; - } + let uids = (1..=exists).collect::>(); + let fetched = authed + .uid_fetch(&uids, FetchItems::PanoramaAll) + .await + .context("error fetching uids")?; + + fetched + .map(Ok) + .try_for_each_concurrent(None, |(uid, attrs)| { + mail_store.store_email(&acct_name, &folder, uid, uidvalidity, attrs) + }) + .await + .context("error during fetch-store")?; } } } diff --git a/src/mail/mod.rs b/src/mail/mod.rs index 0eec232..ec4b1a9 100644 --- a/src/mail/mod.rs +++ b/src/mail/mod.rs @@ -88,7 +88,10 @@ pub async fn run_mail( { Ok(_) => {} Err(err) => { - error!("IMAP Error: {}", err); + error!("error from sync_main: {}", err); + for err in err.chain() { + error!("cause: {}", err); + } } } diff --git a/src/mail/store.rs b/src/mail/store.rs index f8a3121..73b7b70 100644 --- a/src/mail/store.rs +++ b/src/mail/store.rs @@ -1,8 +1,9 @@ //! Package for managing the offline storage of emails -use std::path::{PathBuf, Path}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; -use anyhow::Result; +use anyhow::{Context, Result}; use panorama_imap::response::AttributeValue; use sha2::{Digest, Sha256}; use sqlx::{ @@ -10,7 +11,7 @@ use sqlx::{ sqlite::{Sqlite, SqlitePool}, Error, }; -use tokio::fs; +use tokio::{fs, sync::broadcast}; use crate::config::Config; @@ -24,29 +25,32 @@ pub struct MailStore { config: Config, mail_dir: PathBuf, pool: SqlitePool, + // email_events: broadcast::Sender, } +#[derive(Clone, Debug)] +pub struct EmailUpdateInfo {} + impl MailStore { /// Creates a new MailStore pub async fn new(config: Config) -> Result { - let mail_dir = config.mail_dir.to_string_lossy(); - let mail_dir_str = shellexpand::tilde(mail_dir.as_ref()); - let mail_dir = PathBuf::from(mail_dir_str.as_ref()); + let data_dir = config.data_dir.to_string_lossy(); + let data_dir = PathBuf::from(shellexpand::tilde(data_dir.as_ref()).as_ref()); + + let mail_dir = data_dir.join("mail"); if !mail_dir.exists() { fs::create_dir_all(&mail_dir).await?; } info!("using mail dir: {:?}", mail_dir); // create database parent - let db_path = config.db_path.to_string_lossy(); - let db_path_str = shellexpand::tilde(db_path.as_ref()); - - let db_path = PathBuf::from(db_path_str.as_ref()); + let db_path = data_dir.join("panorama.db"); let db_parent = db_path.parent(); if let Some(path) = db_parent { fs::create_dir_all(path).await?; } + let db_path_str = db_path.to_string_lossy(); let db_path = format!("sqlite:{}", db_path_str); info!("using database path: {}", db_path_str); @@ -59,11 +63,25 @@ impl MailStore { MIGRATOR.run(&pool).await?; debug!("run migrations : {:?}", MIGRATOR); - Ok(MailStore { config, mail_dir, pool }) + // let (new_email_tx, new_email_rx) = broadcast::channel(100); + + Ok(MailStore { + config, + mail_dir, + pool, + // email_events: new_email_tx, + }) } - /// Gets the list of all the UIDs in the given folder that need to be updated - pub fn get_new_uids(&self, exists: u32) {} + // /// Subscribes to the email updates + // pub fn subscribe(&self) -> broadcast::Receiver { + // self.email_events.subscribe() + // } + + /// Try to identify an email based on the UID, message-id, and other heuristics + pub async fn try_identify_email() { + + } /// Stores the given email pub async fn store_email( @@ -91,7 +109,24 @@ impl MailStore { let hash = hasher.finalize(); let filename = format!("{}.mail", hex::encode(hash)); let path = self.mail_dir.join(&filename); - fs::write(path, body).await?; + fs::write(path, &body) + .await + .context("error writing email to file")?; + + // parse email + let mut message_id = None; + let mail = mailparse::parse_mail(body.as_bytes()) + .with_context(|| format!("error parsing email with uid {}", uid))?; + for header in mail.headers.iter() { + let key = header.get_key_ref(); + let key = key.to_ascii_lowercase(); + let value = header.get_value(); + if key == "message-id" { + message_id = Some(value); + } + } + + debug!("message-id: {:?}", message_id); let existing = sqlx::query( r#" @@ -116,20 +151,26 @@ impl MailStore { if !exists { let id = sqlx::query( r#" - INSERT INTO "mail" (account, folder, uid, uidvalidity, filename) - VALUES (?, ?, ?, ?, ?) + INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename) + VALUES (?, ?, ?, ?, ?, ?) "#, ) .bind(acct.as_ref()) + .bind(message_id) .bind(folder.as_ref()) .bind(uid) .bind(uidvalidity) .bind(filename) .execute(&self.pool) - .await? + .await + .context("error inserting email into db")? .last_insert_rowid(); } + // self.email_events + // .send(EmailUpdateInfo {}) + // .context("error sending email update info to the broadcast channel")?; + Ok(()) } }