config data dir

Cargo.lock

@@ -146,6 +146,21 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
 
+[[package]]
+name = "base64"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "base64"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
+
 [[package]]
 name = "base64"
 version = "0.13.0"
@@ -373,6 +388,16 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "charset"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f426e64df1c3de26cbf44593c6ffff5dbfd43bbf9de0d075058558126b3fc73"
+dependencies = [
+ "base64 0.10.1",
+ "encoding_rs",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.19"
@@ -821,6 +846,15 @@ dependencies = [
  "log",
 ]
 
+[[package]]
+name = "encoding_rs"
+version = "0.8.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
 [[package]]
 name = "enumflags2"
 version = "0.6.4"
@@ -1708,6 +1742,17 @@ dependencies = [
  "objc-foundation",
 ]
 
+[[package]]
+name = "mailparse"
+version = "0.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31de1f9043c582efde7dbd93de56600df12b6c4488a67eeaefa74ea364019b22"
+dependencies = [
+ "base64 0.12.3",
+ "charset",
+ "quoted_printable",
+]
+
 [[package]]
 name = "malloc_buf"
 version = "0.0.6"
@@ -1978,6 +2023,7 @@ dependencies = [
  "hex 0.4.3",
  "inotify",
  "log",
+ "mailparse",
  "notify-rust",
  "panorama-imap",
  "panorama-smtp",
@@ -2464,7 +2510,7 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"
 dependencies = [
- "base64",
+ "base64 0.13.0",
  "blake2b_simd",
  "constant_time_eq",
  "crossbeam-utils 0.8.3",
@@ -2500,7 +2546,7 @@ version = "0.19.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
 dependencies = [
- "base64",
+ "base64 0.13.0",
  "log",
  "ring",
  "sct",

Cargo.toml

@@ -45,6 +45,7 @@ sqlx = { version = "0.5.1", features = ["runtime-tokio-rustls", "sqlite"] }
 sha2 = "0.9.3"
 hex = "0.4.3"
 shellexpand = "2.1.0"
+mailparse = "0.13.2"
 
 [dependencies.panorama-imap]
 path = "imap"

migrations/1_initial.sql

@@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS "accounts" (
 
 CREATE TABLE IF NOT EXISTS "mail" (
     "id" INTEGER PRIMARY KEY,
+    "message_id" TEXT,
     "account" TEXT,
     "folder" TEXT,
     "uidvalidity" INTEGER,

src/config.rs

@@ -25,11 +25,8 @@ pub struct Config {
     /// (potentially for migration later?)
     pub version: String,
 
-    /// Directory to store mail in
+    /// Directory to store panorama-related data in
-    pub mail_dir: PathBuf,
+    pub data_dir: PathBuf,
-
-    /// SQLite database path
-    pub db_path: PathBuf,
 
     /// Mail accounts
     #[serde(rename = "mail")]

src/mail/client.rs

@@ -1,7 +1,7 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use futures::{
     future::FutureExt,
-    stream::{Stream, StreamExt},
+    stream::{Stream, StreamExt, TryStreamExt},
 };
 use notify_rust::{Notification, Timeout};
 use panorama_imap::{
@@ -76,15 +76,19 @@ pub async fn sync_main(
 
             if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) {
                 if exists < 10 {
-                    let mut fetched = authed
+                    let uids = (1..=exists).collect::<Vec<_>>();
-                        .uid_fetch(&(1..=exists).collect::<Vec<_>>(), FetchItems::PanoramaAll)
+                    let fetched = authed
-                        .await?;
+                        .uid_fetch(&uids, FetchItems::PanoramaAll)
-                    while let Some((uid, attrs)) = fetched.next().await {
+                        .await
-                        debug!("- {} : {:?}", uid, attrs);
+                        .context("error fetching uids")?;
-                        mail_store
+
-                            .store_email(&acct_name, &folder, uid, uidvalidity, attrs)
+                    fetched
-                            .await?;
+                        .map(Ok)
-                    }
+                        .try_for_each_concurrent(None, |(uid, attrs)| {
+                            mail_store.store_email(&acct_name, &folder, uid, uidvalidity, attrs)
+                        })
+                        .await
+                        .context("error during fetch-store")?;
                 }
             }
         }

src/mail/mod.rs

@@ -88,7 +88,10 @@ pub async fn run_mail(
                     {
                         Ok(_) => {}
                         Err(err) => {
-                            error!("IMAP Error: {}", err);
+                            error!("error from sync_main: {}", err);
+                            for err in err.chain() {
+                                error!("cause: {}", err);
+                            }
                         }
                     }
 

src/mail/store.rs

@@ -1,8 +1,9 @@
 //! Package for managing the offline storage of emails
 
-use std::path::{PathBuf, Path};
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 use panorama_imap::response::AttributeValue;
 use sha2::{Digest, Sha256};
 use sqlx::{
@@ -10,7 +11,7 @@ use sqlx::{
     sqlite::{Sqlite, SqlitePool},
     Error,
 };
-use tokio::fs;
+use tokio::{fs, sync::broadcast};
 
 use crate::config::Config;
 
@@ -24,29 +25,32 @@ pub struct MailStore {
     config: Config,
     mail_dir: PathBuf,
     pool: SqlitePool,
+    // email_events: broadcast::Sender<EmailUpdateInfo>,
 }
 
+#[derive(Clone, Debug)]
+pub struct EmailUpdateInfo {}
+
 impl MailStore {
     /// Creates a new MailStore
     pub async fn new(config: Config) -> Result<Self> {
-        let mail_dir = config.mail_dir.to_string_lossy();
+        let data_dir = config.data_dir.to_string_lossy();
-        let mail_dir_str = shellexpand::tilde(mail_dir.as_ref());
+        let data_dir = PathBuf::from(shellexpand::tilde(data_dir.as_ref()).as_ref());
-        let mail_dir = PathBuf::from(mail_dir_str.as_ref());
+
+        let mail_dir = data_dir.join("mail");
         if !mail_dir.exists() {
             fs::create_dir_all(&mail_dir).await?;
         }
         info!("using mail dir: {:?}", mail_dir);
 
         // create database parent
-        let db_path = config.db_path.to_string_lossy();
+        let db_path = data_dir.join("panorama.db");
-        let db_path_str = shellexpand::tilde(db_path.as_ref());
-
-        let db_path = PathBuf::from(db_path_str.as_ref());
         let db_parent = db_path.parent();
         if let Some(path) = db_parent {
             fs::create_dir_all(path).await?;
         }
 
+        let db_path_str = db_path.to_string_lossy();
         let db_path = format!("sqlite:{}", db_path_str);
         info!("using database path: {}", db_path_str);
 
@@ -59,11 +63,25 @@ impl MailStore {
         MIGRATOR.run(&pool).await?;
         debug!("run migrations : {:?}", MIGRATOR);
 
-        Ok(MailStore { config, mail_dir, pool })
+        // let (new_email_tx, new_email_rx) = broadcast::channel(100);
+
+        Ok(MailStore {
+            config,
+            mail_dir,
+            pool,
+            // email_events: new_email_tx,
+        })
     }
 
-    /// Gets the list of all the UIDs in the given folder that need to be updated
+    // /// Subscribes to the email updates
-    pub fn get_new_uids(&self, exists: u32) {}
+    // pub fn subscribe(&self) -> broadcast::Receiver<EmailUpdateInfo> {
+    //     self.email_events.subscribe()
+    // }
+
+    /// Try to identify an email based on the UID, message-id, and other heuristics
+    pub async fn try_identify_email() {
+
+    }
 
     /// Stores the given email
     pub async fn store_email(
@@ -91,7 +109,24 @@ impl MailStore {
         let hash = hasher.finalize();
         let filename = format!("{}.mail", hex::encode(hash));
         let path = self.mail_dir.join(&filename);
-        fs::write(path, body).await?;
+        fs::write(path, &body)
+            .await
+            .context("error writing email to file")?;
+
+        // parse email
+        let mut message_id = None;
+        let mail = mailparse::parse_mail(body.as_bytes())
+            .with_context(|| format!("error parsing email with uid {}", uid))?;
+        for header in mail.headers.iter() {
+            let key = header.get_key_ref();
+            let key = key.to_ascii_lowercase();
+            let value = header.get_value();
+            if key == "message-id" {
+                message_id = Some(value);
+            }
+        }
+
+        debug!("message-id: {:?}", message_id);
 
         let existing = sqlx::query(
             r#"
@@ -116,20 +151,26 @@ impl MailStore {
         if !exists {
             let id = sqlx::query(
                 r#"
-                INSERT INTO "mail" (account, folder, uid, uidvalidity, filename)
+                INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename)
-                VALUES (?, ?, ?, ?, ?)
+                VALUES (?, ?, ?, ?, ?, ?)
                 "#,
             )
             .bind(acct.as_ref())
+            .bind(message_id)
             .bind(folder.as_ref())
             .bind(uid)
             .bind(uidvalidity)
             .bind(filename)
             .execute(&self.pool)
-            .await?
+            .await
+            .context("error inserting email into db")?
             .last_insert_rowid();
         }
 
+        // self.email_events
+        //     .send(EmailUpdateInfo {})
+        //     .context("error sending email update info to the broadcast channel")?;
+
         Ok(())
     }
 }