config data dir

This commit is contained in:
Michael Zhang 2021-03-25 13:27:57 -05:00
parent 14b07adf66
commit 7402b83f14
Signed by: michael
GPG key ID: BDA47A31A3C8EE6B
7 changed files with 129 additions and 36 deletions

50
Cargo.lock generated
View file

@ -146,6 +146,21 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
dependencies = [
"byteorder",
]
[[package]]
name = "base64"
version = "0.12.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.13.0" version = "0.13.0"
@ -373,6 +388,16 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "charset"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f426e64df1c3de26cbf44593c6ffff5dbfd43bbf9de0d075058558126b3fc73"
dependencies = [
"base64 0.10.1",
"encoding_rs",
]
[[package]] [[package]]
name = "chrono" name = "chrono"
version = "0.4.19" version = "0.4.19"
@ -821,6 +846,15 @@ dependencies = [
"log", "log",
] ]
[[package]]
name = "encoding_rs"
version = "0.8.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065"
dependencies = [
"cfg-if 1.0.0",
]
[[package]] [[package]]
name = "enumflags2" name = "enumflags2"
version = "0.6.4" version = "0.6.4"
@ -1708,6 +1742,17 @@ dependencies = [
"objc-foundation", "objc-foundation",
] ]
[[package]]
name = "mailparse"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31de1f9043c582efde7dbd93de56600df12b6c4488a67eeaefa74ea364019b22"
dependencies = [
"base64 0.12.3",
"charset",
"quoted_printable",
]
[[package]] [[package]]
name = "malloc_buf" name = "malloc_buf"
version = "0.0.6" version = "0.0.6"
@ -1978,6 +2023,7 @@ dependencies = [
"hex 0.4.3", "hex 0.4.3",
"inotify", "inotify",
"log", "log",
"mailparse",
"notify-rust", "notify-rust",
"panorama-imap", "panorama-imap",
"panorama-smtp", "panorama-smtp",
@ -2464,7 +2510,7 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb"
dependencies = [ dependencies = [
"base64", "base64 0.13.0",
"blake2b_simd", "blake2b_simd",
"constant_time_eq", "constant_time_eq",
"crossbeam-utils 0.8.3", "crossbeam-utils 0.8.3",
@ -2500,7 +2546,7 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b" checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
dependencies = [ dependencies = [
"base64", "base64 0.13.0",
"log", "log",
"ring", "ring",
"sct", "sct",

View file

@ -45,6 +45,7 @@ sqlx = { version = "0.5.1", features = ["runtime-tokio-rustls", "sqlite"] }
sha2 = "0.9.3" sha2 = "0.9.3"
hex = "0.4.3" hex = "0.4.3"
shellexpand = "2.1.0" shellexpand = "2.1.0"
mailparse = "0.13.2"
[dependencies.panorama-imap] [dependencies.panorama-imap]
path = "imap" path = "imap"

View file

@ -6,6 +6,7 @@ CREATE TABLE IF NOT EXISTS "accounts" (
CREATE TABLE IF NOT EXISTS "mail" ( CREATE TABLE IF NOT EXISTS "mail" (
"id" INTEGER PRIMARY KEY, "id" INTEGER PRIMARY KEY,
"message_id" TEXT,
"account" TEXT, "account" TEXT,
"folder" TEXT, "folder" TEXT,
"uidvalidity" INTEGER, "uidvalidity" INTEGER,

View file

@ -25,11 +25,8 @@ pub struct Config {
/// (potentially for migration later?) /// (potentially for migration later?)
pub version: String, pub version: String,
/// Directory to store mail in /// Directory to store panorama-related data in
pub mail_dir: PathBuf, pub data_dir: PathBuf,
/// SQLite database path
pub db_path: PathBuf,
/// Mail accounts /// Mail accounts
#[serde(rename = "mail")] #[serde(rename = "mail")]

View file

@ -1,7 +1,7 @@
use anyhow::Result; use anyhow::{Context, Result};
use futures::{ use futures::{
future::FutureExt, future::FutureExt,
stream::{Stream, StreamExt}, stream::{Stream, StreamExt, TryStreamExt},
}; };
use notify_rust::{Notification, Timeout}; use notify_rust::{Notification, Timeout};
use panorama_imap::{ use panorama_imap::{
@ -76,15 +76,19 @@ pub async fn sync_main(
if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) { if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) {
if exists < 10 { if exists < 10 {
let mut fetched = authed let uids = (1..=exists).collect::<Vec<_>>();
.uid_fetch(&(1..=exists).collect::<Vec<_>>(), FetchItems::PanoramaAll) let fetched = authed
.await?; .uid_fetch(&uids, FetchItems::PanoramaAll)
while let Some((uid, attrs)) = fetched.next().await { .await
debug!("- {} : {:?}", uid, attrs); .context("error fetching uids")?;
mail_store
.store_email(&acct_name, &folder, uid, uidvalidity, attrs) fetched
.await?; .map(Ok)
} .try_for_each_concurrent(None, |(uid, attrs)| {
mail_store.store_email(&acct_name, &folder, uid, uidvalidity, attrs)
})
.await
.context("error during fetch-store")?;
} }
} }
} }

View file

@ -88,7 +88,10 @@ pub async fn run_mail(
{ {
Ok(_) => {} Ok(_) => {}
Err(err) => { Err(err) => {
error!("IMAP Error: {}", err); error!("error from sync_main: {}", err);
for err in err.chain() {
error!("cause: {}", err);
}
} }
} }

View file

@ -1,8 +1,9 @@
//! Package for managing the offline storage of emails //! Package for managing the offline storage of emails
use std::path::{PathBuf, Path}; use std::path::{Path, PathBuf};
use std::sync::Arc;
use anyhow::Result; use anyhow::{Context, Result};
use panorama_imap::response::AttributeValue; use panorama_imap::response::AttributeValue;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use sqlx::{ use sqlx::{
@ -10,7 +11,7 @@ use sqlx::{
sqlite::{Sqlite, SqlitePool}, sqlite::{Sqlite, SqlitePool},
Error, Error,
}; };
use tokio::fs; use tokio::{fs, sync::broadcast};
use crate::config::Config; use crate::config::Config;
@ -24,29 +25,32 @@ pub struct MailStore {
config: Config, config: Config,
mail_dir: PathBuf, mail_dir: PathBuf,
pool: SqlitePool, pool: SqlitePool,
// email_events: broadcast::Sender<EmailUpdateInfo>,
} }
#[derive(Clone, Debug)]
pub struct EmailUpdateInfo {}
impl MailStore { impl MailStore {
/// Creates a new MailStore /// Creates a new MailStore
pub async fn new(config: Config) -> Result<Self> { pub async fn new(config: Config) -> Result<Self> {
let mail_dir = config.mail_dir.to_string_lossy(); let data_dir = config.data_dir.to_string_lossy();
let mail_dir_str = shellexpand::tilde(mail_dir.as_ref()); let data_dir = PathBuf::from(shellexpand::tilde(data_dir.as_ref()).as_ref());
let mail_dir = PathBuf::from(mail_dir_str.as_ref());
let mail_dir = data_dir.join("mail");
if !mail_dir.exists() { if !mail_dir.exists() {
fs::create_dir_all(&mail_dir).await?; fs::create_dir_all(&mail_dir).await?;
} }
info!("using mail dir: {:?}", mail_dir); info!("using mail dir: {:?}", mail_dir);
// create database parent // create database parent
let db_path = config.db_path.to_string_lossy(); let db_path = data_dir.join("panorama.db");
let db_path_str = shellexpand::tilde(db_path.as_ref());
let db_path = PathBuf::from(db_path_str.as_ref());
let db_parent = db_path.parent(); let db_parent = db_path.parent();
if let Some(path) = db_parent { if let Some(path) = db_parent {
fs::create_dir_all(path).await?; fs::create_dir_all(path).await?;
} }
let db_path_str = db_path.to_string_lossy();
let db_path = format!("sqlite:{}", db_path_str); let db_path = format!("sqlite:{}", db_path_str);
info!("using database path: {}", db_path_str); info!("using database path: {}", db_path_str);
@ -59,11 +63,25 @@ impl MailStore {
MIGRATOR.run(&pool).await?; MIGRATOR.run(&pool).await?;
debug!("run migrations : {:?}", MIGRATOR); debug!("run migrations : {:?}", MIGRATOR);
Ok(MailStore { config, mail_dir, pool }) // let (new_email_tx, new_email_rx) = broadcast::channel(100);
Ok(MailStore {
config,
mail_dir,
pool,
// email_events: new_email_tx,
})
} }
/// Gets the list of all the UIDs in the given folder that need to be updated // /// Subscribes to the email updates
pub fn get_new_uids(&self, exists: u32) {} // pub fn subscribe(&self) -> broadcast::Receiver<EmailUpdateInfo> {
// self.email_events.subscribe()
// }
/// Try to identify an email based on the UID, message-id, and other heuristics
pub async fn try_identify_email() {
}
/// Stores the given email /// Stores the given email
pub async fn store_email( pub async fn store_email(
@ -91,7 +109,24 @@ impl MailStore {
let hash = hasher.finalize(); let hash = hasher.finalize();
let filename = format!("{}.mail", hex::encode(hash)); let filename = format!("{}.mail", hex::encode(hash));
let path = self.mail_dir.join(&filename); let path = self.mail_dir.join(&filename);
fs::write(path, body).await?; fs::write(path, &body)
.await
.context("error writing email to file")?;
// parse email
let mut message_id = None;
let mail = mailparse::parse_mail(body.as_bytes())
.with_context(|| format!("error parsing email with uid {}", uid))?;
for header in mail.headers.iter() {
let key = header.get_key_ref();
let key = key.to_ascii_lowercase();
let value = header.get_value();
if key == "message-id" {
message_id = Some(value);
}
}
debug!("message-id: {:?}", message_id);
let existing = sqlx::query( let existing = sqlx::query(
r#" r#"
@ -116,20 +151,26 @@ impl MailStore {
if !exists { if !exists {
let id = sqlx::query( let id = sqlx::query(
r#" r#"
INSERT INTO "mail" (account, folder, uid, uidvalidity, filename) INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename)
VALUES (?, ?, ?, ?, ?) VALUES (?, ?, ?, ?, ?, ?)
"#, "#,
) )
.bind(acct.as_ref()) .bind(acct.as_ref())
.bind(message_id)
.bind(folder.as_ref()) .bind(folder.as_ref())
.bind(uid) .bind(uid)
.bind(uidvalidity) .bind(uidvalidity)
.bind(filename) .bind(filename)
.execute(&self.pool) .execute(&self.pool)
.await? .await
.context("error inserting email into db")?
.last_insert_rowid(); .last_insert_rowid();
} }
// self.email_events
// .send(EmailUpdateInfo {})
// .context("error sending email update info to the broadcast channel")?;
Ok(()) Ok(())
} }
} }