From 6b821371245ea66afe2f7b47d5f9e2acf8b352eb Mon Sep 17 00:00:00 2001
From: Michael Zhang <mail@mzhang.io>
Date: Thu, 25 Mar 2021 15:23:39 -0500
Subject: [PATCH] downloading mostly works for larger batches of email now

---
 imap/src/parser/mod.rs       |  3 ++-
 imap/src/parser/rfc3501.pest |  2 +-
 migrations/1_initial.sql     |  2 ++
 src/mail/client.rs           |  4 ++--
 src/mail/store.rs            | 27 ++++++++++++++++++++-------
 5 files changed, 27 insertions(+), 11 deletions(-)

diff --git a/imap/src/parser/mod.rs b/imap/src/parser/mod.rs
index 98b86e2..d7afb82 100644
--- a/imap/src/parser/mod.rs
+++ b/imap/src/parser/mod.rs
@@ -36,10 +36,11 @@ pub fn parse_capability(s: impl AsRef<str>) -> ParseResult<Capability> {
 
 pub fn parse_streamed_response(s: impl AsRef<str>) -> ParseResult<(Response, usize)> {
     let s = s.as_ref();
+    let len = s.len();
     let mut pairs = match Rfc3501::parse(Rule::streamed_response, s) {
         Ok(v) => v,
         Err(e) => {
-            // error!("stream failed: {}", e);
+            // error!("stream failed with len {}: {}", len ,e);
             return Err(e);
         }
     };
diff --git a/imap/src/parser/rfc3501.pest b/imap/src/parser/rfc3501.pest
index e708492..d748998 100644
--- a/imap/src/parser/rfc3501.pest
+++ b/imap/src/parser/rfc3501.pest
@@ -44,7 +44,7 @@ date_month = { "Jan" | "Feb" | "Mar" | "Apr" | "May" | "Jun" | "Jul" | "Aug" | "
 date_time = { dquote_ ~ date_day_fixed ~ "-" ~ date_month ~ "-" ~ date_year ~ sp ~ time ~ sp ~ zone ~ dquote_ }
 date_year = @{ digit{4} }
 digit_nz = @{ '\x31'..'\x39' }
-env_address1 = { "(" ~ address ~ (sp? ~ address)? ~ ")" }
+env_address1 = { "(" ~ address ~ (sp? ~ address)* ~ ")" }
 env_bcc = { env_address1 | nil }
 env_cc = { env_address1 | nil }
 env_date = { nstring }
diff --git a/migrations/1_initial.sql b/migrations/1_initial.sql
index 5dc0089..d3d4216 100644
--- a/migrations/1_initial.sql
+++ b/migrations/1_initial.sql
@@ -6,10 +6,12 @@ CREATE TABLE IF NOT EXISTS "accounts" (
 
 CREATE TABLE IF NOT EXISTS "mail" (
     "id" INTEGER PRIMARY KEY,
+    "internaldate" TEXT,
     "message_id" TEXT,
     "account" TEXT,
     "folder" TEXT,
     "uidvalidity" INTEGER,
+    "subject" TEXT,
     "uid" INTEGER,
     "filename" TEXT
 );
diff --git a/src/mail/client.rs b/src/mail/client.rs
index eaf7352..3811f09 100644
--- a/src/mail/client.rs
+++ b/src/mail/client.rs
@@ -75,14 +75,14 @@ pub async fn sync_main(
             debug!("select response: {:?}", select);
 
             if let (Some(exists), Some(uidvalidity)) = (select.exists, select.uid_validity) {
-                if exists < 10 {
-                    let new_uids = stream::iter(1..exists).map(Ok).try_filter_map(|uid| {
+                let new_uids = stream::iter(1..exists).map(Ok).try_filter_map(|uid| {
                         mail_store.try_identify_email(&acct_name, &folder, uid, uidvalidity, None)
                             // invert the option to only select uids that haven't been downloaded
                             .map_ok(move |o| o.map_or_else(move || Some(uid), |v| None))
                             .map_err(|err| err.context("error checking if the email is already downloaded [try_identify_email]"))
                     }).try_collect::<Vec<_>>().await?;
 
+                if !new_uids.is_empty() {
                     debug!("fetching uids {:?}", new_uids);
                     let fetched = authed
                         .uid_fetch(&new_uids, FetchItems::PanoramaAll)
diff --git a/src/mail/store.rs b/src/mail/store.rs
index e7c7187..e1d572e 100644
--- a/src/mail/store.rs
+++ b/src/mail/store.rs
@@ -127,9 +127,12 @@ impl MailStore {
         attrs: Vec<AttributeValue>,
     ) -> Result<()> {
         let mut body = None;
+        let mut internaldate = None;
         for attr in attrs {
-            if let AttributeValue::BodySection(body_attr) = attr {
-                body = body_attr.data;
+            match attr {
+                AttributeValue::BodySection(body_attr) => body = body_attr.data,
+                AttributeValue::InternalDate(date) => internaldate = Some(date),
+                _ => {}
             }
         }
 
@@ -137,6 +140,10 @@ impl MailStore {
             Some(v) => v,
             None => return Ok(()),
         };
+        let internaldate = match internaldate {
+            Some(v) => v,
+            None => return Ok(()),
+        };
 
         let mut hasher = Sha256::new();
         hasher.update(body.as_bytes());
@@ -149,14 +156,16 @@ impl MailStore {
 
         // parse email
         let mut message_id = None;
+        let mut subject = None;
         let mail = mailparse::parse_mail(body.as_bytes())
             .with_context(|| format!("error parsing email with uid {}", uid))?;
         for header in mail.headers.iter() {
             let key = header.get_key_ref();
-            let key = key.to_ascii_lowercase();
             let value = header.get_value();
-            if key == "message-id" {
-                message_id = Some(value);
+            match key.to_ascii_lowercase().as_str() {
+                "message-id" => message_id = Some(value),
+                "subject" => subject = Some(value),
+                _ => {}
             }
         }
 
@@ -181,16 +190,20 @@ impl MailStore {
         if existing.is_none() {
             let id = sqlx::query(
                 r#"
-                INSERT INTO "mail" (account, message_id, folder, uid, uidvalidity, filename)
-                VALUES (?, ?, ?, ?, ?, ?)
+                INSERT INTO "mail" (
+                    account, subject, message_id, folder, uid, uidvalidity,
+                    filename, internaldate
+                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
                 "#,
             )
             .bind(acct.as_ref())
+            .bind(subject)
             .bind(message_id)
             .bind(folder.as_ref())
             .bind(uid)
             .bind(uidvalidity)
             .bind(filename)
+            .bind(internaldate.to_rfc3339())
             .execute(&self.pool)
             .await
             .context("error inserting email into db")?