From 5c2a35935ea1faedfa98e6ed10adf56b132d44e1 Mon Sep 17 00:00:00 2001 From: Michael Zhang Date: Mon, 3 Jun 2024 01:02:45 -0400 Subject: [PATCH] create node test --- Cargo.lock | 1 + crates/panorama-core/Cargo.toml | 1 + crates/panorama-core/src/lib.rs | 2 + crates/panorama-core/src/migrations.rs | 7 +- crates/panorama-core/src/state/export.rs | 70 +++++++++ crates/panorama-core/src/state/mail.rs | 5 +- crates/panorama-core/src/state/mod.rs | 3 +- crates/panorama-core/src/state/node.rs | 192 ++++++++++++++++++++++- crates/panorama-core/src/tests/mod.rs | 34 +++- crates/panorama-daemon/src/export.rs | 4 +- crates/panorama-daemon/src/journal.rs | 20 ++- crates/panorama-daemon/src/main.rs | 21 +-- crates/panorama-daemon/src/node.rs | 48 +++--- 13 files changed, 353 insertions(+), 55 deletions(-) create mode 100644 crates/panorama-core/src/state/export.rs diff --git a/Cargo.lock b/Cargo.lock index e09c703..b3c9f7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3516,6 +3516,7 @@ dependencies = [ "chrono", "cozo", "futures", + "itertools 0.13.0", "miette", "serde", "serde_json", diff --git a/crates/panorama-core/Cargo.toml b/crates/panorama-core/Cargo.toml index 78fd9ed..f345704 100644 --- a/crates/panorama-core/Cargo.toml +++ b/crates/panorama-core/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" chrono = { version = "0.4.38", features = ["serde"] } cozo = { version = "0.7.6", features = ["storage-rocksdb"] } futures = "0.3.30" +itertools = "0.13.0" miette = "5.5.0" serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" diff --git a/crates/panorama-core/src/lib.rs b/crates/panorama-core/src/lib.rs index 856327b..1875d6b 100644 --- a/crates/panorama-core/src/lib.rs +++ b/crates/panorama-core/src/lib.rs @@ -1,6 +1,8 @@ #[macro_use] extern crate serde; #[macro_use] +extern crate serde_json; +#[macro_use] extern crate sugars; pub mod migrations; diff --git a/crates/panorama-core/src/migrations.rs b/crates/panorama-core/src/migrations.rs index 32ba4e0..33761a6 100644 --- a/crates/panorama-core/src/migrations.rs +++ b/crates/panorama-core/src/migrations.rs @@ -116,6 +116,7 @@ fn migration_01(db: &DbInstance) -> Result<()> { # Inverse mappings for easy querying { :create node_has_key { key: String => id: String } } + { ::index create node_has_key:inverse { id } } { :create node_managed_by_app { node_id: String => app: String } } { :create node_refers_to { node_id: String => other_node_id: String } } { @@ -125,18 +126,18 @@ fn migration_01(db: &DbInstance) -> Result<()> { relation: String, field_name: String, type: String, - fts_enabled: Bool, + is_fts_enabled: Bool, } } { - ?[key, relation, field_name, type, fts_enabled] <- [ + ?[key, relation, field_name, type, is_fts_enabled] <- [ ['panorama/journal/page/content', 'journal', 'content', 'string', true], ['panorama/mail/config/imap_hostname', 'mail_config', 'imap_hostname', 'string', false], ['panorama/mail/config/imap_port', 'mail_config', 'imap_port', 'int', false], ['panorama/mail/config/imap_username', 'mail_config', 'imap_username', 'string', false], ['panorama/mail/config/imap_password', 'mail_config', 'imap_password', 'string', false], ] - :put fqkey_to_dbkey { key, relation, field_name, type, fts_enabled } + :put fqkey_to_dbkey { key, relation, field_name, type, is_fts_enabled } } # Create journal type diff --git a/crates/panorama-core/src/state/export.rs b/crates/panorama-core/src/state/export.rs new file mode 100644 index 0000000..5307692 --- /dev/null +++ b/crates/panorama-core/src/state/export.rs @@ -0,0 +1,70 @@ +use std::collections::HashMap; + +use cozo::ScriptMutability; +use miette::Result; +use serde_json::Value; + +use crate::AppState; + +impl AppState { + pub async fn export(&self) -> Result { + let result = self.db.run_script( + "::relations", + Default::default(), + ScriptMutability::Immutable, + )?; + + let name_index = result.headers.iter().position(|x| x == "name").unwrap(); + let relation_names = result + .rows + .into_iter() + .map(|row| row[name_index].get_str().unwrap().to_owned()) + .collect::>(); + + let mut relation_columns = HashMap::new(); + + for relation_name in relation_names.iter() { + let result = self.db.run_script( + &format!("::columns {relation_name}"), + Default::default(), + ScriptMutability::Immutable, + )?; + + let column_index = + result.headers.iter().position(|x| x == "column").unwrap(); + let columns = result + .rows + .into_iter() + .map(|row| row[column_index].get_str().unwrap().to_owned()) + .collect::>(); + + relation_columns.insert(relation_name.clone(), columns); + } + + let tx = self.db.multi_transaction(false); + + let mut all_relations = hmap! {}; + for relation_name in relation_names.iter() { + let mut relation_info = vec![]; + + let columns = relation_columns.get(relation_name.as_str()).unwrap(); + let columns_str = columns.join(", "); + + let query = + format!("?[{columns_str}] := *{relation_name} {{ {columns_str} }}"); + let result = tx.run_script(&query, Default::default())?; + + for row in result.rows.into_iter() { + let mut object = hmap! {}; + row.into_iter().enumerate().for_each(|(idx, col)| { + object.insert(columns[idx].to_owned(), col); + }); + relation_info.push(object); + } + + all_relations.insert(relation_name.to_owned(), relation_info); + } + + Ok(json!({"relations": all_relations})) + } +} diff --git a/crates/panorama-core/src/state/mail.rs b/crates/panorama-core/src/state/mail.rs index e8b1b99..d309b80 100644 --- a/crates/panorama-core/src/state/mail.rs +++ b/crates/panorama-core/src/state/mail.rs @@ -1,9 +1,8 @@ -use std::{collections::HashMap, default, time::Duration}; +use std::{collections::HashMap, time::Duration}; -use cozo::{DataValue, DbInstance, JsonData, ScriptMutability}; +use cozo::{DataValue, JsonData, ScriptMutability}; use futures::TryStreamExt; use miette::{IntoDiagnostic, Result}; -use serde_json::Value; use tokio::{net::TcpStream, time::sleep}; use uuid::Uuid; diff --git a/crates/panorama-core/src/state/mod.rs b/crates/panorama-core/src/state/mod.rs index ea31f3f..3badbcd 100644 --- a/crates/panorama-core/src/state/mod.rs +++ b/crates/panorama-core/src/state/mod.rs @@ -1,3 +1,4 @@ +pub mod export; pub mod mail; pub mod node; @@ -7,7 +8,7 @@ use cozo::DbInstance; use miette::{IntoDiagnostic, Result}; use tantivy::{ directory::MmapDirectory, - schema::{self, Schema, STORED, STRING, TEXT}, + schema::{Schema, STORED, STRING, TEXT}, Index, }; diff --git a/crates/panorama-core/src/state/node.rs b/crates/panorama-core/src/state/node.rs index 047930b..b27e220 100644 --- a/crates/panorama-core/src/state/node.rs +++ b/crates/panorama-core/src/state/node.rs @@ -1,18 +1,33 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use chrono::{DateTime, Utc}; -use cozo::{DataValue, ScriptMutability}; +use cozo::{DataValue, MultiTransaction, ScriptMutability}; +use itertools::Itertools; use miette::Result; +use serde_json::Value; +use uuid::Uuid; use crate::AppState; +pub type ExtraData = BTreeMap; + +#[derive(Debug)] pub struct NodeInfo { pub node_id: String, pub created_at: DateTime, pub updated_at: DateTime, - pub fields: HashMap, + pub fields: Option>, } +pub struct FieldInfo { + pub relation_name: String, + pub relation_field: String, + pub r#type: String, + pub is_fts_enabled: bool, +} + +pub type FieldMapping = HashMap; + impl AppState { /// Get all properties of a node pub async fn get_node(&self, node_id: impl AsRef) -> Result { @@ -20,15 +35,182 @@ impl AppState { let result = self.db.run_script( " - ?[relation, field_name, type, fts_enabled] := + ?[relation, field_name, type, is_fts_enabled] := *node_has_key { key, id }, - *fqkey_to_dbkey { key, relation, field_name, type, fts_enabled }, + *fqkey_to_dbkey { key, relation, field_name, type, is_fts_enabled }, id = $node_id ", btmap! {"node_id".to_owned() => node_id.clone().into()}, ScriptMutability::Immutable, )?; + println!("FIELDS: {:?}", result); + todo!() } + + pub async fn create_node( + &self, + r#type: impl AsRef, + extra_data: Option, + ) -> Result { + let ty = r#type.as_ref(); + + let node_id = Uuid::now_v7(); + let node_id = node_id.to_string(); + + let tx = self.db.multi_transaction(true); + + let node_result = tx.run_script( + " + ?[id, type] <- [[$node_id, $type]] + :put node { id, type } + :returning + ", + btmap! { + "node_id".to_owned() => DataValue::from(node_id.clone()), + "type".to_owned() => DataValue::from(ty), + }, + )?; + + if let Some(extra_data) = extra_data { + if !extra_data.is_empty() { + let keys = extra_data.keys().map(|s| s.to_owned()).collect::>(); + let field_mapping = + self.get_rows_for_extra_keys(&tx, keys.as_slice())?; + + // Group the keys by which relation they're in + let result_by_relation = field_mapping.iter().into_group_map_by( + |(key, FieldInfo { relation_name, .. })| relation_name, + ); + + for (relation, fields) in result_by_relation.iter() { + let fields_mapping = fields + .into_iter() + .map( + |( + key, + FieldInfo { + relation_field, + r#type, + .. + }, + )| { + let new_value = extra_data.get(*key).unwrap(); + // TODO: Make this more generic + let new_value = match r#type.as_str() { + "int" => DataValue::from(new_value.as_i64().unwrap()), + _ => DataValue::from(new_value.as_str().unwrap()), + }; + (relation_field.to_owned(), new_value) + }, + ) + .collect::>(); + + let keys = fields_mapping.keys().collect::>(); + let keys_joined = keys.iter().join(", "); + + let query = format!( + " + ?[ node_id, {keys_joined} ] <- [$input_data] + :insert {relation} {{ node_id, {keys_joined} }} + " + ); + + let mut params = vec![]; + params.push(DataValue::from(node_id.clone())); + for key in keys { + params.push(fields_mapping[key].clone()); + } + + let result = tx.run_script( + &query, + btmap! { + "input_data".to_owned() => DataValue::List(params), + }, + )?; + } + + let input = DataValue::List( + keys + .iter() + .map(|s| { + DataValue::List(vec![ + DataValue::from(s.to_owned()), + DataValue::from(node_id.clone()), + ]) + }) + .collect_vec(), + ); + tx.run_script( + " + ?[key, id] <- $input_data + :put node_has_key { key, id } + ", + btmap! { + "input_data".to_owned() => input + }, + )?; + } + } + + tx.commit()?; + + let created_at = DateTime::from_timestamp_millis( + (node_result.rows[0][4].get_float().unwrap() * 1000.0) as i64, + ) + .unwrap(); + let updated_at = DateTime::from_timestamp_millis( + (node_result.rows[0][5].get_float().unwrap() * 1000.0) as i64, + ) + .unwrap(); + + Ok(NodeInfo { + node_id, + created_at, + updated_at, + fields: None, + }) + } + + pub fn get_rows_for_extra_keys( + &self, + tx: &MultiTransaction, + keys: &[String], + ) -> Result { + let result = tx.run_script( + " + ?[key, relation, field_name, type, is_fts_enabled] := + *fqkey_to_dbkey{key, relation, field_name, type, is_fts_enabled}, + is_in(key, $keys) + ", + btmap! { + "keys".to_owned() => DataValue::List( + keys.into_iter() + .map(|s| DataValue::from(s.as_str())) + .collect::>() + ), + }, + )?; + + let s = |s: &DataValue| s.get_str().unwrap().to_owned(); + + Ok( + result + .rows + .into_iter() + .map(|row| { + ( + s(&row[0]), + FieldInfo { + relation_name: s(&row[1]), + relation_field: s(&row[2]), + r#type: s(&row[3]), + is_fts_enabled: row[4].get_bool().unwrap(), + }, + ) + }) + .collect::>(), + ) + } } diff --git a/crates/panorama-core/src/tests/mod.rs b/crates/panorama-core/src/tests/mod.rs index 4b421f4..dc52c2a 100644 --- a/crates/panorama-core/src/tests/mod.rs +++ b/crates/panorama-core/src/tests/mod.rs @@ -2,18 +2,40 @@ use cozo::DbInstance; use miette::Result; use tantivy::Index; -use crate::{state::tantivy_schema, AppState}; +use crate::{migrations::run_migrations, state::tantivy_schema, AppState}; -pub fn test_state() -> Result { +pub async fn test_state() -> Result { let db = DbInstance::new("mem", "", "")?; let schema = tantivy_schema(); let tantivy_index = Index::create_in_ram(schema); - Ok(AppState { db, tantivy_index }) + + let state = AppState { db, tantivy_index }; + run_migrations(&state.db).await?; + + Ok(state) } -#[test] -pub fn test_create_node() -> Result<()> { - let state = test_state()?; +#[tokio::test] +pub async fn test_create_node() -> Result<()> { + let state = test_state().await?; + + let node_info = state + .create_node( + "panorama/journal/page", + Some(btmap! { + "panorama/journal/page/content".to_owned() => json!("helloge"), + }), + ) + .await?; + + println!( + "{}", + serde_json::to_string_pretty(&state.export().await.unwrap()).unwrap() + ); + + let node = state.get_node(node_info.node_id).await?; + + println!("node: {:?}", node); Ok(()) } diff --git a/crates/panorama-daemon/src/export.rs b/crates/panorama-daemon/src/export.rs index 6e2e7c6..5f0f508 100644 --- a/crates/panorama-daemon/src/export.rs +++ b/crates/panorama-daemon/src/export.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, - fs::{self, File}, - io::{BufWriter, Write}, + fs::{self}, + io::{Write}, path::PathBuf, }; diff --git a/crates/panorama-daemon/src/journal.rs b/crates/panorama-daemon/src/journal.rs index c9c681e..7bccbf4 100644 --- a/crates/panorama-daemon/src/journal.rs +++ b/crates/panorama-daemon/src/journal.rs @@ -1,11 +1,29 @@ -use axum::{extract::State, Json}; +use axum::{extract::State, routing::get, Json, Router}; use chrono::Local; use cozo::ScriptMutability; use serde_json::Value; +use utoipa::OpenApi; use uuid::Uuid; use crate::{error::AppResult, AppState}; +/// Node API +#[derive(OpenApi)] +#[openapi(paths(get_todays_journal_id), components(schemas()))] +pub(super) struct JournalApi; + +pub(super) fn router() -> Router { + Router::new() + .route("/journal/get_todays_journal_id", get(get_todays_journal_id)) +} + +#[utoipa::path( + get, + path = "/get_todays_journal_id", + responses( + (status = 200), + ), +)] pub async fn get_todays_journal_id( State(state): State, ) -> AppResult> { diff --git a/crates/panorama-daemon/src/main.rs b/crates/panorama-daemon/src/main.rs index 446277a..0cd3aa2 100644 --- a/crates/panorama-daemon/src/main.rs +++ b/crates/panorama-daemon/src/main.rs @@ -18,12 +18,11 @@ use std::fs; use axum::{ http::Method, - routing::{get, post, put}, + routing::{get, put}, Router, }; use miette::{IntoDiagnostic, Result}; use panorama_core::AppState; -use serde_json::Value; use tokio::net::TcpListener; use tower::ServiceBuilder; use tower_http::cors::{self, CorsLayer}; @@ -32,9 +31,8 @@ use utoipa_scalar::{Scalar, Servable}; use crate::{ export::export, - journal::get_todays_journal_id, mail::{get_mail, get_mail_config}, - node::{create_node, node_types, search_nodes, update_node}, + node::search_nodes, }; #[tokio::main] @@ -42,7 +40,10 @@ async fn main() -> Result<()> { #[derive(OpenApi)] #[openapi( modifiers(), - nest((path = "/node", api = crate::node::NodeApi)), + nest( + (path = "/journal", api = crate::journal::JournalApi), + (path = "/node", api = crate::node::NodeApi), + ), )] struct ApiDoc; @@ -59,20 +60,12 @@ async fn main() -> Result<()> { // build our application with a single route let app = Router::new() - // .merge( - // SwaggerUi::new("/swagger-ui") - // .url("/api-docs/openapi.json", ApiDoc::openapi()), - // ) .merge(Scalar::with_url("/api/docs", ApiDoc::openapi())) .route("/", get(|| async { "Hello, World!" })) .route("/export", get(export)) - .route("/node", put(create_node)) .route("/node/search", get(search_nodes)) - // .route("/node/:id", get(get_node)) - // .route("/node/:id", post(update_node)) - .route("/node/types", get(node_types)) .nest("/node", node::router().with_state(state.clone())) - .route("/journal/get_todays_journal_id", get(get_todays_journal_id)) + .nest("/journal", journal::router().with_state(state.clone())) .route("/mail/config", get(get_mail_config)) .route("/mail", get(get_mail)) .layer(ServiceBuilder::new().layer(cors)) diff --git a/crates/panorama-daemon/src/node.rs b/crates/panorama-daemon/src/node.rs index ce273c2..e2cf7ed 100644 --- a/crates/panorama-daemon/src/node.rs +++ b/crates/panorama-daemon/src/node.rs @@ -1,16 +1,14 @@ -use std::{ - collections::{BTreeMap, HashMap}, - result, -}; +use std::collections::{BTreeMap, HashMap}; use axum::{ extract::{Path, Query, State}, http::StatusCode, - routing::get, + routing::{get, post, put}, Json, Router, }; -use cozo::{DataValue, DbInstance, MultiTransaction, ScriptMutability, Vector}; +use cozo::{DataValue, MultiTransaction, ScriptMutability}; use itertools::Itertools; +use panorama_core::state::node::ExtraData; use serde_json::Value; use utoipa::{OpenApi, ToSchema}; use uuid::Uuid; @@ -19,11 +17,17 @@ use crate::{error::AppResult, AppState}; /// Node API #[derive(OpenApi)] -#[openapi(paths(get_node), components(schemas(GetNodeResult)))] +#[openapi( + paths(get_node, update_node, create_node), + components(schemas(GetNodeResult)) +)] pub(super) struct NodeApi; pub(super) fn router() -> Router { - Router::new().route("/:id", get(get_node)) + Router::new() + .route("/:id", get(get_node)) + .route("/:id", post(update_node)) + .route("/", put(create_node)) } #[derive(Serialize, Deserialize, ToSchema, Clone)] @@ -38,12 +42,15 @@ struct GetNodeResult { title: String, } -/// Get all info about a single node +/// Get node info +/// +/// This endpoint retrieves all the fields for a particular node #[utoipa::path( get, path = "/{id}", responses( - (status = 200, body = [GetNodeResult]) + (status = 200, body = [GetNodeResult]), + (status = 404, description = "the node ID provided was not found") ), params( ("id" = String, Path, description = "Node ID"), @@ -102,6 +109,7 @@ pub struct UpdateData { extra_data: Option, } +/// Update node info #[utoipa::path( post, path = "/{id}", @@ -181,14 +189,6 @@ pub async fn update_node( Ok(Json(json!({}))) } -pub async fn node_types() -> AppResult> { - Ok(Json(json!({ - "types": [ - { "id": "panorama/journal/page", "display": "Journal Entry" }, - ] - }))) -} - #[derive(Debug, Deserialize)] pub struct CreateNodeOpts { // TODO: Allow submitting a string @@ -198,6 +198,16 @@ pub struct CreateNodeOpts { extra_data: Option, } +#[utoipa::path( + put, + path = "/", + responses( + (status = 200) + ), + params( + ("id" = String, Path, description = "Node ID"), + ) +)] pub async fn create_node( State(state): State, Json(opts): Json, @@ -323,8 +333,6 @@ pub async fn search_nodes( }))) } -type ExtraData = HashMap; - fn get_rows_for_extra_keys( tx: &MultiTransaction, extra_data: &ExtraData,