create node test

This commit is contained in:
Michael Zhang 2024-06-03 01:02:45 -04:00
parent ae41b32313
commit 5c2a35935e
13 changed files with 353 additions and 55 deletions

1
Cargo.lock generated
View file

@ -3516,6 +3516,7 @@ dependencies = [
"chrono", "chrono",
"cozo", "cozo",
"futures", "futures",
"itertools 0.13.0",
"miette", "miette",
"serde", "serde",
"serde_json", "serde_json",

View file

@ -7,6 +7,7 @@ edition = "2021"
chrono = { version = "0.4.38", features = ["serde"] } chrono = { version = "0.4.38", features = ["serde"] }
cozo = { version = "0.7.6", features = ["storage-rocksdb"] } cozo = { version = "0.7.6", features = ["storage-rocksdb"] }
futures = "0.3.30" futures = "0.3.30"
itertools = "0.13.0"
miette = "5.5.0" miette = "5.5.0"
serde = { version = "1.0.203", features = ["derive"] } serde = { version = "1.0.203", features = ["derive"] }
serde_json = "1.0.117" serde_json = "1.0.117"

View file

@ -1,6 +1,8 @@
#[macro_use] #[macro_use]
extern crate serde; extern crate serde;
#[macro_use] #[macro_use]
extern crate serde_json;
#[macro_use]
extern crate sugars; extern crate sugars;
pub mod migrations; pub mod migrations;

View file

@ -116,6 +116,7 @@ fn migration_01(db: &DbInstance) -> Result<()> {
# Inverse mappings for easy querying # Inverse mappings for easy querying
{ :create node_has_key { key: String => id: String } } { :create node_has_key { key: String => id: String } }
{ ::index create node_has_key:inverse { id } }
{ :create node_managed_by_app { node_id: String => app: String } } { :create node_managed_by_app { node_id: String => app: String } }
{ :create node_refers_to { node_id: String => other_node_id: String } } { :create node_refers_to { node_id: String => other_node_id: String } }
{ {
@ -125,18 +126,18 @@ fn migration_01(db: &DbInstance) -> Result<()> {
relation: String, relation: String,
field_name: String, field_name: String,
type: String, type: String,
fts_enabled: Bool, is_fts_enabled: Bool,
} }
} }
{ {
?[key, relation, field_name, type, fts_enabled] <- [ ?[key, relation, field_name, type, is_fts_enabled] <- [
['panorama/journal/page/content', 'journal', 'content', 'string', true], ['panorama/journal/page/content', 'journal', 'content', 'string', true],
['panorama/mail/config/imap_hostname', 'mail_config', 'imap_hostname', 'string', false], ['panorama/mail/config/imap_hostname', 'mail_config', 'imap_hostname', 'string', false],
['panorama/mail/config/imap_port', 'mail_config', 'imap_port', 'int', false], ['panorama/mail/config/imap_port', 'mail_config', 'imap_port', 'int', false],
['panorama/mail/config/imap_username', 'mail_config', 'imap_username', 'string', false], ['panorama/mail/config/imap_username', 'mail_config', 'imap_username', 'string', false],
['panorama/mail/config/imap_password', 'mail_config', 'imap_password', 'string', false], ['panorama/mail/config/imap_password', 'mail_config', 'imap_password', 'string', false],
] ]
:put fqkey_to_dbkey { key, relation, field_name, type, fts_enabled } :put fqkey_to_dbkey { key, relation, field_name, type, is_fts_enabled }
} }
# Create journal type # Create journal type

View file

@ -0,0 +1,70 @@
use std::collections::HashMap;
use cozo::ScriptMutability;
use miette::Result;
use serde_json::Value;
use crate::AppState;
impl AppState {
pub async fn export(&self) -> Result<Value> {
let result = self.db.run_script(
"::relations",
Default::default(),
ScriptMutability::Immutable,
)?;
let name_index = result.headers.iter().position(|x| x == "name").unwrap();
let relation_names = result
.rows
.into_iter()
.map(|row| row[name_index].get_str().unwrap().to_owned())
.collect::<Vec<_>>();
let mut relation_columns = HashMap::new();
for relation_name in relation_names.iter() {
let result = self.db.run_script(
&format!("::columns {relation_name}"),
Default::default(),
ScriptMutability::Immutable,
)?;
let column_index =
result.headers.iter().position(|x| x == "column").unwrap();
let columns = result
.rows
.into_iter()
.map(|row| row[column_index].get_str().unwrap().to_owned())
.collect::<Vec<_>>();
relation_columns.insert(relation_name.clone(), columns);
}
let tx = self.db.multi_transaction(false);
let mut all_relations = hmap! {};
for relation_name in relation_names.iter() {
let mut relation_info = vec![];
let columns = relation_columns.get(relation_name.as_str()).unwrap();
let columns_str = columns.join(", ");
let query =
format!("?[{columns_str}] := *{relation_name} {{ {columns_str} }}");
let result = tx.run_script(&query, Default::default())?;
for row in result.rows.into_iter() {
let mut object = hmap! {};
row.into_iter().enumerate().for_each(|(idx, col)| {
object.insert(columns[idx].to_owned(), col);
});
relation_info.push(object);
}
all_relations.insert(relation_name.to_owned(), relation_info);
}
Ok(json!({"relations": all_relations}))
}
}

View file

@ -1,9 +1,8 @@
use std::{collections::HashMap, default, time::Duration}; use std::{collections::HashMap, time::Duration};
use cozo::{DataValue, DbInstance, JsonData, ScriptMutability}; use cozo::{DataValue, JsonData, ScriptMutability};
use futures::TryStreamExt; use futures::TryStreamExt;
use miette::{IntoDiagnostic, Result}; use miette::{IntoDiagnostic, Result};
use serde_json::Value;
use tokio::{net::TcpStream, time::sleep}; use tokio::{net::TcpStream, time::sleep};
use uuid::Uuid; use uuid::Uuid;

View file

@ -1,3 +1,4 @@
pub mod export;
pub mod mail; pub mod mail;
pub mod node; pub mod node;
@ -7,7 +8,7 @@ use cozo::DbInstance;
use miette::{IntoDiagnostic, Result}; use miette::{IntoDiagnostic, Result};
use tantivy::{ use tantivy::{
directory::MmapDirectory, directory::MmapDirectory,
schema::{self, Schema, STORED, STRING, TEXT}, schema::{Schema, STORED, STRING, TEXT},
Index, Index,
}; };

View file

@ -1,18 +1,33 @@
use std::collections::HashMap; use std::collections::{BTreeMap, HashMap};
use chrono::{DateTime, Utc}; use chrono::{DateTime, Utc};
use cozo::{DataValue, ScriptMutability}; use cozo::{DataValue, MultiTransaction, ScriptMutability};
use itertools::Itertools;
use miette::Result; use miette::Result;
use serde_json::Value;
use uuid::Uuid;
use crate::AppState; use crate::AppState;
pub type ExtraData = BTreeMap<String, Value>;
#[derive(Debug)]
pub struct NodeInfo { pub struct NodeInfo {
pub node_id: String, pub node_id: String,
pub created_at: DateTime<Utc>, pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>, pub updated_at: DateTime<Utc>,
pub fields: HashMap<String, DataValue>, pub fields: Option<HashMap<String, DataValue>>,
} }
pub struct FieldInfo {
pub relation_name: String,
pub relation_field: String,
pub r#type: String,
pub is_fts_enabled: bool,
}
pub type FieldMapping = HashMap<String, FieldInfo>;
impl AppState { impl AppState {
/// Get all properties of a node /// Get all properties of a node
pub async fn get_node(&self, node_id: impl AsRef<str>) -> Result<NodeInfo> { pub async fn get_node(&self, node_id: impl AsRef<str>) -> Result<NodeInfo> {
@ -20,15 +35,182 @@ impl AppState {
let result = self.db.run_script( let result = self.db.run_script(
" "
?[relation, field_name, type, fts_enabled] := ?[relation, field_name, type, is_fts_enabled] :=
*node_has_key { key, id }, *node_has_key { key, id },
*fqkey_to_dbkey { key, relation, field_name, type, fts_enabled }, *fqkey_to_dbkey { key, relation, field_name, type, is_fts_enabled },
id = $node_id id = $node_id
", ",
btmap! {"node_id".to_owned() => node_id.clone().into()}, btmap! {"node_id".to_owned() => node_id.clone().into()},
ScriptMutability::Immutable, ScriptMutability::Immutable,
)?; )?;
println!("FIELDS: {:?}", result);
todo!() todo!()
} }
pub async fn create_node(
&self,
r#type: impl AsRef<str>,
extra_data: Option<ExtraData>,
) -> Result<NodeInfo> {
let ty = r#type.as_ref();
let node_id = Uuid::now_v7();
let node_id = node_id.to_string();
let tx = self.db.multi_transaction(true);
let node_result = tx.run_script(
"
?[id, type] <- [[$node_id, $type]]
:put node { id, type }
:returning
",
btmap! {
"node_id".to_owned() => DataValue::from(node_id.clone()),
"type".to_owned() => DataValue::from(ty),
},
)?;
if let Some(extra_data) = extra_data {
if !extra_data.is_empty() {
let keys = extra_data.keys().map(|s| s.to_owned()).collect::<Vec<_>>();
let field_mapping =
self.get_rows_for_extra_keys(&tx, keys.as_slice())?;
// Group the keys by which relation they're in
let result_by_relation = field_mapping.iter().into_group_map_by(
|(key, FieldInfo { relation_name, .. })| relation_name,
);
for (relation, fields) in result_by_relation.iter() {
let fields_mapping = fields
.into_iter()
.map(
|(
key,
FieldInfo {
relation_field,
r#type,
..
},
)| {
let new_value = extra_data.get(*key).unwrap();
// TODO: Make this more generic
let new_value = match r#type.as_str() {
"int" => DataValue::from(new_value.as_i64().unwrap()),
_ => DataValue::from(new_value.as_str().unwrap()),
};
(relation_field.to_owned(), new_value)
},
)
.collect::<BTreeMap<_, _>>();
let keys = fields_mapping.keys().collect::<Vec<_>>();
let keys_joined = keys.iter().join(", ");
let query = format!(
"
?[ node_id, {keys_joined} ] <- [$input_data]
:insert {relation} {{ node_id, {keys_joined} }}
"
);
let mut params = vec![];
params.push(DataValue::from(node_id.clone()));
for key in keys {
params.push(fields_mapping[key].clone());
}
let result = tx.run_script(
&query,
btmap! {
"input_data".to_owned() => DataValue::List(params),
},
)?;
}
let input = DataValue::List(
keys
.iter()
.map(|s| {
DataValue::List(vec![
DataValue::from(s.to_owned()),
DataValue::from(node_id.clone()),
])
})
.collect_vec(),
);
tx.run_script(
"
?[key, id] <- $input_data
:put node_has_key { key, id }
",
btmap! {
"input_data".to_owned() => input
},
)?;
}
}
tx.commit()?;
let created_at = DateTime::from_timestamp_millis(
(node_result.rows[0][4].get_float().unwrap() * 1000.0) as i64,
)
.unwrap();
let updated_at = DateTime::from_timestamp_millis(
(node_result.rows[0][5].get_float().unwrap() * 1000.0) as i64,
)
.unwrap();
Ok(NodeInfo {
node_id,
created_at,
updated_at,
fields: None,
})
}
pub fn get_rows_for_extra_keys(
&self,
tx: &MultiTransaction,
keys: &[String],
) -> Result<FieldMapping> {
let result = tx.run_script(
"
?[key, relation, field_name, type, is_fts_enabled] :=
*fqkey_to_dbkey{key, relation, field_name, type, is_fts_enabled},
is_in(key, $keys)
",
btmap! {
"keys".to_owned() => DataValue::List(
keys.into_iter()
.map(|s| DataValue::from(s.as_str()))
.collect::<Vec<_>>()
),
},
)?;
let s = |s: &DataValue| s.get_str().unwrap().to_owned();
Ok(
result
.rows
.into_iter()
.map(|row| {
(
s(&row[0]),
FieldInfo {
relation_name: s(&row[1]),
relation_field: s(&row[2]),
r#type: s(&row[3]),
is_fts_enabled: row[4].get_bool().unwrap(),
},
)
})
.collect::<HashMap<_, _>>(),
)
}
} }

View file

@ -2,18 +2,40 @@ use cozo::DbInstance;
use miette::Result; use miette::Result;
use tantivy::Index; use tantivy::Index;
use crate::{state::tantivy_schema, AppState}; use crate::{migrations::run_migrations, state::tantivy_schema, AppState};
pub fn test_state() -> Result<AppState> { pub async fn test_state() -> Result<AppState> {
let db = DbInstance::new("mem", "", "")?; let db = DbInstance::new("mem", "", "")?;
let schema = tantivy_schema(); let schema = tantivy_schema();
let tantivy_index = Index::create_in_ram(schema); let tantivy_index = Index::create_in_ram(schema);
Ok(AppState { db, tantivy_index })
let state = AppState { db, tantivy_index };
run_migrations(&state.db).await?;
Ok(state)
} }
#[test] #[tokio::test]
pub fn test_create_node() -> Result<()> { pub async fn test_create_node() -> Result<()> {
let state = test_state()?; let state = test_state().await?;
let node_info = state
.create_node(
"panorama/journal/page",
Some(btmap! {
"panorama/journal/page/content".to_owned() => json!("helloge"),
}),
)
.await?;
println!(
"{}",
serde_json::to_string_pretty(&state.export().await.unwrap()).unwrap()
);
let node = state.get_node(node_info.node_id).await?;
println!("node: {:?}", node);
Ok(()) Ok(())
} }

View file

@ -1,7 +1,7 @@
use std::{ use std::{
collections::HashMap, collections::HashMap,
fs::{self, File}, fs::{self},
io::{BufWriter, Write}, io::{Write},
path::PathBuf, path::PathBuf,
}; };

View file

@ -1,11 +1,29 @@
use axum::{extract::State, Json}; use axum::{extract::State, routing::get, Json, Router};
use chrono::Local; use chrono::Local;
use cozo::ScriptMutability; use cozo::ScriptMutability;
use serde_json::Value; use serde_json::Value;
use utoipa::OpenApi;
use uuid::Uuid; use uuid::Uuid;
use crate::{error::AppResult, AppState}; use crate::{error::AppResult, AppState};
/// Node API
#[derive(OpenApi)]
#[openapi(paths(get_todays_journal_id), components(schemas()))]
pub(super) struct JournalApi;
pub(super) fn router() -> Router<AppState> {
Router::new()
.route("/journal/get_todays_journal_id", get(get_todays_journal_id))
}
#[utoipa::path(
get,
path = "/get_todays_journal_id",
responses(
(status = 200),
),
)]
pub async fn get_todays_journal_id( pub async fn get_todays_journal_id(
State(state): State<AppState>, State(state): State<AppState>,
) -> AppResult<Json<Value>> { ) -> AppResult<Json<Value>> {

View file

@ -18,12 +18,11 @@ use std::fs;
use axum::{ use axum::{
http::Method, http::Method,
routing::{get, post, put}, routing::{get, put},
Router, Router,
}; };
use miette::{IntoDiagnostic, Result}; use miette::{IntoDiagnostic, Result};
use panorama_core::AppState; use panorama_core::AppState;
use serde_json::Value;
use tokio::net::TcpListener; use tokio::net::TcpListener;
use tower::ServiceBuilder; use tower::ServiceBuilder;
use tower_http::cors::{self, CorsLayer}; use tower_http::cors::{self, CorsLayer};
@ -32,9 +31,8 @@ use utoipa_scalar::{Scalar, Servable};
use crate::{ use crate::{
export::export, export::export,
journal::get_todays_journal_id,
mail::{get_mail, get_mail_config}, mail::{get_mail, get_mail_config},
node::{create_node, node_types, search_nodes, update_node}, node::search_nodes,
}; };
#[tokio::main] #[tokio::main]
@ -42,7 +40,10 @@ async fn main() -> Result<()> {
#[derive(OpenApi)] #[derive(OpenApi)]
#[openapi( #[openapi(
modifiers(), modifiers(),
nest((path = "/node", api = crate::node::NodeApi)), nest(
(path = "/journal", api = crate::journal::JournalApi),
(path = "/node", api = crate::node::NodeApi),
),
)] )]
struct ApiDoc; struct ApiDoc;
@ -59,20 +60,12 @@ async fn main() -> Result<()> {
// build our application with a single route // build our application with a single route
let app = Router::new() let app = Router::new()
// .merge(
// SwaggerUi::new("/swagger-ui")
// .url("/api-docs/openapi.json", ApiDoc::openapi()),
// )
.merge(Scalar::with_url("/api/docs", ApiDoc::openapi())) .merge(Scalar::with_url("/api/docs", ApiDoc::openapi()))
.route("/", get(|| async { "Hello, World!" })) .route("/", get(|| async { "Hello, World!" }))
.route("/export", get(export)) .route("/export", get(export))
.route("/node", put(create_node))
.route("/node/search", get(search_nodes)) .route("/node/search", get(search_nodes))
// .route("/node/:id", get(get_node))
// .route("/node/:id", post(update_node))
.route("/node/types", get(node_types))
.nest("/node", node::router().with_state(state.clone())) .nest("/node", node::router().with_state(state.clone()))
.route("/journal/get_todays_journal_id", get(get_todays_journal_id)) .nest("/journal", journal::router().with_state(state.clone()))
.route("/mail/config", get(get_mail_config)) .route("/mail/config", get(get_mail_config))
.route("/mail", get(get_mail)) .route("/mail", get(get_mail))
.layer(ServiceBuilder::new().layer(cors)) .layer(ServiceBuilder::new().layer(cors))

View file

@ -1,16 +1,14 @@
use std::{ use std::collections::{BTreeMap, HashMap};
collections::{BTreeMap, HashMap},
result,
};
use axum::{ use axum::{
extract::{Path, Query, State}, extract::{Path, Query, State},
http::StatusCode, http::StatusCode,
routing::get, routing::{get, post, put},
Json, Router, Json, Router,
}; };
use cozo::{DataValue, DbInstance, MultiTransaction, ScriptMutability, Vector}; use cozo::{DataValue, MultiTransaction, ScriptMutability};
use itertools::Itertools; use itertools::Itertools;
use panorama_core::state::node::ExtraData;
use serde_json::Value; use serde_json::Value;
use utoipa::{OpenApi, ToSchema}; use utoipa::{OpenApi, ToSchema};
use uuid::Uuid; use uuid::Uuid;
@ -19,11 +17,17 @@ use crate::{error::AppResult, AppState};
/// Node API /// Node API
#[derive(OpenApi)] #[derive(OpenApi)]
#[openapi(paths(get_node), components(schemas(GetNodeResult)))] #[openapi(
paths(get_node, update_node, create_node),
components(schemas(GetNodeResult))
)]
pub(super) struct NodeApi; pub(super) struct NodeApi;
pub(super) fn router() -> Router<AppState> { pub(super) fn router() -> Router<AppState> {
Router::new().route("/:id", get(get_node)) Router::new()
.route("/:id", get(get_node))
.route("/:id", post(update_node))
.route("/", put(create_node))
} }
#[derive(Serialize, Deserialize, ToSchema, Clone)] #[derive(Serialize, Deserialize, ToSchema, Clone)]
@ -38,12 +42,15 @@ struct GetNodeResult {
title: String, title: String,
} }
/// Get all info about a single node /// Get node info
///
/// This endpoint retrieves all the fields for a particular node
#[utoipa::path( #[utoipa::path(
get, get,
path = "/{id}", path = "/{id}",
responses( responses(
(status = 200, body = [GetNodeResult]) (status = 200, body = [GetNodeResult]),
(status = 404, description = "the node ID provided was not found")
), ),
params( params(
("id" = String, Path, description = "Node ID"), ("id" = String, Path, description = "Node ID"),
@ -102,6 +109,7 @@ pub struct UpdateData {
extra_data: Option<ExtraData>, extra_data: Option<ExtraData>,
} }
/// Update node info
#[utoipa::path( #[utoipa::path(
post, post,
path = "/{id}", path = "/{id}",
@ -181,14 +189,6 @@ pub async fn update_node(
Ok(Json(json!({}))) Ok(Json(json!({})))
} }
pub async fn node_types() -> AppResult<Json<Value>> {
Ok(Json(json!({
"types": [
{ "id": "panorama/journal/page", "display": "Journal Entry" },
]
})))
}
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct CreateNodeOpts { pub struct CreateNodeOpts {
// TODO: Allow submitting a string // TODO: Allow submitting a string
@ -198,6 +198,16 @@ pub struct CreateNodeOpts {
extra_data: Option<ExtraData>, extra_data: Option<ExtraData>,
} }
#[utoipa::path(
put,
path = "/",
responses(
(status = 200)
),
params(
("id" = String, Path, description = "Node ID"),
)
)]
pub async fn create_node( pub async fn create_node(
State(state): State<AppState>, State(state): State<AppState>,
Json(opts): Json<CreateNodeOpts>, Json(opts): Json<CreateNodeOpts>,
@ -323,8 +333,6 @@ pub async fn search_nodes(
}))) })))
} }
type ExtraData = HashMap<String, Value>;
fn get_rows_for_extra_keys( fn get_rows_for_extra_keys(
tx: &MultiTransaction, tx: &MultiTransaction,
extra_data: &ExtraData, extra_data: &ExtraData,