From bcd2e9086bdea8d5a35f1ed87bfaca9951303457 Mon Sep 17 00:00:00 2001 From: Michael Zhang Date: Sun, 26 May 2024 21:56:12 -0500 Subject: [PATCH] add exporting --- Cargo.lock | 4 + crates/panorama-daemon/.gitignore | 1 + crates/panorama-daemon/src/export.rs | 82 +++++++++++++++++++++ crates/panorama-daemon/src/main.rs | 4 + crates/panorama-daemon/src/migrations.rs | 9 +++ crates/panorama-daemon/src/node.rs | 26 ++++++- crates/panorama-daemon/src/query_builder.rs | 2 + crates/panorama-macros/Cargo.toml | 9 +++ crates/panorama-macros/src/lib.rs | 1 + 9 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 crates/panorama-daemon/.gitignore create mode 100644 crates/panorama-daemon/src/export.rs create mode 100644 crates/panorama-daemon/src/query_builder.rs create mode 100644 crates/panorama-macros/Cargo.toml create mode 100644 crates/panorama-macros/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a1584bf..9d16270 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2790,6 +2790,10 @@ dependencies = [ "uuid", ] +[[package]] +name = "panorama-macros" +version = "0.1.0" + [[package]] name = "parking_lot" version = "0.12.3" diff --git a/crates/panorama-daemon/.gitignore b/crates/panorama-daemon/.gitignore new file mode 100644 index 0000000..1f5d640 --- /dev/null +++ b/crates/panorama-daemon/.gitignore @@ -0,0 +1 @@ +export \ No newline at end of file diff --git a/crates/panorama-daemon/src/export.rs b/crates/panorama-daemon/src/export.rs new file mode 100644 index 0000000..ac27c62 --- /dev/null +++ b/crates/panorama-daemon/src/export.rs @@ -0,0 +1,82 @@ +use std::{ + collections::HashMap, + fs::{self, File}, + io::{BufWriter, Write}, + path::PathBuf, +}; + +use axum::extract::State; +use cozo::ScriptMutability; + +use crate::{error::AppResult, AppState}; + +pub async fn export(State(state): State) -> AppResult<()> { + let result = state.db.run_script( + "::relations", + Default::default(), + ScriptMutability::Immutable, + )?; + + let name_index = result.headers.iter().position(|x| x == "name").unwrap(); + let relation_names = result + .rows + .into_iter() + .map(|row| row[name_index].get_str().unwrap().to_owned()) + .collect::>(); + + let mut relation_columns = HashMap::new(); + + for relation_name in relation_names.iter() { + let result = state.db.run_script( + &format!("::columns {relation_name}"), + Default::default(), + ScriptMutability::Immutable, + )?; + + let column_index = + result.headers.iter().position(|x| x == "column").unwrap(); + let columns = result + .rows + .into_iter() + .map(|row| row[column_index].get_str().unwrap().to_owned()) + .collect::>(); + + relation_columns.insert(relation_name.clone(), columns); + } + + println!("columns: {relation_columns:?}"); + + let base_dir = PathBuf::from("export"); + fs::create_dir_all(&base_dir); + + let tx = state.db.multi_transaction(false); + + for relation_name in relation_names.iter() { + let relation_path = base_dir.join(format!("{relation_name}.ndjson")); + let mut file = File::create(&relation_path).unwrap(); + + let columns = relation_columns + .get(relation_name.as_str()) + .unwrap() + .join(", "); + + let query = format!("?[{columns}] := *{relation_name} {{ {columns} }}"); + println!("Query: {query}"); + let result = tx.run_script(&query, Default::default())?; + + for row in result.rows.into_iter() { + let mut object = HashMap::new(); + + for (idx, col) in row.into_iter().enumerate() { + let row_name = result.headers[idx].clone(); + object.insert(row_name, col); + } + + let serialized = serde_json::to_string(&object).unwrap(); + file.write(serialized.as_bytes()); + file.write(b"\n"); + } + } + + Ok(()) +} diff --git a/crates/panorama-daemon/src/main.rs b/crates/panorama-daemon/src/main.rs index 10948ec..1475487 100644 --- a/crates/panorama-daemon/src/main.rs +++ b/crates/panorama-daemon/src/main.rs @@ -8,9 +8,11 @@ extern crate serde_json; extern crate sugars; mod error; +mod export; mod journal; mod migrations; mod node; +mod query_builder; use std::fs; @@ -27,6 +29,7 @@ use tower::ServiceBuilder; use tower_http::cors::{self, CorsLayer}; use crate::{ + export::export, journal::get_todays_journal_id, migrations::run_migrations, node::{get_node, node_types, update_node}, @@ -64,6 +67,7 @@ async fn main() -> Result<()> { // build our application with a single route let app = Router::new() .route("/", get(|| async { "Hello, World!" })) + .route("/export", get(export)) .route("/node/:id", get(get_node)) .route("/node/:id", post(update_node)) .route("/node/types", get(node_types)) diff --git a/crates/panorama-daemon/src/migrations.rs b/crates/panorama-daemon/src/migrations.rs index fa4fd86..2f584a9 100644 --- a/crates/panorama-daemon/src/migrations.rs +++ b/crates/panorama-daemon/src/migrations.rs @@ -117,6 +117,15 @@ fn migration_01(db: &DbInstance) -> Result<()> { { :create node_has_key { key: String => id: String } } { :create node_managed_by_app { node_id: String => app: String } } { :create node_refers_to { node_id: String => other_node_id: String } } + { + :create fqkey_to_dbkey { + key: String + => + relation: String, + field_name: String, + type: String, + } + } # Create journal type { :create journal { node_id: String => content: Json } } diff --git a/crates/panorama-daemon/src/node.rs b/crates/panorama-daemon/src/node.rs index 559f21d..061623c 100644 --- a/crates/panorama-daemon/src/node.rs +++ b/crates/panorama-daemon/src/node.rs @@ -5,7 +5,7 @@ use axum::{ http::StatusCode, Json, }; -use cozo::ScriptMutability; +use cozo::{DataValue, ScriptMutability}; use serde_json::Value; use crate::{error::AppResult, AppState}; @@ -70,6 +70,30 @@ pub async fn update_node( ) -> AppResult> { println!("Update data: {:?}", update_data); + let tx = state.db.multi_transaction(true); + + if let Some(extra_data) = update_data.extra_data { + let result = tx.run_script( + " + ?[relation, field_name, type] := + *fqkey_to_dbkey{key, relation, field_name, type}, + key = $key + ", + btmap! { + "key".to_owned() => DataValue::List( + extra_data + .keys() + .map(|s| DataValue::from(s.as_str())) + .collect::>() + ), + }, + )?; + + println!("Result: {result:?}"); + } + + tx.commit()?; + Ok(Json(json!({}))) } diff --git a/crates/panorama-daemon/src/query_builder.rs b/crates/panorama-daemon/src/query_builder.rs new file mode 100644 index 0000000..d843084 --- /dev/null +++ b/crates/panorama-daemon/src/query_builder.rs @@ -0,0 +1,2 @@ +#[derive(Default)] +pub struct QueryBuilder {} diff --git a/crates/panorama-macros/Cargo.toml b/crates/panorama-macros/Cargo.toml new file mode 100644 index 0000000..8a48040 --- /dev/null +++ b/crates/panorama-macros/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "panorama-macros" +version = "0.1.0" +edition = "2021" + +[lib] +proc-macro = true + +[dependencies] diff --git a/crates/panorama-macros/src/lib.rs b/crates/panorama-macros/src/lib.rs new file mode 100644 index 0000000..5ffe4f6 --- /dev/null +++ b/crates/panorama-macros/src/lib.rs @@ -0,0 +1 @@ +// TODO: derive named rows