michael/panorama
1
0
Fork 0
Code Issues 11 Pull requests Projects 1 Releases Wiki Activity

compiles again

Browse source
This commit is contained in:
Michael Zhang 2024-06-18 16:28:15 -05:00
parent f771a7d20f
commit 3a2eda20cb
4 changed files with 209 additions and 178 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -3,4 +3,4 @@ dist
target target
.DS_Store .DS_Store
**/export/export.json **/export/export.json
test.db test.db*

2
crates/panorama-core/migrations/00001_initial.sql
View file

@ -1,7 +1,7 @@
CREATE TABLE node ( CREATE TABLE node (
node_id TEXT PRIMARY KEY, node_id TEXT PRIMARY KEY,
node_type TEXT NOT NULL, node_type TEXT NOT NULL,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at INTEGER NOT NULL DEFAULT CURRENT_TIMESTAMP,
extra_data JSON extra_data JSON
); );

371
crates/panorama-core/src/state/node.rs
View file

@ -1,6 +1,9 @@
use std::collections::{BTreeMap, HashMap}; use std::{
collections::{BTreeMap, HashMap},
str::FromStr,
};
use chrono::{DateTime, Utc}; use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
use itertools::Itertools; use itertools::Itertools;
use miette::{bail, Context, Error, IntoDiagnostic, Report, Result}; use miette::{bail, Context, Error, IntoDiagnostic, Report, Result};
use serde_json::Value; use serde_json::Value;
@ -8,7 +11,11 @@ use sqlx::{
query::Query, sqlite::SqliteArguments, Acquire, Connection, Executor, query::Query, sqlite::SqliteArguments, Acquire, Connection, Executor,
FromRow, QueryBuilder, Sqlite, FromRow, QueryBuilder, Sqlite,
}; };
use tantivy::schema::{OwnedValue, Value as _}; use tantivy::{
schema::{OwnedValue, Value as _},
time::Date,
Term,
};
use uuid::Uuid; use uuid::Uuid;
use crate::{state::node_raw::FieldMappingRow, AppState, NodeId}; use crate::{state::node_raw::FieldMappingRow, AppState, NodeId};
@ -16,6 +23,8 @@ use crate::{state::node_raw::FieldMappingRow, AppState, NodeId};
// use super::utils::owned_value_to_json_value; // use super::utils::owned_value_to_json_value;
pub type ExtraData = BTreeMap<String, Value>; pub type ExtraData = BTreeMap<String, Value>;
pub type FieldsByTable<'a> =
HashMap<(&'a i64, &'a String), Vec<&'a FieldMappingRow>>;
#[derive(Debug)] #[derive(Debug)]
pub struct NodeInfo { pub struct NodeInfo {
@ -99,7 +108,7 @@ impl AppState {
async fn query_related_fields<'e, 'c: 'e, X>( async fn query_related_fields<'e, 'c: 'e, X>(
x: X, x: X,
fields_by_table: &HashMap<(&i64, &String), Vec<&FieldMappingRow>>, fields_by_table: &FieldsByTable<'_>,
) -> sqlx::Result<HashMap<String, Value>> ) -> sqlx::Result<HashMap<String, Value>>
where where
X: 'e + Executor<'c, Database = Sqlite>, X: 'e + Executor<'c, Database = Sqlite>,
@ -186,193 +195,215 @@ pub enum CreateOrUpdate {
impl AppState { impl AppState {
// TODO: Split this out into create and update // TODO: Split this out into create and update
// pub async fn create_or_update_node( pub async fn create_or_update_node(
// &self, &self,
// opts: CreateOrUpdate, opts: CreateOrUpdate,
// extra_data: Option<ExtraData>, extra_data: Option<ExtraData>,
// ) -> Result<NodeInfo> { ) -> Result<NodeInfo> {
// let node_id = match opts { let node_id = match opts {
// CreateOrUpdate::Create { .. } => NodeId(Uuid::now_v7()), CreateOrUpdate::Create { .. } => NodeId(Uuid::now_v7()),
// CreateOrUpdate::Update { ref node_id } => node_id.clone(), CreateOrUpdate::Update { ref node_id } => node_id.clone(),
// }; };
// let node_id = node_id.to_string(); let node_id = node_id.to_string();
// let action = match opts { let action = match opts {
// CreateOrUpdate::Create { .. } => "put", CreateOrUpdate::Create { .. } => "put",
// CreateOrUpdate::Update { .. } => "update", CreateOrUpdate::Update { .. } => "update",
// }; };
// println!("Request: {opts:?} {extra_data:?}"); println!("Request: {opts:?} {extra_data:?}");
// let (created_at, updated_at) = match opts { let mut conn = self.conn().await?;
// CreateOrUpdate::Create { ref r#type } => {
// let node_result = tx.run_script(
// "
// ?[id, type] <- [[$node_id, $type]]
// :put node { id, type }
// :returning
// ",
// btmap! {
// "node_id".to_owned() => DataValue::from(node_id.clone()),
// "type".to_owned() => DataValue::from(r#type.to_owned()),
// },
// )?;
// let created_at = DateTime::from_timestamp_millis(
// (node_result.rows[0][3].get_float().unwrap() * 1000.0) as i64,
// )
// .unwrap();
// let updated_at = DateTime::from_timestamp_millis(
// (node_result.rows[0][4].get_float().unwrap() * 1000.0) as i64,
// )
// .unwrap();
// (created_at, updated_at)
// }
// CreateOrUpdate::Update { .. } => {
// let node_result = tx.run_script(
// "
// ?[id, type, created_at, updated_at] := *node { id, type, created_at, updated_at },
// id = $node_id
// ",
// btmap! {
// "node_id".to_owned() => DataValue::from(node_id.clone()),
// },
// )?;
// let created_at = DateTime::from_timestamp_millis(
// (node_result.rows[0][2].get_float().unwrap() * 1000.0) as i64,
// )
// .unwrap();
// let updated_at = DateTime::from_timestamp_millis(
// (node_result.rows[0][3].get_float().unwrap() * 1000.0) as i64,
// )
// .unwrap();
// (created_at, updated_at)
// }
// };
// if let Some(extra_data) = extra_data { conn
// let node_id_field = self .transaction::<_, _, sqlx::Error>(|tx| {
// .tantivy_field_map Box::pin(async move {
// .get_by_left("node_id") let node_info = match opts {
// .unwrap() CreateOrUpdate::Create { r#type } => {
// .clone(); AppState::create_node_raw(&mut **tx, &r#type).await?
}
CreateOrUpdate::Update { node_id } => todo!(),
};
// if !extra_data.is_empty() { if let Some(extra_data) = extra_data {
// let keys = extra_data.keys().map(|s| s.to_owned()).collect::<Vec<_>>(); if !extra_data.is_empty() {
// let field_mapping = let node_id_str = node_id.to_string();
// self.get_rows_for_extra_keys(&tx, keys.as_slice())?; let field_mapping = AppState::get_related_field_list_for_node_id(
&mut **tx,
&node_id_str,
)
.await?;
// // Group the keys by which relation they're in // Group the keys by which relation they're in
// let result_by_relation = field_mapping.iter().into_group_map_by( let fields_by_table = field_mapping.iter().into_group_map_by(
// |(_, FieldInfo { relation_name, .. })| relation_name, |FieldMappingRow {
// ); app_id,
app_table_name,
..
}| (app_id, app_table_name),
);
// for (relation, fields) in result_by_relation.iter() { AppState::write_extra_data(
// let mut doc = btmap! { node_id_field.clone() => OwnedValue::Str(node_id.to_owned()) }; &mut **tx,
// let fields_mapping = fields &node_id_str,
// .into_iter() &fields_by_table,
// .map( extra_data,
// |( )
// key, .await?;
// FieldInfo { }
// relation_field, }
// r#type,
// is_fts_enabled,
// ..
// },
// )| {
// let new_value = extra_data.get(*key).unwrap();
// // TODO: Make this more generic Ok(node_info)
// let new_value = match r#type.as_str() { })
// "int" => DataValue::from(new_value.as_i64().unwrap()), })
// _ => DataValue::from(new_value.as_str().unwrap()), .await
// }; .into_diagnostic()
}
// if *is_fts_enabled { async fn create_node_raw<'e, 'c: 'e, X>(
// if let Some(field) = self.tantivy_field_map.get_by_left(*key) x: X,
// { r#type: &str,
// doc.insert( ) -> sqlx::Result<NodeInfo>
// field.clone(), where
// OwnedValue::Str(new_value.get_str().unwrap().to_owned()), X: 'e + Executor<'c, Database = Sqlite>,
// ); {
// } let node_id = Uuid::now_v7();
// } let node_id_str = node_id.to_string();
// (relation_field.to_owned(), new_value) #[derive(FromRow)]
// }, struct Result {
// ) updated_at: i64,
// .collect::<BTreeMap<_, _>>(); }
// let mut writer = let result = sqlx::query_as!(
// self.tantivy_index.writer(15_000_000).into_diagnostic()?; Result,
r#"
INSERT INTO node (node_id, node_type, extra_data)
VALUES (?, ?, "{}")
RETURNING updated_at
"#,
node_id_str,
r#type,
)
.fetch_one(x)
.await?;
// let delete_term = let updated_at =
// Term::from_field_text(node_id_field.clone(), &node_id); DateTime::from_timestamp_millis(result.updated_at * 1000).unwrap();
// writer.delete_term(delete_term); let created_at = DateTime::from_timestamp_millis(
node_id.get_timestamp().unwrap().to_unix().0 as i64 * 1000,
)
.unwrap();
// writer.add_document(doc).into_diagnostic()?; Ok(NodeInfo {
// writer.commit().into_diagnostic()?; node_id: NodeId(node_id),
// drop(writer); created_at,
updated_at,
fields: None,
})
}
// let keys = fields_mapping.keys().collect::<Vec<_>>(); async fn write_extra_data<'e, 'c: 'e, X>(
// let keys_joined = keys.iter().join(", "); x: X,
node_id: &str,
fields_by_table: &FieldsByTable<'_>,
extra_data: ExtraData,
) -> sqlx::Result<()>
where
X: 'e + Executor<'c, Database = Sqlite>,
{
// Update Tantivy indexes
// for ((app_id, app_table_name), fields) in fields_by_table.iter() {
// let mut writer =
// self.tantivy_index.writer(15_000_000).into_diagnostic()?;
// if !keys.is_empty() { // let delete_term = Term::from_field_text(node_id_field.clone(), &node_id);
// let query = format!( // writer.delete_term(delete_term);
// "
// ?[ node_id, {keys_joined} ] <- [$input_data]
// :{action} {relation} {{ node_id, {keys_joined} }}
// "
// );
// let mut params = vec![]; // writer.add_document(doc).into_diagnostic()?;
// params.push(DataValue::from(node_id.clone())); // writer.commit().into_diagnostic()?;
// for key in keys { // drop(writer);
// params.push(fields_mapping[key].clone()); // }
// }
// let result = tx.run_script( // Update database
// &query, let mut node_has_keys = Vec::new();
// btmap! { for ((app_id, app_table_name), fields) in fields_by_table.iter() {
// "input_data".to_owned() => DataValue::List(params), for field_info in fields {
// }, node_has_keys.push(&field_info.full_key);
// ); }
// }
// }
// let input = DataValue::List( // let mut doc =
// keys // btmap! { node_id_field.clone() => OwnedValue::Str(node_id.to_owned()) };
// .iter() // let fields_mapping = fields
// .map(|s| { // .into_iter()
// DataValue::List(vec![ // .map(
// DataValue::from(s.to_owned()), // |(
// DataValue::from(node_id.clone()), // key,
// ]) // FieldInfo {
// }) // relation_field,
// .collect_vec(), // r#type,
// ); // is_fts_enabled,
// ..
// },
// )| {
// let new_value = extra_data.get(*key).unwrap();
// tx.run_script( // // TODO: Make this more generic
// " // let new_value = match r#type.as_str() {
// ?[key, id] <- $input_data // "int" => DataValue::from(new_value.as_i64().unwrap()),
// :put node_has_key { key, id } // _ => DataValue::from(new_value.as_str().unwrap()),
// ", // };
// btmap! {
// "input_data".to_owned() => input
// },
// )?;
// }
// }
// tx.commit()?; // if *is_fts_enabled {
// if let Some(field) = self.tantivy_field_map.get_by_left(*key) {
// doc.insert(
// field.clone(),
// OwnedValue::Str(new_value.get_str().unwrap().to_owned()),
// );
// }
// }
// Ok(NodeInfo { // (relation_field.to_owned(), new_value)
// node_id: NodeId(Uuid::from_str(&node_id).unwrap()), // },
// created_at, // )
// updated_at, // .collect::<BTreeMap<_, _>>();
// fields: None,
// }) // let keys = fields_mapping.keys().collect::<Vec<_>>();
// } // let keys_joined = keys.iter().join(", ");
// if !keys.is_empty() {
// let query = format!(
// "
// ?[ node_id, {keys_joined} ] <- [$input_data]
// :{action} {relation} {{ node_id, {keys_joined} }}
// "
// );
// let mut params = vec![];
// params.push(DataValue::from(node_id.clone()));
// for key in keys {
// params.push(fields_mapping[key].clone());
// }
// let result = tx.run_script(
// &query,
// btmap! {
// "input_data".to_owned() => DataValue::List(params),
// },
// );
// }
}
let mut query =
QueryBuilder::new("INSERT INTO node_has_key (node_id, full_key) VALUES ");
query.push_values(node_has_keys, |mut b, key| {
b.push_bind(node_id).push_bind(key);
});
println!("Query: {:?}", query.sql());
query.build().execute(x).await?;
Ok(())
}
} }
// impl AppState { // impl AppState {

12
crates/panorama-core/src/state/node_raw.rs
View file

@ -4,15 +4,15 @@ use crate::AppState;
#[derive(FromRow)] #[derive(FromRow)]
pub struct FieldMappingRow { pub struct FieldMappingRow {
pub(crate) full_key: String, pub full_key: String,
pub(crate) app_id: i64, pub app_id: i64,
pub(crate) app_table_name: String, pub app_table_name: String,
pub(crate) app_table_field: String, pub app_table_field: String,
pub(crate) db_table_name: Option<String>, pub db_table_name: Option<String>,
} }
impl AppState { impl AppState {
pub async fn get_related_field_list_for_node_id<'e, 'c: 'e, X>( pub(crate) async fn get_related_field_list_for_node_id<'e, 'c: 'e, X>(
x: X, x: X,
node_id: &str, node_id: &str,
) -> sqlx::Result<Vec<FieldMappingRow>> ) -> sqlx::Result<Vec<FieldMappingRow>>