download charts

lib/chartDownloader/chartDownloader.worker.ts

@@ -1,10 +1,69 @@
+import { APP_DATA_VERSION, dbStatusAtom, jotaiStore } from "../../src/globals";
+import stepchartsUrl from "../../data/stepData.ndjson?url";
 import { RpcProvider } from "worker-rpc";
+import ndjsonStream from "../ndjsonStream";
 
 async function init() {
   const rpcProvider = new RpcProvider((message, transfer) =>
     self.postMessage(message, undefined, transfer),
   );
   self.addEventListener("message", (evt) => rpcProvider.dispatch(evt.data));
+
+  rpcProvider.registerSignalHandler("start", async () => {
+    const result = await rpcProvider.rpc("db", {
+      cmd: "exec",
+      s: "SELECT version FROM _appDataVersion",
+    });
+    const appDataVersion = result[0].values[0][0];
+    if (appDataVersion === APP_DATA_VERSION) {
+      console.log(`data version is up to date! (${appDataVersion})`);
+      return;
+    }
+
+    console.log(
+      `outdated data version ${appDataVersion} < ${APP_DATA_VERSION}`,
+    );
+    await rpcProvider.rpc("db", {
+      cmd: "run",
+      s: "DELETE FROM charts",
+    });
+
+    const response = await fetch(stepchartsUrl);
+    const stream = ndjsonStream<ChartData>(response.body);
+    const reader = stream.getReader();
+    const iter = iterStream(reader);
+
+    // TODO: Actually stream this into the DB somehow?
+    // For slow connections where this download process actually takes quite a bit
+    const lol = [];
+    for await (const thing of iter) {
+      lol.push(thing);
+    }
+    console.log(lol);
+    console.log(`got ${lol.length} entries. inserting...`);
+    const h = await rpcProvider.rpc("db", {
+      cmd: "bulkInsert",
+      s: `
+        INSERT INTO charts
+          (artist, title, difficulty)
+        VALUES ($artist, $title, $difficulty)
+      `,
+      data: lol,
+    });
+
+    jotaiStore.set(dbStatusAtom, (prev) => ({
+      ...prev,
+      lastUpdated: new Date().getTime(),
+    }));
+  });
+}
+
+async function* iterStream(reader: ReadableStreamReader<ChartData>) {
+  let result;
+  while (!result || !result.done) {
+    result = await reader.read();
+    if (result.value) yield result.value;
+  }
 }
 
 init();

lib/chartDownloader/client.ts

@@ -1,5 +1,7 @@
 import { RpcProvider } from "worker-rpc";
 import ChartDownloaderWorker from "./chartDownloader.worker?worker";
+import { dbClient, getDbClient } from "../db/client";
+import type { BindParams } from "sql.js";
 
 const worker = new ChartDownloaderWorker();
 const rpcProvider = new RpcProvider((message, transfer) =>
@@ -8,6 +10,16 @@ const rpcProvider = new RpcProvider((message, transfer) =>
 worker.onmessage = (e) => rpcProvider.dispatch(e.data);
 
 export const chartDownloaderEvent = new Event("startDownloadingCharts");
-document.addEventListener("startDownloadingCharts", () => {
+
-  console.log("SHIET");
+document.addEventListener("startDownloadingCharts", async () => {
+  rpcProvider.registerRpcHandler("db", async ({ cmd, ...args }) =>
+    dbClient.rpc(cmd, args),
+  );
+
+  rpcProvider.signal("start");
 });
+
+interface Args {
+  s: string;
+  p?: BindParams;
+}

lib/db/client.ts

@@ -2,6 +2,7 @@ import type { BindParams, Database, QueryExecResult } from "sql.js";
 import { RpcProvider } from "worker-rpc";
 import DbWorker from "./db.worker?worker";
 import { dbStatusAtom, jotaiStore } from "../../src/globals";
+import { DbStatusCode } from "./constants";
 
 const worker = new DbWorker();
 const rpcProvider = new RpcProvider((message, transfer) =>
@@ -16,13 +17,14 @@ rpcProvider.registerSignalHandler("db", ({ status }) => {
 interface AsyncDatabase {
   exec(sql: string, params?: BindParams): Promise<QueryExecResult[]>;
   run(sql: string, params?: BindParams): Promise<Database>;
+  rpc(cmd: string, args: any): Promise<any>;
 }
 
 export let dbClient: AsyncDatabase;
 
 export async function getDbClient() {
   await new Promise<void>((resolve) => {
-    rpcProvider.registerSignalHandler("ready", () => {
+    rpcProvider.registerSignalHandler(DbStatusCode.READY, () => {
       resolve();
     });
   });
@@ -34,5 +36,10 @@ export async function getDbClient() {
     async exec(s: string, p?: BindParams): Promise<QueryExecResult[]> {
       return await rpcProvider.rpc("exec", { s, p });
     },
+    async rpc(cmd, args): Promise<any> {
+      return await rpcProvider.rpc(cmd, args);
+    },
   };
+
+  console.log("done :3");
 }

lib/db/db.worker.ts

@@ -2,7 +2,7 @@ import sqlWasmUrl from "@jlongster/sql.js/dist/sql-wasm.wasm?url";
 import { MigrateDeploy } from "@prisma/migrate";
 import initSqlJs from "@jlongster/sql.js";
 import { SQLiteFS } from "absurd-sql";
-import type { BindParams, Database } from "sql.js";
+import type { BindParams, Database, Statement } from "sql.js";
 import IndexedDBBackend from "absurd-sql/dist/indexeddb-backend";
 import { RpcProvider } from "worker-rpc";
 import executeMigrations from "./migrations";
@@ -14,6 +14,12 @@ async function init() {
   );
   self.addEventListener("message", (evt) => rpcProvider.dispatch(evt.data));
 
+  const preparedStatementHandles = new Map<number, Statement>();
+  let ctr = 0;
+  const fresh = () => {
+    return ctr++;
+  };
+
   const SQL = await initSqlJs({
     locateFile: (file) => {
       switch (file) {
@@ -52,7 +58,42 @@ async function init() {
 
   rpcProvider.registerRpcHandler("exec", ({ s, p }: Args) => db.exec(s, p));
 
+  rpcProvider.registerRpcHandler("prepare", ({ s, p }: Args) => {
+    const preparedStatement = db.prepare(s, p);
+    const id = fresh();
+    preparedStatementHandles.set(id, preparedStatement);
+    return id;
+  });
+
+  rpcProvider.registerRpcHandler("preparedFree", ({ h }) => {
+    const prepared = preparedStatementHandles.get(h);
+    if (!prepared) return;
+    prepared.free();
+    prepared.freemem();
+  });
+
+  rpcProvider.registerRpcHandler("preparedRun", ({ h, p }) => {
+    const prepared = preparedStatementHandles.get(h);
+    if (!prepared) return;
+    prepared.run(p);
+  });
+
+  rpcProvider.registerRpcHandler("bulkInsert", ({ s, p, data }) => {
+    const preparedStatement = db.prepare(s, p);
+    db.run("BEGIN TRANSACTION");
+
+    // TODO: yo how do bulk inserts work again
+    for (const entry of data) {
+      const mapped = Object.fromEntries(
+        Object.entries(entry).map(([key, value]) => [`$${key}`, value]),
+      );
+      preparedStatement.run(mapped);
+    }
+    db.run("COMMIT TRANSACTION");
+  });
+
   rpcProvider.signal("db", { status: DbStatusCode.READY });
+  rpcProvider.signal(DbStatusCode.READY);
 }
 
 init();

lib/db/migrations.ts

@@ -29,7 +29,7 @@ export default async function executeMigrations(db: Database) {
       INSERT INTO _migrations (name) VALUES (NULL);
 
       CREATE TABLE IF NOT EXISTS _appDataVersion (version INTEGER PRIMARY KEY);
-      INSERT INTO _appDataVersion (version) VALUES (NULL);
+      INSERT INTO _appDataVersion (version) VALUES (0);
     `);
     await db.exec("COMMIT TRANSACTION");
     console.log("Created table.");
@@ -52,15 +52,22 @@ export default async function executeMigrations(db: Database) {
 async function m01_initial(db: Database) {
   db.exec(`
     CREATE TABLE charts (
-      chart_id INTEGER PRIMARY KEY AUTOINCREMENT
+      chart_id INTEGER PRIMARY KEY AUTOINCREMENT,
+      artist TEXT,
+      title TEXT,
+      difficulty TEXT,
+      banner_image TEXT
     );
 
     CREATE TABLE scores (
-      score_id INTEGER PRIMARY KEY AUTOINCREMENT
+      score_id INTEGER PRIMARY KEY AUTOINCREMENT,
+      chart_id INTEGER,
+      
+      FOREIGN KEY (chart_id) REFERENCES charts(chart_id)
     );
 
     CREATE TABLE banners (
-      hash TEXT PRIMARY KEY
+      name TEXT PRIMARY KEY
     )
   `);
 }

lib/stepcharts/stepcharts.d.ts

@@ -90,3 +90,11 @@ type SongDifficultyType = {
   mix: Mix;
   type: StepchartType;
 };
+
+type ChartData = {
+  artist: string;
+  bannerFilename: string;
+  difficulty: string;
+  title: string;
+  titleRomanized: string | null;
+};

src/App.tsx

@@ -52,15 +52,6 @@ const router = createBrowserRouter(
 );
 
 export function AppWrapper() {
-  const navigate = useNavigate();
-  const matches = useMatches();
-  const handle = matches.reduceRight(
-    // biome-ignore lint/performance/noAccumulatingSpread: <explanation>
-    (prev, curr) => ({ ...prev, ...curr.handle }),
-    {},
-  );
-  const navId = handle.navId ?? 0;
-  const urls = ["/charts", "/scores", "/settings"];
   return (
     <>
       <div className={styles.container}>
@@ -69,17 +60,6 @@ export function AppWrapper() {
           <Outlet />
         </div>
         <NavBar />
-        {/* <BottomNavigation
-          showLabels
-          value={navId}
-          onChange={(event, newValue) => {
-            navigate(urls[newValue]);
-          }}
-        >
-          <BottomNavigationAction label="Charts" icon={<ManageSearchIcon />} />
-          <BottomNavigationAction label="Scores" icon={<LineWeightIcon />} />
-          <BottomNavigationAction label="Settings" icon={<SettingsIcon />} />
-        </BottomNavigation> */}
       </div>
     </>
   );

src/components/ChartsTable.module.scss

@@ -0,0 +1,19 @@
+.tableBody {}
+
+.tableRow {
+    background-color: #eeeeee;
+
+    &:nth-child(even) {
+        background-color: #dddddd;
+    }
+}
+
+.chartCard {
+    display: flex;
+    flex-direction: column;
+    align-items: center;
+
+    .artist {
+        font-size: .8rem;
+    }
+}

src/components/ChartsTable.tsx

@@ -1,32 +1,31 @@
 import {
+  createColumnHelper,
   flexRender,
   getCoreRowModel,
   useReactTable,
 } from "@tanstack/react-table";
+import styles from "./ChartsTable.module.scss";
+
+const columnHelper = createColumnHelper<ChartData>();
 
 export default function ChartsTable({ data }) {
   console.log(data);
   const table = useReactTable({
     data,
     columns: [
-      {
+      columnHelper.accessor("title", {
-        accessorKey: "bannerFilename",
+        header: "Chart",
-        cell: ({ cell, row }) => {
+        cell: (props) => {
-          const value = cell.getValue();
+          const entry = props.row.original;
           return (
-            <div>
+            <div className={styles.chartCard}>
-              <img
+              <div>{entry.difficulty}</div>
-                style={{ width: "100px" }}
+              <div className={styles.artist}>{entry.artist}</div>
-                src={`${import.meta.env.BASE_URL}bannerImages/${value}`}
+              <div>{entry.title}</div>
-                alt={`Banner image`}
-              />
             </div>
           );
         },
-      },
+      }),
-      { accessorKey: "difficulty" },
-      { accessorKey: "artist" },
-      { accessorKey: "title" },
     ],
     getCoreRowModel: getCoreRowModel(),
   });
@@ -51,7 +50,7 @@ export default function ChartsTable({ data }) {
       </thead>
       <tbody>
         {table.getRowModel().rows.map((row) => (
-          <tr key={row.id}>
+          <tr key={row.id} className={styles.tableRow}>
             {row.getVisibleCells().map((cell) => (
               <td key={cell.id}>
                 {flexRender(cell.column.columnDef.cell, cell.getContext())}

src/components/WarningBars.tsx

@@ -5,7 +5,9 @@ import WarningIcon from "@mui/icons-material/Warning";
 import Chip from "./Chip";
 import { dbStatusAtom } from "../globals";
 import { useAtom, useAtomValue } from "jotai";
-import type { ReactNode } from "react";
+import { useEffect, useState, type ReactNode } from "react";
+import { DbStatusCode } from "../../lib/db/constants";
+import { chartDownloaderEvent } from "../../lib/chartDownloader/client";
 
 export default function WarningBars() {
   return (
@@ -52,9 +54,22 @@ function StatusBar() {
 }
 
 function DbStatusChip(): ReactNode {
+  const [firstReady, setFirstReady] = useState(true);
+
   const dbStatus = useAtomValue(dbStatusAtom);
   const prevStatus = usePrevious(dbStatus);
 
+  const justReady =
+    prevStatus?.status !== DbStatusCode.READY &&
+    dbStatus?.status === DbStatusCode.READY;
+
+  useEffect(() => {
+    if (firstReady && justReady) {
+      document.dispatchEvent(chartDownloaderEvent);
+      setFirstReady(false);
+    }
+  }, [justReady, firstReady]);
+
   switch (dbStatus.status) {
     default:
       return dbStatus.status;

src/globals.ts

@@ -11,10 +11,12 @@ export const APP_DATA_VERSION =
 export const CHART_STORE_CREATION_EVENT = new Event("chartStoreCreate");
 
 export interface DbStatus {
+  lastUpdated: number;
   persistence: "unknown" | "persisted" | "unpersisted";
   status: "uninit" | DbStatusCode | "upgrading";
 }
 export const dbStatusAtom = atom<DbStatus>({
+  lastUpdated: new Date().getTime(),
   persistence: "unknown",
   status: "uninit",
 });

src/importCharts.ts

@@ -1,115 +0,0 @@
-import stepchartsUrl from "../data/stepData.ndjson?url";
-import ndjsonStream from "../lib/ndjsonStream";
-import { APP_DATA_VERSION, CHART_STORE_CREATION_EVENT } from "./globals";
-
-export async function importCharts() {
-  const {
-    db: { result: db },
-    refetchCharts,
-  } = await openDb();
-
-  console.log("refetch", refetchCharts);
-  console.log("db", db);
-
-  if (refetchCharts) {
-    const response = await fetch(stepchartsUrl);
-    const reader = ndjsonStream(response.body).getReader();
-    const iter = iterStream(reader);
-
-    await addToDb(db, iter);
-  }
-}
-
-function addToDb(db: IDBDatabase, iter: AsyncGenerator<any>): Promise<void> {
-  return new Promise((resolve) => {
-    function openTransaction() {
-      const tx = db.transaction("chartStore", "readwrite");
-      return tx.objectStore("chartStore");
-    }
-
-    (async () => {
-      let batch = [];
-      for await (const obj of iter) {
-        if (batch.length >= 100) {
-          const store = openTransaction();
-          for (const obj of batch) {
-            const req = store.add(obj);
-            req.onsuccess = (evt) => {
-              console.log("Inserted", evt.target.result);
-            };
-          }
-          batch = [];
-        }
-
-        batch.push(obj);
-      }
-
-      const store = openTransaction();
-      for (const obj of batch) {
-        const req = store.add(obj);
-        req.onsuccess = (evt) => {
-          console.log("Inserted", evt.target.result);
-        };
-      }
-      batch = [];
-    })();
-  });
-}
-
-async function* iterStream(reader: ReadableStreamReader) {
-  let result;
-  while (!result || !result.done) {
-    result = await reader.read();
-    yield result.value;
-  }
-}
-
-function migrateToVersion1(db: IDBOpenDBRequest) {
-  try {
-    const store = db.result.createObjectStore("chartStore", {
-      // TODO: Try to use the Konami ID here
-      autoIncrement: true,
-    });
-    store.createIndex("title", "title", { unique: false });
-    store.createIndex("artist", "artist", { unique: false });
-    store.createIndex("mode", "mode", { unique: false });
-    console.log("created object store");
-    self.postMessage({ kind: "chartStoreCreate" });
-  } catch (e) {}
-}
-
-function openDb() {
-  return new Promise((resolve) => {
-    console.log("opening db...");
-    const db = indexedDB.open("ddrDb", APP_DATA_VERSION);
-
-    let refetchCharts = false;
-
-    db.addEventListener("error", (evt) => {
-      console.log("ERROR", evt);
-    });
-
-    db.addEventListener("blocked", (evt) => {
-      console.log("BLOCKED", evt);
-      self.postMessage({ kind: "dbIsBlocked" });
-    });
-
-    db.addEventListener("upgradeneeded", (evt) => {
-      console.log("IDB need upgrade", evt.oldVersion, "to", evt.newVersion);
-      refetchCharts = true;
-      migrateToVersion1(db);
-
-      console.log("done upgrading");
-    });
-
-    db.addEventListener("success", (evt) => {
-      console.log("IDB success", db.result.version, evt);
-      resolve({ db, refetchCharts });
-    });
-  });
-}
-
-addEventListener("message", (evt) => {
-  console.log("message!", evt);
-  importCharts();
-});

src/pages/Charts.tsx

@@ -1,7 +1,17 @@
-import { useQuery, useQueryClient } from "@tanstack/react-query";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
-import { APP_DATA_VERSION, CHART_STORE_CREATION_EVENT } from "../globals";
+import {
+  APP_DATA_VERSION,
+  CHART_STORE_CREATION_EVENT,
+  dbStatusAtom,
+} from "../globals";
 import { useReactTable } from "@tanstack/react-table";
 import ChartsTable from "../components/ChartsTable";
+import { dbClient } from "../../lib/db/client";
+import { atom, useAtomValue } from "jotai";
+import { useEffect } from "react";
+import { usePrevious } from "@uidotdev/usehooks";
+import { zip } from "lodash";
+import { CircularProgress } from "@mui/material";
 
 function openDb(): Promise<IDBOpenDBRequest> {
   return new Promise((resolve) => {
@@ -17,39 +27,36 @@ async function openStore(): Promise<IDBObjectStore> {
 }
 
 async function fetchCharts() {
-  let store = null;
+  const results = await dbClient.exec(
-  try {
+    "SELECT artist, title, difficulty FROM charts LIMIT 40",
-    store = await openStore();
+  );
-  } catch (e) {
+  const result = results[0];
-    console.log("could not open store", e.message);
+  const { columns, values } = result;
+  const valuesMapped = values.map((value) =>
+    Object.fromEntries(zip(columns, value)),
+  );
+
+  return valuesMapped;
 }
 
-  if (!store) {
+const dbLastUpdatedAtom = atom((get) => get(dbStatusAtom).lastUpdated);
-    await new Promise<void>((resolve) => {
-      document.addEventListener("chartStoreCreate", (evt) => {
-        console.log("SHIET");
-        resolve();
-      });
-    });
-
-    store = await openStore();
-  }
-
-  const entries = await new Promise<any[]>((resolve) => {
-    const req = store.getAll(undefined, 100);
-    req.onsuccess = (evt) => resolve(req.result);
-  });
-  console.log("entries", entries);
-  return entries;
-}
 
 export default function Charts() {
+  const dbLastUpdated = useAtomValue(dbLastUpdatedAtom);
+  const prevDbLastUpdated = usePrevious(dbLastUpdated);
+
+  const queryClient = useQueryClient();
   const fetchChartsQuery = useQuery({
     queryKey: ["fetchCharts"],
     queryFn: fetchCharts,
   });
 
-  let inner = undefined;
+  let inner = <CircularProgress />;
+
+  useEffect(() => {
+    if (dbLastUpdated > prevDbLastUpdated)
+      queryClient.invalidateQueries({ queryKey: ["fetchCharts"] });
+  }, [dbLastUpdated, prevDbLastUpdated, queryClient.invalidateQueries]);
 
   if (fetchChartsQuery.isSuccess) {
     inner = (
@@ -63,9 +70,6 @@ export default function Charts() {
     <>
       <h1>Charts</h1>
 
-      {fetchChartsQuery.status}
-      {fetchChartsQuery.error?.message}
-
       {inner}
     </>
   );