DDRCompanion/src/importCharts.ts

import stepchartsUrl from "../data/stepData.ndjson?url";
import { APP_DATA_VERSION } from "./constants";
import ndjsonStream from "can-ndjson-stream";

export async function importCharts() {
  const {
    db: { result: db },
    refetchCharts,
  } = await openDb();
  console.log("db", db);

  if (refetchCharts) {
    const response = await fetch(stepchartsUrl);
    const reader = ndjsonStream(response.body).getReader();
    const iter = iterStream(reader);

    await addToDb(db, iter);
  }
}

function addToDb(db: IDBDatabase, iter: AsyncGenerator<any>): Promise<void> {
  return new Promise((resolve) => {
    function openTransaction() {
      const tx = db.transaction("chartStore", "readwrite");
      return tx.objectStore("chartStore");
    }

    (async () => {
      let batch = [];
      for await (const obj of iter) {
        if (batch.length >= 100) {
          const store = openTransaction();
          for (const obj of batch) {
            const req = store.add(obj);
            req.onsuccess = (evt) => {
              console.log("Inserted", evt.target.result);
            };
          }
          batch = [];
        }

        batch.push(obj);
      }

      const store = openTransaction();
      for (const obj of batch) {
        const req = store.add(obj);
        req.onsuccess = (evt) => {
          console.log("Inserted", evt.target.result);
        };
      }
      batch = [];
    })();
  });
}

async function* iterStream(reader: ReadableStreamReader) {
  let result;
  while (!result || !result.done) {
    result = await reader.read();
    yield result.value;
  }
}

function openDb() {
  return new Promise((resolve) => {
    const db = indexedDB.open("ddrDb", APP_DATA_VERSION);
    let refetchCharts = false;
    db.onupgradeneeded = (evt) => {
      console.log("UPGRADE", evt.oldVersion, evt.newVersion);

      refetchCharts = true;

      {
        const store = db.result.createObjectStore("chartStore", {
          // TODO: Try to use the Konami ID here
          autoIncrement: true,
        });
        store.createIndex("title", "title", { unique: false });
        store.createIndex("artist", "artist", { unique: false });
        store.createIndex("mode", "mode", { unique: false });
      }
    };
    db.onsuccess = () => {
      resolve({ db, refetchCharts });
    };
  });
}

await importCharts();