DDRCompanion/src/importCharts.ts

import stepchartsUrl from "../data/stepData.ndjson?url";
import ndjsonStream from "../lib/ndjsonStream";
import { APP_DATA_VERSION, CHART_STORE_CREATION_EVENT } from "./globals";

export async function importCharts() {
  const {
    db: { result: db },
    refetchCharts,
  } = await openDb();

  console.log("refetch", refetchCharts);
  console.log("db", db);

  if (refetchCharts) {
    const response = await fetch(stepchartsUrl);
    const reader = ndjsonStream(response.body).getReader();
    const iter = iterStream(reader);

    await addToDb(db, iter);
  }
}

function addToDb(db: IDBDatabase, iter: AsyncGenerator<any>): Promise<void> {
  return new Promise((resolve) => {
    function openTransaction() {
      const tx = db.transaction("chartStore", "readwrite");
      return tx.objectStore("chartStore");
    }

    (async () => {
      let batch = [];
      for await (const obj of iter) {
        if (batch.length >= 100) {
          const store = openTransaction();
          for (const obj of batch) {
            const req = store.add(obj);
            req.onsuccess = (evt) => {
              console.log("Inserted", evt.target.result);
            };
          }
          batch = [];
        }

        batch.push(obj);
      }

      const store = openTransaction();
      for (const obj of batch) {
        const req = store.add(obj);
        req.onsuccess = (evt) => {
          console.log("Inserted", evt.target.result);
        };
      }
      batch = [];
    })();
  });
}

async function* iterStream(reader: ReadableStreamReader) {
  let result;
  while (!result || !result.done) {
    result = await reader.read();
    yield result.value;
  }
}

function migrateToVersion1(db: IDBOpenDBRequest) {
  try {
    const store = db.result.createObjectStore("chartStore", {
      // TODO: Try to use the Konami ID here
      autoIncrement: true,
    });
    store.createIndex("title", "title", { unique: false });
    store.createIndex("artist", "artist", { unique: false });
    store.createIndex("mode", "mode", { unique: false });
    console.log("created object store");
    self.postMessage({ kind: "chartStoreCreate" });
  } catch (e) {}
}

function openDb() {
  return new Promise((resolve) => {
    console.log("opening db...");
    const db = indexedDB.open("ddrDb", APP_DATA_VERSION);

    let refetchCharts = false;

    db.addEventListener("error", (evt) => {
      console.log("ERROR", evt);
    });

    db.addEventListener("blocked", (evt) => {
      console.log("BLOCKED", evt);
    });

    db.addEventListener("upgradeneeded", (evt) => {
      console.log("IDB need upgrade", evt.oldVersion, "to", evt.newVersion);
      refetchCharts = true;
      migrateToVersion1(db);

      console.log("done upgrading");
    });

    db.addEventListener("success", (evt) => {
      console.log("IDB success", db.result.version, evt);
      resolve({ db, refetchCharts });
    });
  });
}

addEventListener("message", (evt) => {
  console.log("message!", evt);
  importCharts();
});