wip

2024-05-15 13:57:16 -04:00 · 2024-05-15 13:57:16 -04:00 · cb553b52bd
parent 4a6997d232
commit cb553b52bd
27 changed files with 1846 additions and 20 deletions
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,6 @@
 node_modules
 dist
-certs
+certs
+.env.local
+.env.production
+stepData.ndjson
--- a/3
+++ b/3
@ -0,0 +1,3 @@
+deploy:
+	pnpm run build --base=/ddr
+	rsync -azrP dist/ root@veil:/home/blogDeploy/public/ddr
--- a/index.html
+++ b/index.html
@ -1,7 +1,11 @@
 <!DOCTYPE html>
 <html lang="en">
    <head>
+        <title>DDR Companion</title>
+        <meta name="description" content="My Awesome App description">
+        <link rel="icon" href="/ddr/favicon.ico">
        <meta name="viewport" content="width=device-width, initial-scale=1" />
+        <meta name="theme-color" content="#ffffff">
    </head>
    <body>
        <div id="root"></div>
--- a/lib/parseDwi.ts
+++ b/lib/parseDwi.ts
@ -0,0 +1,430 @@
+import { readdir } from "node:fs/promises";
+import Fraction from "fraction.js";
+import type { RawSimfile } from "./parseSimfile";
+import {
+  determineBeat,
+  mergeSimilarBpmRanges,
+  normalizedDifficultyMap,
+} from "./util";
+
+const metaTagsToConsume = ["title", "artist"];
+
+const dwiToSMDirection: Record<string, Arrow["direction"]> = {
+  0: "0000",
+  1: "1100", // down-left
+  2: "0100", // down
+  3: "0101", // down-right
+  4: "1000", // left
+  6: "0001", // right
+  7: "1010", // up-left
+  8: "0010", // up
+  9: "0011", // up-right
+  A: "0110", // up-down jump
+  B: "1001", // left-right jump
+};
+
+const smToDwiDirection = Object.entries(dwiToSMDirection).reduce<
+  Record<string, string>
+>((building, entry) => {
+  building[entry[1]] = entry[0];
+  return building;
+}, {});
+
+type ArrowParseResult = {
+  arrows: Arrow[];
+  freezes: FreezeBody[];
+};
+
+function combinePadsIntoOneStream(
+  p1: ArrowParseResult,
+  p2: ArrowParseResult,
+): ArrowParseResult {
+  const arrows = p1.arrows
+    .concat(p2.arrows)
+    .sort((a, b) => a.offset - b.offset);
+
+  const combinedArrows = arrows.reduce<Arrow[]>((building, arrow, i, rest) => {
+    const prevArrow = rest[i - 1];
+
+    // since previous offset matches, the previous one already
+    // grabbed and combined with this arrow, throw it away
+    if (prevArrow?.offset === arrow.offset) {
+      return building;
+    }
+
+    const nextArrow = rest[i + 1];
+
+    if (nextArrow?.offset === arrow.offset) {
+      return building.concat({
+        ...arrow,
+        direction: arrow.direction + nextArrow.direction,
+      } as Arrow);
+    }
+
+    return building.concat({
+      ...arrow,
+      direction: p1.arrows.includes(arrow)
+        ? `${arrow.direction}0000`
+        : `0000${arrow.direction}`,
+    } as Arrow);
+  }, []);
+
+  const bumpedP2Freezes = p2.freezes.map((f) => {
+    return {
+      ...f,
+      direction: f.direction + 4,
+    } as FreezeBody;
+  });
+
+  const freezes = p1.freezes
+    .concat(bumpedP2Freezes)
+    .sort((a, b) => a.startOffset - b.startOffset);
+
+  return {
+    arrows: combinedArrows,
+    freezes,
+  };
+}
+
+function findFirstNonEmptyMeasure(
+  p1Notes: string,
+  p2Notes: string | undefined,
+): number {
+  let i = 0;
+
+  while (
+    p1Notes.startsWith("00000000") &&
+    (!p2Notes || p2Notes.startsWith("00000000"))
+  ) {
+    p1Notes = p1Notes.substring(8);
+    p2Notes = p2Notes?.substring(8);
+    i += 8;
+  }
+
+  return i;
+}
+
+function parseArrowStream(
+  notes: string,
+  firstNonEmptyMeasureIndex: number,
+): ArrowParseResult {
+  const arrows: Arrow[] = [];
+  const freezes: FreezeBody[] = [];
+
+  let currentFreezeDirections: string[] = [];
+  const openFreezes: Record<
+    FreezeBody["direction"],
+    Partial<FreezeBody> | null
+  > = {
+    0: null,
+    1: null,
+    2: null,
+    3: null,
+    4: null,
+    5: null,
+    6: null,
+    7: null,
+  };
+
+  let curOffset = new Fraction(0);
+  // dwi's default increment is 8th notes
+  let curMeasureFraction = new Fraction(1).div(8);
+
+  for (
+    let i = firstNonEmptyMeasureIndex;
+    i < notes.length && notes[i] !== ";";
+    ++i
+  ) {
+    let note = notes[i];
+    const nextNote = notes[i + 1];
+
+    const smDirection = dwiToSMDirection[note];
+
+    // give the current note a chance to conclude any freezes that may be pending
+    if (smDirection) {
+      const smDirectionSplit = smDirection.split("");
+      for (let d = 0; d < smDirection.length; ++d) {
+        if (
+          smDirection[d] === "1" &&
+          openFreezes[d as FreezeBody["direction"]]
+        ) {
+          const of = openFreezes[d as FreezeBody["direction"]];
+          of!.endOffset = curOffset.n / curOffset.d + 0.25;
+          freezes.push(of as FreezeBody);
+          openFreezes[d as FreezeBody["direction"]] = null;
+          smDirectionSplit[d] = "0";
+        }
+      }
+
+      note = smToDwiDirection[smDirectionSplit.join("")];
+    }
+
+    if (nextNote === "!") {
+      // B!602080B
+      // this means the freeze starts with B (left and right), but then only right (6) has the freeze body
+      // during the freeze there is down (2) then up (8), concluding with the second B
+
+      const freezeNote = notes[i + 2];
+
+      const smDirection = dwiToSMDirection[freezeNote];
+
+      for (let d = 0; d < smDirection.length; ++d) {
+        if (smDirection[d] === "1") {
+          openFreezes[d as FreezeBody["direction"]] = {
+            direction: d as FreezeBody["direction"],
+            startOffset: curOffset.n / curOffset.d,
+          };
+        }
+      }
+
+      // the head of a freeze is still an arrow
+      arrows.push({
+        direction: dwiToSMDirection[note].replace(
+          /1/g,
+          "2",
+        ) as Arrow["direction"],
+        beat: determineBeat(curOffset),
+        offset: curOffset.n / curOffset.d,
+      });
+
+      // remember the direction to know when to close the freeze
+      currentFreezeDirections.push(freezeNote);
+
+      // move past the exclamation and trailing note
+      i += 2;
+      curOffset = curOffset.add(curMeasureFraction);
+    } else if (note === "(") {
+      curMeasureFraction = new Fraction(1).div(16);
+    } else if (note === "[") {
+      curMeasureFraction = new Fraction(1).div(24);
+    } else if (note === "{") {
+      curMeasureFraction = new Fraction(1).div(64);
+    } else if (note === "`") {
+      curMeasureFraction = new Fraction(1).div(192);
+    } else if ([")", "]", "}", "'"].includes(note)) {
+      curMeasureFraction = new Fraction(1).div(8);
+    } else if (note === "0") {
+      curOffset = curOffset.add(curMeasureFraction);
+    } else {
+      const direction = dwiToSMDirection[note];
+
+      if (direction) {
+        arrows.push({
+          direction,
+          beat: determineBeat(curOffset),
+          offset: curOffset.n / curOffset.d,
+        });
+      }
+
+      curOffset = curOffset.add(curMeasureFraction);
+    }
+  }
+
+  return { arrows, freezes };
+}
+
+async function findBanner(titlePath: string): Promise<string> {
+  const files = await readdir(titlePath);
+
+  const bannerFile = files.find(
+    (f) => f.endsWith(".png") && f.indexOf("-bg.png") === -1,
+  );
+
+  return bannerFile ?? null;
+}
+
+async function parseDwi(dwi: string, titlePath?: string): Promise<RawSimfile> {
+  let bpm: string | null = null;
+  let changebpm: string | null = null;
+  let displaybpm: string | null = null;
+  let stops: string | null = null;
+
+  const lines = dwi.split("\n").map((l) => l.trim());
+
+  let i = 0;
+
+  const sc: Partial<RawSimfile> = {
+    charts: {},
+    availableTypes: [],
+    banner: titlePath ? await findBanner(titlePath) : null,
+  };
+
+  function parseNotes(mode: "single" | "double", rawNotes: string) {
+    const values = rawNotes.split(":");
+    const difficulty = normalizedDifficultyMap[values[0].toLowerCase()];
+    const feet = Number(values[1]);
+    const notes = values[2];
+    const playerTwoNotes = values[3];
+
+    const firstNonEmptyMeasureIndex = findFirstNonEmptyMeasure(
+      notes,
+      playerTwoNotes,
+    );
+
+    let arrowResult = parseArrowStream(notes, firstNonEmptyMeasureIndex);
+
+    if (mode === "double") {
+      const playerTwoResult = parseArrowStream(
+        playerTwoNotes,
+        firstNonEmptyMeasureIndex,
+      );
+
+      arrowResult = combinePadsIntoOneStream(arrowResult, playerTwoResult);
+    }
+
+    sc.availableTypes!.push({
+      slug: `${mode}-${difficulty}`,
+      mode,
+      difficulty: difficulty as any,
+      feet,
+    });
+
+    sc.charts![`${mode}-${difficulty}`] = {
+      arrows: arrowResult.arrows,
+      freezes: arrowResult.freezes,
+      bpm: determineBpm(firstNonEmptyMeasureIndex),
+      stops: determineStops(firstNonEmptyMeasureIndex),
+    };
+  }
+
+  function determineStops(emptyOffset: number) {
+    if (!stops) {
+      return [];
+    }
+
+    return stops.split(",").map((s) => {
+      const [eigthNoteS, stopDurationS] = s.split("=");
+
+      return {
+        offset: Number(eigthNoteS) * (1 / 16) - emptyOffset * (1 / 8),
+        duration: Number(stopDurationS),
+      };
+    });
+  }
+
+  function determineBpm(emptyOffset: number) {
+    let finalBpms: Bpm[] = [];
+
+    if (bpm && !Number.isNaN(Number(bpm))) {
+      finalBpms = [{ startOffset: 0, endOffset: null, bpm: Number(bpm) }];
+    }
+
+    if (changebpm) {
+      if (!finalBpms) {
+        throw new Error("parseDwi: a simfile has changebpm but not bpm");
+      }
+
+      const entries = changebpm.split(",");
+      const additionalBpms = entries.map((bpmES, i, a) => {
+        const [eigthNoteS, bpmVS] = bpmES.split("=");
+        const nextEigthNoteS = a[i + 1]?.split("=")[0] ?? null;
+
+        const startOffset =
+          Number(eigthNoteS) * (1 / 16) - emptyOffset * (1 / 8);
+        let endOffset = null;
+
+        if (nextEigthNoteS) {
+          endOffset = Number(nextEigthNoteS) * (1 / 16) - emptyOffset * (1 / 8);
+        }
+
+        return {
+          startOffset,
+          endOffset,
+          bpm: Number(bpmVS),
+        };
+      });
+
+      finalBpms = finalBpms.concat(additionalBpms);
+      finalBpms[0].endOffset = finalBpms[1].startOffset;
+    }
+
+    if (!finalBpms) {
+      throw new Error("parseDwi, determineBpm: failed to get bpm");
+    }
+
+    finalBpms = mergeSimilarBpmRanges(finalBpms);
+
+    if (displaybpm) {
+      if (!Number.isNaN(Number(displaybpm))) {
+        sc.displayBpm = displaybpm;
+      } else if (displaybpm.indexOf("..") > -1) {
+        const [min, max] = displaybpm.split("..");
+        sc.displayBpm = `${min}-${max}`;
+      } else {
+        // displayBpm is allowed to be '*', I know of no simfiles
+        // that actually do that though
+        sc.displayBpm = displaybpm;
+      }
+    } else {
+      const minBpm = Math.min(...finalBpms.map((b) => b.bpm));
+      const maxBpm = Math.max(...finalBpms.map((b) => b.bpm));
+
+      if (minBpm === maxBpm) {
+        sc.displayBpm = Math.round(minBpm).toString();
+      } else {
+        sc.displayBpm = `${Math.round(minBpm)}-${Math.round(maxBpm)}`;
+      }
+    }
+
+    return finalBpms;
+  }
+
+  function parseTag(lines: string[], index: number): number {
+    const line = lines[index];
+
+    const r = /#([A-Za-z]+):([^;]*)/;
+    const result = r.exec(line);
+
+    if (result) {
+      const tag = result[1].toLowerCase();
+      const value = result[2];
+
+      if (metaTagsToConsume.includes(tag)) {
+        // @ts-ignore
+        sc[tag] = value;
+      } else if (tag === "displaybpm") {
+        displaybpm = value;
+      } else if (tag === "bpm") {
+        bpm = value;
+      } else if (tag === "changebpm") {
+        changebpm = value;
+      } else if (tag === "freeze") {
+        stops = value;
+      } else if (tag === "single" || tag === "double") {
+        parseNotes(tag, value);
+      }
+    }
+
+    return index + 1;
+  }
+
+  try {
+    while (i < lines.length) {
+      const line = lines[i];
+
+      if (!line.length || line.startsWith("//")) {
+        i += 1;
+        continue;
+      }
+
+      if (line.startsWith("#")) {
+        i = parseTag(lines, i);
+      } else {
+        i += 1;
+      }
+    }
+
+    if (!displaybpm && !bpm) {
+      throw new Error(`No BPM found for ${titlePath}`);
+    }
+
+    return sc as RawSimfile;
+  } catch (e) {
+    const message = e instanceof Error ? e.message : String(e);
+    const stack = e instanceof Error ? e.stack : "";
+
+    throw new Error(`error, ${message}, ${stack}, parsing ${dwi}`);
+  }
+}
+
+export { parseDwi };
--- a/lib/parseSimfile.ts
+++ b/lib/parseSimfile.ts
@ -0,0 +1,114 @@
+import { stat, readFile, readdir, copyFile } from "node:fs/promises";
+import { join, extname, resolve } from "node:path";
+import { parseDwi } from "./parseDwi";
+import { parseSm } from "./parseSm";
+
+type RawSimfile = Omit<Simfile, "mix" | "title"> & {
+  title: string;
+  titletranslit: string | null;
+  banner: string | null;
+  bannerEncoded: string | null;
+  displayBpm: string | undefined;
+};
+type Parser = (simfileSource: string, titleDir: string) => Promise<RawSimfile>;
+
+const parsers: Record<string, Parser> = {
+  ".sm": parseSm,
+  ".dwi": parseDwi,
+};
+
+async function getSongFile(songDir: string): Promise<string> {
+  const files = await readdir(songDir);
+
+  // TODO: support more than .sm
+  const extensions = Object.keys(parsers);
+
+  const songFile = files.find((f) => extensions.some((ext) => f.endsWith(ext)));
+
+  if (!songFile) {
+    throw new Error(`No song file found in ${songDir}`);
+  }
+
+  return songFile;
+}
+
+function toSafeName(name: string): string {
+  let name2 = name;
+  name2 = name2.replace(".png", "");
+  name2 = name2.replace(/\s/g, "-").replace(/[^\w]/g, "_");
+
+  return `${name}.png`;
+}
+
+function getBpms(sm: RawSimfile): number[] {
+  const chart = Object.values(sm.charts)[0];
+
+  return chart.bpm.map((b) => b.bpm);
+}
+
+async function parseSimfile(
+  rootDir: string,
+  mixDir: string,
+  titleDir: string,
+): Promise<Omit<Simfile, "mix">> {
+  const stepchartSongDirPath = join(rootDir, mixDir, titleDir);
+  const songFile = await getSongFile(stepchartSongDirPath);
+  const stepchartPath = join(stepchartSongDirPath, songFile);
+  const extension = extname(stepchartPath);
+
+  const parser = parsers[extension];
+
+  if (!parser) {
+    throw new Error(`No parser registered for extension: ${extension}`);
+  }
+
+  const fileContents = await readFile(stepchartPath);
+  const rawStepchart = await parser(
+    fileContents.toString(),
+    stepchartSongDirPath,
+  );
+
+  const bannerPath = join(stepchartSongDirPath, rawStepchart.banner);
+  try {
+    const bannerData = await readFile(bannerPath);
+    rawStepchart.bannerEncoded = bannerData.toString("base64");
+  } catch (e) {}
+
+  //   if (bannerMeta !== undefined) {
+  //     const publicName = toSafeName(`${mixDir}-${rawStepchart.banner}`);
+  //     const srcPath = resolve(stepchartSongDirPath, rawStepchart.banner);
+  //     const destPath = resolve("public/bannerImages", publicName);
+  //     await copyFile(srcPath, destPath);
+  //     await copyFile(
+  //       join(stepchartSongDirPath, rawStepchart.banner),
+  //       join("components/bannerImages", publicName),
+  //     );
+  //     rawStepchart.banner = publicName;
+  //   } else {
+  //     rawStepchart.banner = null;
+  //   }
+
+  const bpms = getBpms(rawStepchart);
+  const minBpm = Math.round(Math.min(...bpms));
+  const maxBpm = Math.round(Math.max(...bpms));
+
+  const displayBpm =
+    minBpm === maxBpm ? minBpm.toString() : `${minBpm}-${maxBpm}`;
+
+  return {
+    ...rawStepchart,
+    title: {
+      titleName: rawStepchart.title,
+      translitTitleName: rawStepchart.titletranslit ?? null,
+      titleDir,
+      banner: rawStepchart.banner,
+    },
+    minBpm,
+    maxBpm,
+    displayBpm,
+    stopCount: Object.values(rawStepchart.charts)[0].stops.length,
+  };
+}
+
+export { parseSimfile };
+export type { RawSimfile };
--- a/lib/parseSm.ts
+++ b/lib/parseSm.ts
@ -0,0 +1,336 @@
+import Fraction from "fraction.js";
+import type { RawSimfile } from "./parseSimfile";
+import {
+  determineBeat,
+  mergeSimilarBpmRanges,
+  normalizedDifficultyMap,
+} from "./util";
+
+const metaTagsToConsume = ["title", "titletranslit", "artist", "banner"];
+
+function concludesANoteTag(line: string | undefined): boolean {
+  if (line === undefined) {
+    return true;
+  }
+
+  return line[0] === ";" || (line[0] === "," && line[1] === ";");
+}
+
+function getMeasureLength(lines: string[], i: number): number {
+  let measureLength = 0;
+
+  for (
+    ;
+    i < lines.length && !concludesANoteTag(lines[i]) && lines[i][0] !== ",";
+    ++i
+  ) {
+    if (lines[i].trim() !== "") {
+      measureLength += 1;
+    }
+  }
+
+  return measureLength;
+}
+
+function trimNoteLine(line: string, mode: "single" | "double"): string {
+  if (mode === "single") return line.substring(0, 4);
+  return line.substring(0, 8);
+}
+
+function isRest(line: string): boolean {
+  return line.split("").every((d) => d === "0");
+}
+
+function findFirstNonEmptyMeasure(
+  mode: "single" | "double",
+  lines: string[],
+  i: number,
+): { firstNonEmptyMeasureIndex: number; numMeasuresSkipped: number } {
+  let numMeasuresSkipped = 0;
+  let measureIndex = i;
+
+  for (; i < lines.length && !concludesANoteTag(lines[i]); ++i) {
+    const line = lines[i];
+    if (line.trim() === "") {
+      continue;
+    }
+
+    if (line.startsWith(",")) {
+      measureIndex = i + 1;
+      numMeasuresSkipped += 1;
+      continue;
+    }
+
+    if (!isRest(trimNoteLine(line, mode))) {
+      return { firstNonEmptyMeasureIndex: measureIndex, numMeasuresSkipped };
+    }
+  }
+
+  throw new Error(
+    "findFirstNonEmptyMeasure, failed to find a non-empty measure in entire song",
+  );
+}
+
+async function parseSm(sm: string, _titlePath: string): Promise<RawSimfile> {
+  const lines = sm.split("\n").map((l) => l.trim());
+
+  let i = 0;
+  let bpmString: string | null = null;
+  let stopsString: string | null = null;
+
+  const sc: Partial<RawSimfile> = {
+    charts: {},
+    availableTypes: [],
+    banner: null,
+  };
+
+  function parseStops(
+    stopsString: string | null,
+    emptyOffsetInMeasures: number,
+  ) {
+    if (!stopsString) {
+      return [];
+    }
+
+    const entries = stopsString.split(",");
+
+    return entries.map((s) => {
+      const [stopS, durationS] = s.split("=");
+      return {
+        offset: Number(stopS) * 0.25 - emptyOffsetInMeasures,
+        duration: Number(durationS),
+      };
+    });
+  }
+
+  function parseBpms(bpmString: string, emptyOffsetInMeasures: number) {
+    // 0=79.3,4=80,33=79.8,36=100,68=120,100=137,103=143,106=139,108=140,130=141.5,132=160,164=182,166=181,168=180;
+    const entries = bpmString.split(",");
+
+    const bpms = entries.map((e, i, a) => {
+      const [beatS, bpmS] = e.split("=");
+      const nextBeatS = a[i + 1]?.split("=")[0] ?? null;
+
+      return {
+        startOffset: Number(beatS) * 0.25 - emptyOffsetInMeasures,
+        endOffset:
+          nextBeatS === null
+            ? null
+            : Number(nextBeatS) * 0.25 - emptyOffsetInMeasures,
+        bpm: Number(bpmS),
+      };
+    });
+
+    const mergedBpms = mergeSimilarBpmRanges(bpms);
+
+    const minBpm = Math.min(...mergedBpms.map((b) => b.bpm));
+    const maxBpm = Math.max(...mergedBpms.map((b) => b.bpm));
+
+    if (Math.abs(minBpm - maxBpm) < 2) {
+      sc.displayBpm = Math.round(minBpm).toString();
+    } else {
+      sc.displayBpm = `${Math.round(minBpm)}-${Math.round(maxBpm)}`;
+    }
+
+    return mergedBpms;
+  }
+
+  function parseFreezes(
+    lines: string[],
+    i: number,
+    mode: string,
+    difficulty: string,
+  ): FreezeBody[] {
+    const freezes: FreezeBody[] = [];
+    const open: Record<number, Partial<FreezeBody> | undefined> = {};
+
+    let curOffset = new Fraction(0);
+    let curMeasureFraction = new Fraction(1).div(
+      getMeasureLength(lines, i) || 1,
+    );
+
+    for (; i < lines.length && !concludesANoteTag(lines[i]); ++i) {
+      const line = lines[i];
+
+      if (line.trim() === "") {
+        continue;
+      }
+
+      if (line[0] === ",") {
+        curMeasureFraction = new Fraction(1).div(
+          getMeasureLength(lines, i + 1) || 1,
+        );
+        continue;
+      }
+
+      if (line.indexOf("2") === -1 && line.indexOf("3") === -1) {
+        curOffset = curOffset.add(curMeasureFraction);
+        continue;
+      }
+
+      const cleanedLine = line.replace(/[^23]/g, "0");
+
+      for (let d = 0; d < cleanedLine.length; ++d) {
+        if (cleanedLine[d] === "2") {
+          if (open[d]) {
+            throw new Error(
+              `${sc.title}, ${mode}, ${difficulty} -- error parsing freezes, found a new starting freeze before a previous one finished`,
+            );
+          }
+          const startBeatFraction = curOffset;
+          open[d] = {
+            direction: d as FreezeBody["direction"],
+            startOffset: startBeatFraction.n / startBeatFraction.d,
+          };
+        } else if (cleanedLine[d] === "3") {
+          if (!open[d]) {
+            throw new Error(
+              `${sc.title}, ${mode}, ${difficulty} -- error parsing freezes, needed to close a freeze that never opened`,
+            );
+          }
+
+          const endBeatFraction = curOffset.add(new Fraction(1).div(4));
+          open[d]!.endOffset = endBeatFraction.n / endBeatFraction.d;
+          freezes.push(open[d] as FreezeBody);
+          open[d] = undefined;
+        }
+      }
+
+      curOffset = curOffset.add(curMeasureFraction);
+    }
+
+    return freezes;
+  }
+
+  function parseNotes(lines: string[], i: number, bpmString: string): number {
+    // move past #NOTES into the note metadata
+    i++;
+    const mode = lines[i++].replace("dance-", "").replace(":", "");
+    i++; // skip author for now
+    const difficulty =
+      normalizedDifficultyMap[lines[i++].replace(":", "").toLowerCase()];
+    const feet = Number(lines[i++].replace(":", ""));
+    i++; // skip groove meter data for now
+
+    // skip couple, versus, etc for now
+    if (mode !== "single" && mode !== "double") {
+      return i + 1;
+    }
+
+    // now i is pointing at the first measure
+    const arrows: Arrow[] = [];
+
+    const { firstNonEmptyMeasureIndex, numMeasuresSkipped } =
+      findFirstNonEmptyMeasure(mode, lines, i);
+    i = firstNonEmptyMeasureIndex;
+
+    const firstMeasureIndex = i;
+    let curOffset = new Fraction(0);
+    // in case the measure is size zero, fall back to dividing by one
+    // this is just being defensive, this would mean the stepfile has no notes in it
+    let curMeasureFraction = new Fraction(1).div(
+      getMeasureLength(lines, i) || 1,
+    );
+
+    for (; i < lines.length && !concludesANoteTag(lines[i]); ++i) {
+      // for now, remove freeze ends as they are handled in parseFreezes
+      // TODO: deal with freezes here, no need to have two functions doing basically the same thing
+      const line = trimNoteLine(lines[i], mode).replace(/3/g, "0");
+
+      if (line.trim() === "") {
+        continue;
+      }
+
+      if (line.startsWith(",")) {
+        curMeasureFraction = new Fraction(1).div(
+          getMeasureLength(lines, i + 1) || 1,
+        );
+        continue;
+      }
+
+      if (!isRest(line)) {
+        arrows.push({
+          beat: determineBeat(curOffset),
+          offset: curOffset.n / curOffset.d,
+          direction: line as Arrow["direction"],
+        });
+      }
+
+      curOffset = curOffset.add(curMeasureFraction);
+    }
+
+    const freezes = parseFreezes(lines, firstMeasureIndex, mode, difficulty);
+
+    sc.charts![`${mode}-${difficulty}`] = {
+      arrows,
+      freezes,
+      bpm: parseBpms(bpmString, numMeasuresSkipped),
+      stops: parseStops(stopsString, numMeasuresSkipped),
+    };
+
+    sc.availableTypes!.push({
+      slug: `${mode}-${difficulty}`,
+      mode,
+      difficulty: difficulty as any,
+      feet,
+    });
+
+    return i + 1;
+  }
+
+  function parseTag(lines: string[], index: number): number {
+    const line = lines[index];
+
+    const r = /#([A-Za-z]+):([^;]*)/;
+    const result = r.exec(line);
+
+    if (result) {
+      const tag = result[1].toLowerCase();
+      const value = result[2];
+
+      if (metaTagsToConsume.includes(tag)) {
+        // @ts-ignore
+        sc[tag] = value;
+      } else if (tag === "bpms") {
+        bpmString = value;
+      } else if (tag === "stops") {
+        stopsString = value;
+      } else if (tag === "notes") {
+        if (!bpmString) {
+          throw new Error("parseSm: about to parse notes but never got bpm");
+        }
+        return parseNotes(lines, index, bpmString);
+      }
+    }
+
+    return index + 1;
+  }
+
+  try {
+    while (i < lines.length) {
+      const line = lines[i];
+
+      if (!line.length || line.startsWith("//")) {
+        i += 1;
+        continue;
+      }
+
+      if (line.startsWith("#")) {
+        i = parseTag(lines, i);
+      } else {
+        i += 1;
+      }
+    }
+
+    return sc as RawSimfile;
+  } catch (e) {
+    const message = e instanceof Error ? e.message : String(e);
+    const stack = e instanceof Error ? e.stack : "";
+
+    throw new Error(
+      `error, ${message}, ${stack}, parsing ${sm.substring(0, 300)}`,
+    );
+  }
+}
+
+export { parseSm };
--- a/lib/stepcharts.d.ts
+++ b/lib/stepcharts.d.ts
@ -0,0 +1,91 @@
+type Arrow = {
+  // other beats such as 5ths and 32nds end up being colored
+  // the same as 6ths. This probably should be "color" not "beat" TODO
+  beat: 4 | 6 | 8 | 12 | 16;
+  direction:
+    | `${0 | 1 | 2}${0 | 1 | 2}${0 | 1 | 2}${0 | 1 | 2}`
+    | `${0 | 1 | 2}${0 | 1 | 2}${0 | 1 | 2}${0 | 1 | 2}${0 | 1 | 2}${
+        | 0
+        | 1
+        | 2}${0 | 1 | 2}${0 | 1 | 2}`;
+  offset: number;
+};
+
+type FreezeBody = {
+  direction: 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7;
+  startOffset: number;
+  endOffset: number;
+};
+
+type Mode = "single" | "double";
+type Difficulty =
+  | "beginner"
+  | "basic"
+  | "difficult"
+  | "expert"
+  | "challenge"
+  | "edit";
+
+type StepchartType = {
+  slug: string;
+  mode: Mode;
+  difficulty: Difficulty;
+  feet: number;
+};
+
+type Stats = {
+  jumps: number;
+  jacks: number;
+  freezes: number;
+  gallops: number;
+};
+
+type Bpm = {
+  startOffset: number;
+  endOffset: number | null;
+  bpm: number;
+};
+
+type Stop = {
+  offset: number;
+  duration: number;
+};
+
+type Stepchart = {
+  arrows: Arrow[];
+  freezes: FreezeBody[];
+  bpm: Bpm[];
+  stops: Stop[];
+};
+
+type Simfile = {
+  title: Title;
+  artist: string;
+  mix: Mix;
+  availableTypes: StepchartType[];
+  charts: Record<string, Stepchart>;
+  minBpm: number;
+  maxBpm: number;
+  displayBpm: string;
+  stopCount: number;
+  stats: Stats;
+};
+
+type Mix = {
+  mixName: string;
+  mixDir: string;
+  songCount: number;
+};
+
+type Title = {
+  titleName: string;
+  translitTitleName: string | null;
+  titleDir: string;
+  banner: string | null;
+};
+
+type SongDifficultyType = {
+  title: Title;
+  mix: Mix;
+  type: StepchartType;
+};
--- a/lib/util.ts
+++ b/lib/util.ts
@ -0,0 +1,66 @@
+import Fraction from "fraction.js";
+
+const beats = [
+  new Fraction(1).div(4),
+  new Fraction(1).div(6),
+  new Fraction(1).div(8),
+  new Fraction(1).div(12),
+  new Fraction(1).div(16),
+];
+
+function determineBeat(offset: Fraction): Arrow["beat"] {
+  const match = beats.find((b) => offset.mod(b).n === 0);
+
+  if (!match) {
+    // didn't find anything? then it's a weirdo like a 5th note or 32nd note, they get colored
+    // the same as 6ths
+    return 6;
+  }
+
+  return match.d as Arrow["beat"];
+}
+
+const normalizedDifficultyMap: Record<string, Difficulty> = {
+  beginner: "beginner",
+  easy: "basic",
+  basic: "basic",
+  trick: "difficult",
+  another: "difficult",
+  medium: "difficult",
+  difficult: "expert",
+  expert: "expert",
+  maniac: "expert",
+  ssr: "expert",
+  hard: "expert",
+  challenge: "challenge",
+  smaniac: "challenge",
+  // TODO: filter edits out altogether
+  edit: "edit",
+};
+
+function similarBpm(a: Bpm, b: Bpm): boolean {
+  return Math.abs(a.bpm - b.bpm) < 1;
+}
+
+function mergeSimilarBpmRanges(bpm: Bpm[]): Bpm[] {
+  return bpm.reduce<Bpm[]>((building, b, i, a) => {
+    const prev = a[i - 1];
+    const next = a[i + 1];
+
+    if (prev && similarBpm(prev, b)) {
+      // this bpm was merged on the last iteration, so skip it
+      return building;
+    }
+
+    if (next && similarBpm(next, b)) {
+      return building.concat({
+        ...b,
+        endOffset: next.endOffset,
+      });
+    }
+
+    return building.concat(b);
+  }, []);
+}
+
+export { determineBeat, normalizedDifficultyMap, mergeSimilarBpmRanges };
--- a/package.json
+++ b/package.json
@ -5,7 +5,8 @@
  "main": "index.js",
  "scripts": {
    "dev": "vite",
-    "build": "vite build"
+    "build": "vite build",
+    "generate-pwa-assets": "pwa-assets-generator"
  },
  "type": "module",
  "keywords": [],
@ -13,10 +14,13 @@
  "license": "ISC",
  "devDependencies": {
    "@biomejs/biome": "^1.7.3",
+    "@types/ndjson": "^2.0.4",
    "@types/react": "^18.3.2",
    "@types/react-dom": "^18.3.0",
+    "@vite-pwa/assets-generator": "^0.2.4",
    "@vitejs/plugin-basic-ssl": "^1.1.0",
    "@vitejs/plugin-react-swc": "^3.6.0",
+    "fraction.js": "^4.3.7",
    "sass": "^1.77.1",
    "typescript": "^5.4.5",
    "vite": "^5.2.11",
@ -28,9 +32,13 @@
    "@emotion/styled": "^11.11.5",
    "@mui/icons-material": "^5.15.17",
    "@mui/material": "^5.15.17",
+    "@tanstack/react-query": "^5.36.1",
+    "can-ndjson-stream": "^1.0.2",
+    "ndjson": "^2.0.0",
    "react": "^18.3.1",
    "react-dom": "^18.3.1",
    "react-router": "^6.23.1",
-    "react-router-dom": "^6.23.1"
+    "react-router-dom": "^6.23.1",
+    "stream-json": "^1.8.0"
  }
 }
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/public/apple-touch-icon-180x180.png
+++ b/public/apple-touch-icon-180x180.png
--- a/public/favicon.ico
+++ b/public/favicon.ico
--- a/public/favicon.png
+++ b/public/favicon.png
--- a/public/maskable-icon-512x512.png
+++ b/public/maskable-icon-512x512.png
--- a/public/pwa-192x192.png
+++ b/public/pwa-192x192.png
--- a/public/pwa-512x512.png
+++ b/public/pwa-512x512.png
--- a/public/pwa-64x64.png
+++ b/public/pwa-64x64.png
--- a/public/robots.txt
+++ b/public/robots.txt
@ -0,0 +1,2 @@
+User-agent: *
+Allow: /
--- a/pwa-assets.config.ts
+++ b/pwa-assets.config.ts
@ -0,0 +1,6 @@
+import { defineConfig } from "@vite-pwa/assets-generator/config";
+
+export default defineConfig({
+  preset: "minimal",
+  images: ["public/favicon.png"],
+});
--- a/scripts/compileStepcharts.ts
+++ b/scripts/compileStepcharts.ts
@ -0,0 +1,47 @@
+// TODO: Make this concurrent instead of serial
+
+import { join, extname } from "node:path";
+import { pipeline } from "node:stream/promises";
+import { Readable } from "node:stream";
+import * as ndjson from "ndjson";
+import { readdir, writeFile, stat } from "node:fs/promises";
+import { createWriteStream } from "node:fs";
+import { parseSm } from "../lib/parseSm";
+import { parseSimfile } from "../lib/parseSimfile";
+
+const rootDir = process.env.STEPCHARTS_DIR ?? "/tmp/stepcharts";
+const stepchartsDir = join(rootDir, "prodStepcharts");
+const outFile = process.env.OUT_FILE ?? "data/stepData.ndjson";
+
+async function* mixToSongs(
+  mixDirs: string[],
+): AsyncGenerator<[string, string], void, unknown> {
+  for (const mix of mixDirs) {
+    const mixDir = join(stepchartsDir, mix);
+    const files = await readdir(mixDir);
+    for (const song of files) {
+      yield [mix, song];
+    }
+  }
+}
+
+async function* processSong(gen: AsyncGenerator<[string, string]>) {
+  for await (const [mix, song] of gen) {
+    try {
+      const parsed = await parseSimfile(stepchartsDir, mix, song);
+      yield parsed;
+    } catch (e) {
+      console.error("failed on", mix, song, ":", e.message);
+    }
+  }
+}
+
+const mixDirs = await readdir(stepchartsDir);
+const songIter = mixToSongs(mixDirs);
+const outIter = processSong(songIter);
+
+const writer = createWriteStream(outFile);
+for await (const songObj of outIter) {
+  writer.write(JSON.stringify(songObj));
+  writer.write("\n");
+}
--- a/src/App.tsx
+++ b/src/App.tsx
@ -2,7 +2,7 @@ import { BottomNavigation, BottomNavigationAction, Icon } from "@mui/material";
 import LineWeightIcon from "@mui/icons-material/LineWeight";
 import ManageSearchIcon from "@mui/icons-material/ManageSearch";
 import SettingsIcon from "@mui/icons-material/Settings";
-import { useState } from "react";
+import { useEffect, useState } from "react";
 import {
  createBrowserRouter,
  RouterProvider,
@ -18,6 +18,10 @@ import Charts from "./pages/Charts";
 import styles from "./App.module.scss";
 import Scores from "./pages/Scores";
 import Settings from "./pages/Settings";
+import ImportChartWorker from "./importCharts?worker";
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query";
+
+const queryClient = new QueryClient();

 const router = createBrowserRouter(
  [
@ -70,9 +74,15 @@ export function AppWrapper() {
 }

 export default function App() {
+  useEffect(() => {
+    const worker = new ImportChartWorker();
+  });
+
  return (
    <>
-      <RouterProvider router={router} />
+      <QueryClientProvider client={queryClient}>
+        <RouterProvider router={router} />
+      </QueryClientProvider>
    </>
  );
 }
--- a/src/constants.ts
+++ b/src/constants.ts
@ -0,0 +1 @@
+export const APP_DATA_VERSION = 1;
--- a/src/global.scss
+++ b/src/global.scss
@ -1,3 +1,12 @@
+:root {
+    --font: sans-serif;
+}
+
+html,
+body {
+    font-family: var(--font);
+}
+
 html,
 body,
 #root {
--- a/src/importCharts.ts
+++ b/src/importCharts.ts
@ -0,0 +1,90 @@
+import stepchartsUrl from "../data/stepData.ndjson?url";
+import { APP_DATA_VERSION } from "./constants";
+import ndjsonStream from "can-ndjson-stream";
+
+export async function importCharts() {
+  const {
+    db: { result: db },
+    refetchCharts,
+  } = await openDb();
+  console.log("db", db);
+
+  if (refetchCharts) {
+    const response = await fetch(stepchartsUrl);
+    const reader = ndjsonStream(response.body).getReader();
+    const iter = iterStream(reader);
+
+    await addToDb(db, iter);
+  }
+}
+
+function addToDb(db: IDBDatabase, iter: AsyncGenerator<any>): Promise<void> {
+  return new Promise((resolve) => {
+    function openTransaction() {
+      const tx = db.transaction("chartStore", "readwrite");
+      return tx.objectStore("chartStore");
+    }
+
+    (async () => {
+      let batch = [];
+      for await (const obj of iter) {
+        if (batch.length >= 100) {
+          const store = openTransaction();
+          for (const obj of batch) {
+            const req = store.add(obj);
+            req.onsuccess = (evt) => {
+              console.log("Inserted", evt.target.result);
+            };
+          }
+          batch = [];
+        }
+
+        batch.push(obj);
+      }
+
+      const store = openTransaction();
+      for (const obj of batch) {
+        const req = store.add(obj);
+        req.onsuccess = (evt) => {
+          console.log("Inserted", evt.target.result);
+        };
+      }
+      batch = [];
+    })();
+  });
+}
+
+async function* iterStream(reader: ReadableStreamReader) {
+  let result;
+  while (!result || !result.done) {
+    result = await reader.read();
+    yield result.value;
+  }
+}
+
+function openDb() {
+  return new Promise((resolve) => {
+    const db = indexedDB.open("ddrDb", APP_DATA_VERSION);
+    let refetchCharts = false;
+    db.onupgradeneeded = (evt) => {
+      console.log("UPGRADE", evt.oldVersion, evt.newVersion);
+
+      refetchCharts = true;
+
+      {
+        const store = db.result.createObjectStore("chartStore", {
+          // TODO: Try to use the Konami ID here
+          autoIncrement: true,
+        });
+        store.createIndex("title", "title", { unique: false });
+        store.createIndex("artist", "artist", { unique: false });
+        store.createIndex("mode", "mode", { unique: false });
+      }
+    };
+    db.onsuccess = () => {
+      resolve({ db, refetchCharts });
+    };
+  });
+}
+
+await importCharts();
--- a/src/pages/Charts.tsx
+++ b/src/pages/Charts.tsx
@ -1,7 +1,23 @@
+import { useQuery, useQueryClient } from "@tanstack/react-query";
+import { APP_DATA_VERSION } from "../constants";
+
+async function fetchCharts() {
+  const db = indexedDB.open("ddrDb", APP_DATA_VERSION);
+  return [];
+}
+
 export default function Charts() {
+  const queryClient = useQueryClient();
+  const fetchChartsQuery = useQuery({
+    queryKey: ["fetchCharts"],
+    queryFn: fetchCharts,
+  });
+
  return (
    <>
      <h1>Charts</h1>
+
+      {JSON.stringify(fetchChartsQuery)}
    </>
  );
 }
--- a/tsconfig.json
+++ b/tsconfig.json
@ -2,6 +2,7 @@
  "compilerOptions": {
    "jsx": "react-jsx",
    "module": "ESNext",
+    "target": "ESNext",
    "moduleResolution": "Node"
  }
 }
--- a/vite.config.ts
+++ b/vite.config.ts
@ -10,7 +10,12 @@ export default defineConfig({
  },
  plugins: [
    react(),
-    VitePWA({ registerType: "autoUpdate" }),
+    VitePWA({
+      registerType: "autoUpdate",
+      manifest: {
+        icons: [{ src: "pwa-192x192.png", purpose: "any", sizes: "192x192" }],
+      },
+    }),

    basicSsl({
      /** name of certification */