From d9eefa17d4a0e1af0180c6c445e0714b16471852 Mon Sep 17 00:00:00 2001 From: Michael Zhang Date: Thu, 30 May 2024 12:22:44 -0500 Subject: [PATCH] initial --- .envrc | 1 + .gitignore | 6 + biome.json | 17 ++ bun.lockb | Bin 0 -> 9090 bytes main.ts | 218 ++++++++++++++++++ package.json | 13 ++ .../20240530161311_initial/migration.sql | 25 ++ .../20240530162224_bigint/migration.sql | 26 +++ .../migrations/20240530162755_a/migration.sql | 22 ++ .../migrations/20240530165230_a/migration.sql | 58 +++++ .../migrations/20240530165947_a/migration.sql | 34 +++ prisma/migrations/migration_lock.toml | 3 + prisma/schema.prisma | 64 +++++ test.py | 48 ++++ 14 files changed, 535 insertions(+) create mode 100644 .envrc create mode 100644 .gitignore create mode 100644 biome.json create mode 100755 bun.lockb create mode 100644 main.ts create mode 100644 package.json create mode 100644 prisma/migrations/20240530161311_initial/migration.sql create mode 100644 prisma/migrations/20240530162224_bigint/migration.sql create mode 100644 prisma/migrations/20240530162755_a/migration.sql create mode 100644 prisma/migrations/20240530165230_a/migration.sql create mode 100644 prisma/migrations/20240530165947_a/migration.sql create mode 100644 prisma/migrations/migration_lock.toml create mode 100644 prisma/schema.prisma create mode 100644 test.py diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..4e46a90 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +layout python3 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a6cadb1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +node_modules +# Keep environment variables out of version control +.env +.venv + +prisma/dev.db \ No newline at end of file diff --git a/biome.json b/biome.json new file mode 100644 index 0000000..772c0eb --- /dev/null +++ b/biome.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.7.3/schema.json", + "organizeImports": { + "enabled": true + }, + "formatter": { + "enabled": true, + "indentStyle": "space", + "indentWidth": 2 + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true + } + } +} diff --git a/bun.lockb b/bun.lockb new file mode 100755 index 0000000000000000000000000000000000000000..0ee2d4f690b4d09a797792be54f00fc23777d97c GIT binary patch literal 9090 zcmeHNd011|62CxH1O-HJ!37Y-jVy!!LBIvEU|q3_iUJKJKrjSM0?Ja2b+?L&;zm^v zP{fu8wPF<)sv=etM8$mp!Tmix#RV-byqP2iZe1|8{?Yfo&i946=brQX&78SsyLYg$ zW=G3q>>#m}Et1Aq1w~5@;K!6i2G0r+OGM0&NLhqH!HnS;=u#B*?9jihjwF@twQW2# zZusTZCJ&rmyEJqz9ebUZ?CJl|NnY0n8i90GD@y+jt<;oVNdrxpq7Z06QA(*u7=$9}& zfJcAGDA9(5;EzIMw2yeSudNRR-v)p@zyp+mM@+%v)k+B71@L^0eY~e_9|(Rn;D>4O zFz&S$g3kp!3-B;pDY#Vqv>_q*`v4lE!6WWt`)>_IV*!s?I|;1LJwgVsXqTS4H_{-4DQ0guoB&*H~Ir{mv&w}$(J;r=fv zPV*eRymr0WG2(wQv>W{n{G09HEEBv5IPUrv@M8e~SM09`JkDR7cNoKu9X}TUgySFg z2Uyo}?bk|3|GUG48KLRF1t8G}ZG_;sNtZKWMF#2*T3w2gg0L5=GH=?`iw z;}}N`d!I^BW0}~cYrNk{QxAJ8h4ZTm{7_J1J=wrf;~Io9!23Nkx+iE9)Oa7|f79ThE(!K6k$`$+B`e5D|ZIj1gg&HSKgJLW276zBeNJyn zMPWwo;hY1Fhi@of{g!7i9ffs;=Mua)73-Xtf#qPc zM2q7d@qvQ=S1OB~s!n(HFE?6f?A!g=QpYOJS5K};0-Fj-&s-d}g(Io$vM6PDgzj@z zriu4Rz9DBmjhB8tVff1z?;o^q<+;L<=7GUI7S|ekZ>;O#C(iWfxbn;NrBB9sPw-29 z+?8L}cYm)mh7RAZq9Cx$V&C0U>z5~1uIg9boyH4i?`B~bJex9*wZlKl@KRMSGs`s4 zU2xW>cUk8gi`hXnJ|_q5T<02F<{n-cX=V5Y_h7YmrS{f!_pWgG!2P0>s zT9%44WVx9mN@|BbcJ{pU{M;^!zC9N2jK1jS(~zH4aY3^A`l?AlM8*SIf03IJKGL*c zaLK?&W|jMYV6^EqzH`$-^9eU*eREpJ^TsdJ&Zef^u1gr*(2X&E!rC+`>#mq>H6+)s zOGCik-_xsjI;Pn)Ub2p>c$GHOWr=lnves@ z&m?ItzFSRwIj*!1KWhYkOh%smys*u>73o()e7Y?EoyJQ)^Ds7a5Dwh7tn&bKxzRFF zazD4FB_po{k5)2!JH`Gh$4)=Da;mGb*?|0+CZiWRoLN)j;+b&Kc)_sfsp9Rox0RI$ zFVc9){-W}C#kSo}o?#VZeUH|DpWb<9P#x28Nw$aMO|$aludZiYy~q4)L9bN*Imazr z8(nqRuN@U9H=bM(?oU;m|K?VJB07r3OZF!fZ<7_bZ9QXEk5`umUd`z`YiN1py2PhU zv+%~mfN7y-8K%)e`?`-)CYUa~vtx5vZQsEh#_76}sUt3wrcIw!T5~Rx#*6zL5ypy~ zxU4Qx^B46Ac40-stlE54pWNZWCUaiaVhdAupV$I}sqSHsPo`K-Ip3}W=dgK%a8z_; z-t6Mgz54%r zw0s_a-PsKJ&%EQe+ZlYdYe`Hg|4a|RW4``%$F4t3*q-v^0w5y1oe3#u#u1^ZPw}-4 zNrTEQ!)t3ET$}RmhgHjV-j2$b-Q+fn@lCSPH*P%e(kIt%VxGg9?U7cQCa zek7jJtupU9jTg^BL>S6W+rp#gR-7Gk`S2{SLpG9Ai!wZhZA>m2alT}pMU&^)fl9Zd zRTH?+Zttk`3}SQYL>DpwEWaI3Tm+;bD;NGke_* zOJAj#Zid@eOt9lWm*yEg)XMJyC^7ylT*@qWxnjYI`CVN@J4@uVbRY%8|hdheQ zpz)IP2+}in!*a@~?(6csUmQw2G&^zj#(b}ei&1%T^=B6L-TnOhwW1@IOT-T|0_U0x z=)2;_d(}oMzfF3?`gH=ky0=}_udl`;$tM=vf62Ft2DpaxO!cU)|N96iHSfUdHQK;UE(>OfrE-NpB4Lp)=&Z>yk!YwD zmt!Rt$B80C?5ucJ0*N?O8W}>)xQ$I}!HD6$R;VHD)=eEl1Gi`g?ohzJhXL0wT)Xkz z4Bw;hod@4f@ZB5FH~3D0=YKq_qfI;m<9j-uQSdy0dq3{ec-F*oES?R$HMS3F=>z(W zT!aH{pik%<+C%Kpu0tH0lq@8SA70ro6D%ic*974T9?zMw-5pXJE1=QQt$c03diQPILZ&e9wm(dnXo;ONNaxNh`3SFIf9PDXJ)`es-pvE3N z;I-smNIpZw#$`EC6v@Pp%!aOx1Ir%CnsYNG*8yx`A41Ta-Fe#%Ie(B?o9PS!CKQsn zA(;?e9ZjeylDi?fkd_*f%^}&5mKu`xA$gRR8j=|znHF80D=7(30UIHLc!OvyzNF(Mgj)Lex~CKX7&P{_p6 z(4kiLEfa{}Jb|WZm>%^}KM`1$V6NG}bE*wj#72qaY#i*G%Q^Dl>W9k?h9M(WM9QO? zb3{R`aCw+Q5hWkOX3IpOkl~ZXvZPTF;c`}_ER@~Sh|Q!Gn5ucfQiR5}f<@baLd8_g zFl-NrPJt|zDHjQ3!C_1Vtj(}c3nY-{6oiVRr9zQR9vmqXwL;s{2*_IsU;-wJ#soV! zLfCr8)N>FR)E7*gfKDIi5XpfeN|O!+t6W$-S`P%RhXAFO0Bz5K8TkRYScEXKh^d;` z*jgMBArguOiq_)P+y=nS*FeGfU_ak}MnLiQO^6v6qSm_es+AMqDy>35k6VYRQ^$Z& zUoh3{_6I db.loadDatabase(resolve)); + +async function getUserToken() { + const rlInterface = createInterface({ + input: process.stdin, + output: process.stdout, + }); + + let params = new URLSearchParams(); + params.set("client_id", process.env.OSU_CLIENT_ID); + params.set("redirect_uri", redirectUri); + params.set("response_type", "code"); + params.set( + "scope", + [ + "chat.read", + "chat.write", + "chat.write_manage", + "friends.read", + "identify", + "public", + ].join(" "), + ); + console.log(`https://osu.ppy.sh/oauth/authorize?${params.toString()}`); + const answer = await new Promise((resolve) => { + rlInterface.question("Code: ", (answer) => { + resolve(answer); + rlInterface.close(); + }); + }); + + params = new URLSearchParams(); + params.set("client_id", process.env.OSU_CLIENT_ID); + params.set("client_secret", process.env.OSU_CLIENT_SECRET); + params.set("code", answer); + params.set("grant_type", "authorization_code"); + params.set("redirect_uri", redirectUri); + const resp = await fetch("https://osu.ppy.sh/oauth/token", { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: params.toString(), + }); + const data = await resp.json(); + console.log("data", data); + return data.access_token; +} + +const token = process.env.OSU_TOKEN ?? (await getUserToken()); +const headers = { Authorization: `Bearer ${token}` }; + +async function fetchApi(url: string, init?: RequestInit) { + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }; + const resp = await fetch(`https://osu.ppy.sh/api/v2${url}`, { + headers, + ...(init ?? {}), + }); + const data = await resp.json(); + return data; +} + +const client = new Client(token); + +function sleep(time): Promise { + return new Promise((resolve) => { + setTimeout(() => resolve(), time); + }); +} + +// async function listen() { +// const url = "https://osu.ppy.sh/api/v2/notifications"; +// const ws = new WebSocket(url, [], { headers }); +// ws.on("message", (data) => { +// console.log("data", data.toString()); +// }); +// ws.send(JSON.stringify({ event: "chat.start" })); +// } + +async function ensureBeatmap(beatmap, beatmapSet) { + try { + { + const { + id, + artist, + artist_unicode: artistUnicode, + title, + title_unicode: titleUnicode, + ranked, + } = beatmapSet; + const add = { + artist, + artistUnicode, + title, + titleUnicode, + ranked: ranked === 1, + }; + await prisma.beatmapSet.upsert({ + where: { id }, + create: { ...add, id }, + update: add, + }); + } + + { + const { id, version: difficulty, ...rest } = beatmap; + const add = { difficulty, beatmapset_id: beatmapSet.id }; + await prisma.beatmap.upsert({ + where: { id }, + create: { ...add, id }, + update: add, + }); + } + } catch (e) { + console.log("failed on", beatmapSet.id, beatmap.id, e.message); + } +} + +async function scrapeUser(userId) { + const scores: Score[] = await fetchApi( + `/users/${userId}/scores/recent?include_fails=1&limit=50&mode=osu`, + ); + + if (!Array.isArray(scores)) return; + + const newScores = await Promise.all( + scores.map(async (score) => { + const core = { + user_id: score.user_id, + beatmap_id: score.beatmap.id, + created_at: score.created_at, + score: score.score, + }; + const add = { + beatmapset_id: score.beatmapset.id, + accuracy: score.accuracy, + score_id: score.id, + best_id: score.best_id, + }; + await ensureBeatmap(score.beatmap, score.beatmapset); + return await prisma.score.upsert({ + where: { user_id_beatmap_id_created_at_score: core }, + create: { ...core, ...add }, + update: { ...add }, + }); + }), + ); + + newScores.sort((a, b) => a.created_at.getTime() - b.created_at.getTime()); + + for (let i = 1; i < newScores.length; ++i) { + const prevScore = newScores[i - 1]; + const currScore = newScores[i]; + const msBetween = + currScore.created_at.getTime() - prevScore.created_at.getTime(); + + const core = { before_id: prevScore.id, after_id: currScore.id }; + const add = { ms_between: msBetween, user_id: currScore.user_id }; + await prisma.transition.upsert({ + where: { before_id_after_id: core }, + create: { ...core, ...add }, + update: add, + }); + } +} + +async function scrapeSingle(channelId) { + const messages = + (await fetchApi(`/chat/channels/${channelId}/messages?limit=50`)) ?? []; + + if (!Array.isArray(messages)) return; + + const userIds = messages + .map((msg) => msg.sender_id) + .filter((id) => Number.isInteger(id)); + await Promise.all(userIds.map((userId) => scrapeUser(userId))); +} + +async function scrapeChannels() { + const channels: Channel[] = await fetchApi("/chat/channels"); + // // biome-ignore lint/style/noNonNullAssertion: + // const osuChannel = channels.find((channel) => channel.name === "#osu")!; + // const { channel_id: osuChannelId } = osuChannel; + await Promise.all( + channels.map((channel) => scrapeSingle(channel.channel_id)), + ); +} + +async function mainLoop() { + while (true) { + await scrapeChannels(); + await sleep(10000); + } +} + +mainLoop(); diff --git a/package.json b/package.json new file mode 100644 index 0000000..d81f46e --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "dependencies": { + "@prisma/client": "^5.14.0", + "dotenv": "^16.4.5", + "nedb": "^1.8.0", + "osu-web.js": "^2.4.0" + }, + "devDependencies": { + "@types/nedb": "^1.8.16", + "@types/ws": "^8.5.10", + "prisma": "^5.14.0" + } +} diff --git a/prisma/migrations/20240530161311_initial/migration.sql b/prisma/migrations/20240530161311_initial/migration.sql new file mode 100644 index 0000000..4a817f3 --- /dev/null +++ b/prisma/migrations/20240530161311_initial/migration.sql @@ -0,0 +1,25 @@ +-- CreateTable +CREATE TABLE "Score" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "accuracy" REAL NOT NULL, + "best_id" INTEGER, + "created_at" DATETIME NOT NULL, + "score_id" INTEGER, + "score" INTEGER NOT NULL, + "beatmap_id" INTEGER NOT NULL, + "beatmapset_id" INTEGER NOT NULL, + "user_id" INTEGER NOT NULL +); + +-- CreateTable +CREATE TABLE "Transition" ( + "before_id" INTEGER NOT NULL, + "after_id" INTEGER NOT NULL, + + PRIMARY KEY ("before_id", "after_id"), + CONSTRAINT "Transition_before_id_fkey" FOREIGN KEY ("before_id") REFERENCES "Score" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "Transition_after_id_fkey" FOREIGN KEY ("after_id") REFERENCES "Score" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); + +-- CreateIndex +CREATE UNIQUE INDEX "Score_user_id_beatmap_id_created_at_score_key" ON "Score"("user_id", "beatmap_id", "created_at", "score"); diff --git a/prisma/migrations/20240530162224_bigint/migration.sql b/prisma/migrations/20240530162224_bigint/migration.sql new file mode 100644 index 0000000..8ee82ee --- /dev/null +++ b/prisma/migrations/20240530162224_bigint/migration.sql @@ -0,0 +1,26 @@ +/* + Warnings: + + - You are about to alter the column `best_id` on the `Score` table. The data in that column could be lost. The data in that column will be cast from `Int` to `BigInt`. + - You are about to alter the column `score_id` on the `Score` table. The data in that column could be lost. The data in that column will be cast from `Int` to `BigInt`. + +*/ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Score" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "accuracy" REAL NOT NULL, + "best_id" BIGINT, + "created_at" DATETIME NOT NULL, + "score_id" BIGINT, + "score" INTEGER NOT NULL, + "beatmap_id" INTEGER NOT NULL, + "beatmapset_id" INTEGER NOT NULL, + "user_id" INTEGER NOT NULL +); +INSERT INTO "new_Score" ("accuracy", "beatmap_id", "beatmapset_id", "best_id", "created_at", "id", "score", "score_id", "user_id") SELECT "accuracy", "beatmap_id", "beatmapset_id", "best_id", "created_at", "id", "score", "score_id", "user_id" FROM "Score"; +DROP TABLE "Score"; +ALTER TABLE "new_Score" RENAME TO "Score"; +CREATE UNIQUE INDEX "Score_user_id_beatmap_id_created_at_score_key" ON "Score"("user_id", "beatmap_id", "created_at", "score"); +PRAGMA foreign_key_check("Score"); +PRAGMA foreign_keys=ON; diff --git a/prisma/migrations/20240530162755_a/migration.sql b/prisma/migrations/20240530162755_a/migration.sql new file mode 100644 index 0000000..0246fb8 --- /dev/null +++ b/prisma/migrations/20240530162755_a/migration.sql @@ -0,0 +1,22 @@ +/* + Warnings: + + - Added the required column `ms_between` to the `Transition` table without a default value. This is not possible if the table is not empty. + +*/ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Transition" ( + "before_id" INTEGER NOT NULL, + "after_id" INTEGER NOT NULL, + "ms_between" BIGINT NOT NULL, + + PRIMARY KEY ("before_id", "after_id"), + CONSTRAINT "Transition_before_id_fkey" FOREIGN KEY ("before_id") REFERENCES "Score" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "Transition_after_id_fkey" FOREIGN KEY ("after_id") REFERENCES "Score" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_Transition" ("after_id", "before_id") SELECT "after_id", "before_id" FROM "Transition"; +DROP TABLE "Transition"; +ALTER TABLE "new_Transition" RENAME TO "Transition"; +PRAGMA foreign_key_check("Transition"); +PRAGMA foreign_keys=ON; diff --git a/prisma/migrations/20240530165230_a/migration.sql b/prisma/migrations/20240530165230_a/migration.sql new file mode 100644 index 0000000..b9f0d07 --- /dev/null +++ b/prisma/migrations/20240530165230_a/migration.sql @@ -0,0 +1,58 @@ +/* + Warnings: + + - Added the required column `user_id` to the `Transition` table without a default value. This is not possible if the table is not empty. + +*/ +-- CreateTable +CREATE TABLE "Beatmap" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "difficulty" TEXT NOT NULL +); + +-- CreateTable +CREATE TABLE "BeatmapSet" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "artist" TEXT NOT NULL, + "artistUnicode" TEXT NOT NULL, + "title" TEXT NOT NULL, + "titleUnicode" TEXT NOT NULL, + "genre" INTEGER NOT NULL, + "language" INTEGER NOT NULL +); + +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Transition" ( + "before_id" INTEGER NOT NULL, + "after_id" INTEGER NOT NULL, + "user_id" INTEGER NOT NULL, + "ms_between" BIGINT NOT NULL, + + PRIMARY KEY ("before_id", "after_id"), + CONSTRAINT "Transition_before_id_fkey" FOREIGN KEY ("before_id") REFERENCES "Score" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "Transition_after_id_fkey" FOREIGN KEY ("after_id") REFERENCES "Score" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_Transition" ("after_id", "before_id", "ms_between") SELECT "after_id", "before_id", "ms_between" FROM "Transition"; +DROP TABLE "Transition"; +ALTER TABLE "new_Transition" RENAME TO "Transition"; +CREATE TABLE "new_Score" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "accuracy" REAL NOT NULL, + "best_id" BIGINT, + "created_at" DATETIME NOT NULL, + "score_id" BIGINT, + "score" INTEGER NOT NULL, + "beatmap_id" INTEGER NOT NULL, + "beatmapset_id" INTEGER NOT NULL, + "user_id" INTEGER NOT NULL, + CONSTRAINT "Score_beatmap_id_fkey" FOREIGN KEY ("beatmap_id") REFERENCES "Beatmap" ("id") ON DELETE RESTRICT ON UPDATE CASCADE, + CONSTRAINT "Score_beatmapset_id_fkey" FOREIGN KEY ("beatmapset_id") REFERENCES "BeatmapSet" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_Score" ("accuracy", "beatmap_id", "beatmapset_id", "best_id", "created_at", "id", "score", "score_id", "user_id") SELECT "accuracy", "beatmap_id", "beatmapset_id", "best_id", "created_at", "id", "score", "score_id", "user_id" FROM "Score"; +DROP TABLE "Score"; +ALTER TABLE "new_Score" RENAME TO "Score"; +CREATE UNIQUE INDEX "Score_user_id_beatmap_id_created_at_score_key" ON "Score"("user_id", "beatmap_id", "created_at", "score"); +PRAGMA foreign_key_check("Transition"); +PRAGMA foreign_key_check("Score"); +PRAGMA foreign_keys=ON; diff --git a/prisma/migrations/20240530165947_a/migration.sql b/prisma/migrations/20240530165947_a/migration.sql new file mode 100644 index 0000000..8a6ae89 --- /dev/null +++ b/prisma/migrations/20240530165947_a/migration.sql @@ -0,0 +1,34 @@ +/* + Warnings: + + - You are about to drop the column `genre` on the `BeatmapSet` table. All the data in the column will be lost. + - You are about to drop the column `language` on the `BeatmapSet` table. All the data in the column will be lost. + - Added the required column `beatmapset_id` to the `Beatmap` table without a default value. This is not possible if the table is not empty. + - Added the required column `ranked` to the `BeatmapSet` table without a default value. This is not possible if the table is not empty. + +*/ +-- RedefineTables +PRAGMA foreign_keys=OFF; +CREATE TABLE "new_Beatmap" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "difficulty" TEXT NOT NULL, + "beatmapset_id" INTEGER NOT NULL, + CONSTRAINT "Beatmap_beatmapset_id_fkey" FOREIGN KEY ("beatmapset_id") REFERENCES "BeatmapSet" ("id") ON DELETE RESTRICT ON UPDATE CASCADE +); +INSERT INTO "new_Beatmap" ("difficulty", "id") SELECT "difficulty", "id" FROM "Beatmap"; +DROP TABLE "Beatmap"; +ALTER TABLE "new_Beatmap" RENAME TO "Beatmap"; +CREATE TABLE "new_BeatmapSet" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "artist" TEXT NOT NULL, + "artistUnicode" TEXT NOT NULL, + "title" TEXT NOT NULL, + "titleUnicode" TEXT NOT NULL, + "ranked" BOOLEAN NOT NULL +); +INSERT INTO "new_BeatmapSet" ("artist", "artistUnicode", "id", "title", "titleUnicode") SELECT "artist", "artistUnicode", "id", "title", "titleUnicode" FROM "BeatmapSet"; +DROP TABLE "BeatmapSet"; +ALTER TABLE "new_BeatmapSet" RENAME TO "BeatmapSet"; +PRAGMA foreign_key_check("Beatmap"); +PRAGMA foreign_key_check("BeatmapSet"); +PRAGMA foreign_keys=ON; diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..e5e5c47 --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "sqlite" \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma new file mode 100644 index 0000000..e25a962 --- /dev/null +++ b/prisma/schema.prisma @@ -0,0 +1,64 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "sqlite" + url = env("DATABASE_URL") +} + +model Beatmap { + id Int @id + difficulty String + + beatmapset BeatmapSet @relation(fields: [beatmapset_id], references: [id]) + beatmapset_id Int + + scores Score[] +} + +model BeatmapSet { + id Int @id + artist String + artistUnicode String + title String + titleUnicode String + ranked Boolean + + scores Score[] + beatmaps Beatmap[] +} + +model Score { + id Int @id @default(autoincrement()) + accuracy Float + best_id BigInt? + created_at DateTime + score_id BigInt? + score Int + beatmap Beatmap @relation(fields: [beatmap_id], references: [id]) + beatmap_id Int + beatmapset BeatmapSet @relation(fields: [beatmapset_id], references: [id]) + beatmapset_id Int + user_id Int + + transition_from Transition[] @relation("before") + transition_to Transition[] @relation("after") + + @@unique([user_id, beatmap_id, created_at, score]) +} + +model Transition { + before Score @relation("before", fields: [before_id], references: [id]) + before_id Int + after Score @relation("after", fields: [after_id], references: [id]) + after_id Int + + user_id Int + ms_between BigInt + + @@id([before_id, after_id]) +} diff --git a/test.py b/test.py new file mode 100644 index 0000000..c24bf05 --- /dev/null +++ b/test.py @@ -0,0 +1,48 @@ +from surprise import Dataset, SVD +import numpy as np +import pandas as pd +import sqlite3 +from lightfm import LightFM +from lightfm.datasets import fetch_movielens + +c = sqlite3.connect("./prisma/dev.db") +df = pd.read_sql_query( + """ + SELECT + s1.user_id as user_id, + s1.beatmap_id as before_beatmap_id, + s2.beatmap_id as after_beatmap_id, + ms_between + FROM Transition as t + JOIN Score as s1 ON s1.id = t.before_id + JOIN Score as s2 ON s2.id = t.after_id + """, + c, +) +print(df) + +# Beatmaps with most data: +""" +SELECT + beatmapset_id, artist, title, COUNT(*) as count +FROM Score +JOIN BeatmapSet ON Score.beatmapset_id = BeatmapSet.id +GROUP BY beatmapset_id +ORDER BY count DESC; +""" + +# Given a specific beatmap, what maps do they go on to +""" +SELECT + bs1.artist, bs1.title, b1.difficulty, bs2.artist, bs2.title, b2.difficulty, COUNT(*) as count +FROM Transition +JOIN Score as s1 ON s1.id = Transition.before_id +JOIN Score as s2 ON s2.id = Transition.after_id +JOIN Beatmap as b1 on s1.beatmap_id = b1.id +JOIN BeatmapSet as bs1 on s1.beatmapset_id = bs1.id +JOIN Beatmap as b2 on s2.beatmap_id = b2.id +JOIN BeatmapSet as bs2 on s2.beatmapset_id = bs2.id +WHERE s1.beatmapset_id = 320118 +GROUP BY s2.beatmap_id +ORDER BY count DESC; +"""