import "dotenv/config"; import Datastore from "nedb"; import WebSocket from "ws"; import { createInterface } from "readline"; import { Client, type Channel, type Score } from "osu-web.js"; import { PrismaClient } from "@prisma/client"; const prisma = new PrismaClient(); const redirectUri = "http://localhost:3000/auth/callback"; async function getUserToken() { const rlInterface = createInterface({ input: process.stdin, output: process.stdout, }); let params = new URLSearchParams(); params.set("client_id", process.env.OSU_CLIENT_ID); params.set("redirect_uri", redirectUri); params.set("response_type", "code"); params.set( "scope", [ "chat.read", "chat.write", "chat.write_manage", "friends.read", "identify", "public", ].join(" "), ); console.log(`https://osu.ppy.sh/oauth/authorize?${params.toString()}`); const answer = await new Promise((resolve) => { rlInterface.question("Code: ", (answer) => { resolve(answer); rlInterface.close(); }); }); params = new URLSearchParams(); params.set("client_id", process.env.OSU_CLIENT_ID); params.set("client_secret", process.env.OSU_CLIENT_SECRET); params.set("code", answer); params.set("grant_type", "authorization_code"); params.set("redirect_uri", redirectUri); const resp = await fetch("https://osu.ppy.sh/oauth/token", { method: "POST", headers: { "Content-Type": "application/x-www-form-urlencoded" }, body: params.toString(), }); const data = await resp.json(); console.log("data", data); return data.access_token; } const token = process.env.OSU_TOKEN ?? (await getUserToken()); const headers = { Authorization: `Bearer ${token}` }; async function fetchApi(url: string, init?: RequestInit) { const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json", }; const resp = await fetch(`https://osu.ppy.sh/api/v2${url}`, { headers, ...(init ?? {}), }); const data = await resp.json(); return data; } const client = new Client(token); function sleep(time): Promise { return new Promise((resolve) => { setTimeout(() => resolve(), time); }); } // async function listen() { // const url = "https://osu.ppy.sh/api/v2/notifications"; // const ws = new WebSocket(url, [], { headers }); // ws.on("message", (data) => { // console.log("data", data.toString()); // }); // ws.send(JSON.stringify({ event: "chat.start" })); // } async function ensureBeatmap(beatmap, beatmapSet) { try { { const { id, artist, artist_unicode: artistUnicode, title, title_unicode: titleUnicode, ranked, } = beatmapSet; const add = { artist, artistUnicode, title, titleUnicode, ranked: ranked === 1, }; await prisma.beatmapSet.upsert({ where: { id }, create: { ...add, id }, update: add, }); } { const { id, version: difficulty, ...rest } = beatmap; const add = { difficulty, beatmapset_id: beatmapSet.id }; await prisma.beatmap.upsert({ where: { id }, create: { ...add, id }, update: add, }); } } catch (e) { console.log("failed on", beatmapSet.id, beatmap.id, e.message); } } async function scrapeUser(userId) { const scores: Score[] = await fetchApi( `/users/${userId}/scores/recent?include_fails=1&limit=50&mode=osu`, ); if (!Array.isArray(scores)) return; const newScores = await Promise.all( scores.map(async (score) => { const core = { user_id: score.user_id, beatmap_id: score.beatmap.id, created_at: score.created_at, score: score.score, }; const add = { beatmapset_id: score.beatmapset.id, accuracy: score.accuracy, score_id: score.id, best_id: score.best_id, }; await ensureBeatmap(score.beatmap, score.beatmapset); return await prisma.score.upsert({ where: { user_id_beatmap_id_created_at_score: core }, create: { ...core, ...add }, update: { ...add }, }); }), ); newScores.sort((a, b) => a.created_at.getTime() - b.created_at.getTime()); for (let i = 1; i < newScores.length; ++i) { const prevScore = newScores[i - 1]; const currScore = newScores[i]; const msBetween = currScore.created_at.getTime() - prevScore.created_at.getTime(); const core = { before_id: prevScore.id, after_id: currScore.id }; const add = { ms_between: msBetween, user_id: currScore.user_id }; await prisma.transition.upsert({ where: { before_id_after_id: core }, create: { ...core, ...add }, update: add, }); } } async function scrapeSingle(channelId) { const messages = (await fetchApi(`/chat/channels/${channelId}/messages?limit=50`)) ?? []; if (!Array.isArray(messages)) return; const userIds = messages .map((msg) => msg.sender_id) .filter((id) => Number.isInteger(id)); await Promise.all(userIds.map((userId) => scrapeUser(userId))); } async function scrapeChannels() { const channels: Channel[] = await fetchApi("/chat/channels"); if (!Array.isArray(channels)) return; await Promise.all( channels.map((channel) => scrapeSingle(channel.channel_id)), ); } async function getStats(elapsed) { // console.clear(); const result = await prisma.score.count(); const users = (await prisma.score.groupBy({ by: ["user_id"] })).length; console.log( `${result} total scores, ${users} total users, (prev query: ${ Math.round(elapsed / 10) / 100 }s)`, ); const result2 = await prisma.$queryRaw` SELECT beatmapset_id, artist, title, COUNT(*) as count FROM Score JOIN BeatmapSet ON Score.beatmapset_id = BeatmapSet.id GROUP BY beatmapset_id ORDER BY count DESC LIMIT 5; `; for (const row of result2) { console.log(` ${row.count}\t${row.artist} - ${row.title}`); } } while (true) { const start = performance.now(); await scrapeChannels(); const end = performance.now(); await getStats(end - start); await Bun.sleep(10000); }