package scrape import ( "fmt" "log" "subscribe-bot/db" "subscribe-bot/osuapi" "time" ) func (s *Scraper) scrapePendingMaps() { // build a list of currently tracked mappers trackedMappers := make(map[int]int) s.db.IterAllTrackedMappers(func(mapper db.User) error { trackedMappers[mapper.ID] = 1 return nil }) // TODO: is this sorted for sure?? pendingSets, err := s.api.SearchBeatmaps("pending") if err != nil { log.Println("error fetching pending sets", err) s.bot.NotifyError("failed to fetch pending sets", err) return } allNewMaps := make(map[int][]osuapi.Beatmapset, 0) var newLastUpdateTime = time.Unix(0, 0) for _, beatmapSet := range pendingSets.Beatmapsets { updatedTime, err := time.Parse(time.RFC3339, beatmapSet.LastUpdated) if err != nil { log.Println("error parsing last updated time", updatedTime) } if updatedTime.After(newLastUpdateTime) { // update lastUpdateTime to latest updated map newLastUpdateTime = updatedTime } if !updatedTime.After(lastUpdateTime) { break } mapperId := beatmapSet.UserID if _, ok := trackedMappers[mapperId]; ok { if _, ok2 := allNewMaps[mapperId]; !ok2 { allNewMaps[mapperId] = make([]osuapi.Beatmapset, 0) } allNewMaps[mapperId] = append(allNewMaps[mapperId], beatmapSet) } } if len(allNewMaps) > 0 { for mapperId, newMaps := range allNewMaps { channels := make([]string, 0) s.db.IterTrackingChannels(mapperId, func(channel db.DiscordChannel) error { channels = append(channels, channel.ID) return nil }) err := s.bot.NotifyNewBeatmap(channels, newMaps) if err != nil { log.Println("error notifying new maps:", err) } } } lastUpdateTime = newLastUpdateTime log.Println("last updated time", lastUpdateTime) } func getNewMaps(db *db.Db, api *osuapi.Osuapi, userId int) (newMaps []osuapi.Event, err error) { // see if there's a last event hasLastEvent, lastEventId := db.MapperLastEvent(userId) newMaps = make([]osuapi.Event, 0) var ( events []osuapi.Event newLatestEvent = 0 updateLatestEvent = false ) if hasLastEvent { offset := 0 loop: for { events, err = api.GetUserEvents(userId, 50, offset) if err != nil { err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err) return } if len(events) == 0 { break } for _, event := range events { if event.ID == lastEventId { break loop } if event.ID > newLatestEvent { updateLatestEvent = true newLatestEvent = event.ID } if event.Type == "beatmapsetUpload" || event.Type == "beatmapsetRevive" || event.Type == "beatmapsetUpdate" { newMaps = append(newMaps, event) } } offset += len(events) } } else { log.Printf("no last event id found for %d\n", userId) events, err = api.GetUserEvents(userId, 50, 0) if err != nil { return } for _, event := range events { if event.ID > newLatestEvent { updateLatestEvent = true newLatestEvent = event.ID } if event.Type == "beatmapsetUpload" || event.Type == "beatmapsetRevive" || event.Type == "beatmapsetUpdate" { newMaps = append(newMaps, event) } } } if updateLatestEvent { err = db.UpdateMapperLastEvent(userId, newLatestEvent) if err != nil { return } } return }