diff --git a/osuapi.go b/osuapi.go index 4d6c8e6..d9a1fd5 100644 --- a/osuapi.go +++ b/osuapi.go @@ -18,6 +18,7 @@ import ( const BASE_URL = "https://osu.ppy.sh/api/v2" type Osuapi struct { + httpClient *http.Client lock *semaphore.Weighted token string expires time.Time @@ -26,9 +27,13 @@ type Osuapi struct { } func NewOsuapi(config *Config) *Osuapi { + client := &http.Client{ + Timeout: 10 * time.Second, + } + // want to cap at around 1000 requests a minute, OSU cap is 1200 lock := semaphore.NewWeighted(1000) - return &Osuapi{lock, "", time.Now(), config.ClientId, config.ClientSecret} + return &Osuapi{client, lock, "", time.Now(), config.ClientId, config.ClientSecret} } func (api *Osuapi) Token() (token string, err error) { @@ -84,7 +89,7 @@ func (api *Osuapi) Request0(action string, url string) (resp *http.Response, err return } - resp, err = http.DefaultClient.Do(req) + resp, err = api.httpClient.Do(req) if err != nil { return } diff --git a/scrape.go b/scrape.go index 7fff65f..51b49b6 100644 --- a/scrape.go +++ b/scrape.go @@ -15,11 +15,10 @@ func RunScraper(bot *Bot, db *Db, api *Osuapi, requests chan int) { go startTimers(db, requests) for userId := range requests { - log.Println("scraping", userId) + log.Println("scraping user", userId) newMaps, err := getNewMaps(db, api, userId) if err != nil { log.Println("err getting new maps:", err) - exit_chan <- 1 } db.IterTrackingChannels(userId, func(channelId string) error { @@ -45,12 +44,10 @@ func getNewMaps(db *Db, api *Osuapi, userId int) (newMaps []Event, err error) { updateLatestEvent = false ) if hasLastEvent { - log.Printf("last event id for %d is %d\n", userId, lastEventId) offset := 0 loop: for { - log.Println("loading user events from", offset) events, err = api.GetUserEvents(userId, 50, offset) if err != nil { err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err)