osu/subscribe-bot
1
0
Fork 0
Code Issues 3 Pull requests Releases Wiki Activity

don't use golang's default http client

Browse source
This commit is contained in:
Michael Zhang 2020-10-12 01:33:45 -05:00
parent 9d24724eb9
commit df9faa9479
Signed by: michael
GPG key ID: BDA47A31A3C8EE6B
2 changed files with 8 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
osuapi.go
View file

@ -18,6 +18,7 @@ import (
const BASE_URL = "https://osu.ppy.sh/api/v2" const BASE_URL = "https://osu.ppy.sh/api/v2"
type Osuapi struct { type Osuapi struct {
httpClient *http.Client
lock *semaphore.Weighted lock *semaphore.Weighted
token string token string
expires time.Time expires time.Time
@ -26,9 +27,13 @@ type Osuapi struct {
} }
func NewOsuapi(config *Config) *Osuapi { func NewOsuapi(config *Config) *Osuapi {
client := &http.Client{
Timeout: 10 * time.Second,
}
// want to cap at around 1000 requests a minute, OSU cap is 1200 // want to cap at around 1000 requests a minute, OSU cap is 1200
lock := semaphore.NewWeighted(1000) lock := semaphore.NewWeighted(1000)
return &Osuapi{lock, "", time.Now(), config.ClientId, config.ClientSecret} return &Osuapi{client, lock, "", time.Now(), config.ClientId, config.ClientSecret}
} }
func (api *Osuapi) Token() (token string, err error) { func (api *Osuapi) Token() (token string, err error) {
@ -84,7 +89,7 @@ func (api *Osuapi) Request0(action string, url string) (resp *http.Response, err
return return
} }
resp, err = http.DefaultClient.Do(req) resp, err = api.httpClient.Do(req)
if err != nil { if err != nil {
return return
} }

5
scrape.go
View file

@ -15,11 +15,10 @@ func RunScraper(bot *Bot, db *Db, api *Osuapi, requests chan int) {
go startTimers(db, requests) go startTimers(db, requests)
for userId := range requests { for userId := range requests {
log.Println("scraping", userId) log.Println("scraping user", userId)
newMaps, err := getNewMaps(db, api, userId) newMaps, err := getNewMaps(db, api, userId)
if err != nil { if err != nil {
log.Println("err getting new maps:", err) log.Println("err getting new maps:", err)
exit_chan <- 1
} }
db.IterTrackingChannels(userId, func(channelId string) error { db.IterTrackingChannels(userId, func(channelId string) error {
@ -45,12 +44,10 @@ func getNewMaps(db *Db, api *Osuapi, userId int) (newMaps []Event, err error) {
updateLatestEvent = false updateLatestEvent = false
) )
if hasLastEvent { if hasLastEvent {
log.Printf("last event id for %d is %d\n", userId, lastEventId)
offset := 0 offset := 0
loop: loop:
for { for {
log.Println("loading user events from", offset)
events, err = api.GetUserEvents(userId, 50, offset) events, err = api.GetUserEvents(userId, 50, offset)
if err != nil { if err != nil {
err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err) err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err)