.gitignore

@@ -1,5 +1,4 @@
 /subscribe-bot
 /test.db
-/test.db-sqlite
 /config.toml
 /repos

Makefile

@@ -1,12 +0,0 @@
-GIT_COMMIT = $(shell git rev-parse --short HEAD)
-SOURCES = $(shell find . -type f -and -name "*.go" -or -name "*.html")
-
-all: subscribe-bot
-
-subscribe-bot: $(SOURCES)
-	go build -o $@ -ldflags "-X main.GitCommit=$(GIT_COMMIT)"
-
-checkFmt:
-	[ -z "$$(git ls-files | grep '\.go$$' | xargs gofmt -l)" ] || (exit 1)
-
-.PHONY: all checkFmt

README.md

@@ -4,9 +4,8 @@ subscribe-bot
 Subscribes to OSU map updates and stores versions for later review.
 
 Please don't run a separate bot, the official one is `subscribe-bot#8789`. If
-you want to contribute or test the bot, instructions on how to run it are below.
+you want to contribute or test the bot, then here are instructions on how to
-
+run it:
-Join the [Discord][2]
 
 How to run
 ----------
@@ -23,19 +22,9 @@ How to run
 1. Run the executable, passing `-config {path}` in case you want to use a
    different config file than `config.toml`.
 
-Architecture
-------------
-
-There's several independent services running within:
-
-- Discord bot, which posts updates to the relevant channels in discord
-- Web server, which hosts an HTTP server allowing you to view changes
-- Scraper, which actually polls the OSU API for new updates
-
 License
 -------
 
 [GPL3][1]
 
 [1]: https://www.gnu.org/licenses/gpl-3.0.en.html
-[2]: https://discord.gg/eqjVG2H

config/config.go

@@ -19,15 +19,13 @@ type Config struct {
 }
 
 type OauthConfig struct {
-	ClientId     string `toml:"client_id"`
+	ClientId     int    `toml:"client_id"`
 	ClientSecret string `toml:"client_secret"`
 }
 
 type WebConfig struct {
-	Host          string `toml:"host"`
+	Host string `toml:"host"`
-	Port          int    `toml:"port"`
+	Port int    `toml:"port"`
-	ServedAt      string `toml:"served_at"`
-	SessionSecret string `toml:"session_secret"`
 }
 
 func ReadConfig(path string) (config Config, err error) {

db/beatmap.go

@@ -1,17 +0,0 @@
-package db
-
-import (
-	"time"
-
-	"gorm.io/gorm"
-)
-
-type Beatmapset struct {
-	gorm.Model
-	ID          int `gorm:"primaryKey"`
-	Artist      string
-	Title       string
-	MapperID    int       `gorm:"mapper_id"`
-	Mapper      User      `gorm:"foreignKey:MapperID"`
-	LastUpdated time.Time `gorm:"last_updated"`
-}

db/channel.go

@@ -1,9 +0,0 @@
-package db
-
-import "gorm.io/gorm"
-
-type DiscordChannel struct {
-	gorm.Model
-	ID             string `gorm:"primaryKey"`
-	TrackedMappers []User `gorm:"many2many:tracked_mappers"`
-}

db/config.go

@@ -1,9 +0,0 @@
-package db
-
-import "gorm.io/gorm"
-
-type Config struct {
-	gorm.Model
-	Key   string `gorm:"key;primaryKey"`
-	Value string `gorm:"value"`
-}

db/db.go

@@ -8,175 +8,235 @@ package db
 import (
 	"strconv"
 
-	"github.com/pkg/errors"
+	bolt "go.etcd.io/bbolt"
-	"gorm.io/driver/sqlite"
-	"gorm.io/gorm"
-	"gorm.io/gorm/clause"
-	"gorm.io/gorm/logger"
 
-	"subscribe-bot/config"
 	"subscribe-bot/osuapi"
 )
 
 var (
 	LATEST_EVENT = []byte("latestEvent")
-	MAPPERS      = []byte("mapper")
-	CHANNELS     = []byte("channels")
 )
 
 type Db struct {
-	gorm *gorm.DB
+	*bolt.DB
-	api  *osuapi.Osuapi
+	api *osuapi.Osuapi
 }
 
-func OpenDb(cfg config.Config, api *osuapi.Osuapi) (db *Db, err error) {
+func OpenDb(path string, api *osuapi.Osuapi) (db *Db, err error) {
-	gormConfig := &gorm.Config{}
+	inner, err := bolt.Open(path, 0666, nil)
-	if cfg.Debug {
+	db = &Db{inner, api}
-		gormConfig.Logger = logger.Default.LogMode(logger.Info)
-	}
-
-	gorm, err := gorm.Open(sqlite.Open(cfg.DatabasePath), gormConfig)
-	if err != nil {
-		panic("failed to connect database")
-	}
-
-	// auto-migrate
-	gorm.AutoMigrate(&Config{})
-	gorm.AutoMigrate(&User{})
-	gorm.AutoMigrate(&Beatmapset{})
-	gorm.AutoMigrate(&DiscordChannel{})
-
-	db = &Db{gorm, api}
-	return
-}
-
-func (db *Db) SaveBeatmap(beatmap Beatmapset) {
-	db.gorm.Clauses(clause.OnConflict{UpdateAll: true}).Create(&beatmap)
-	return
-}
-
-func (db *Db) IterTrackedBeatmapsets(limit int, fn func(beatmapset Beatmapset) error) (err error) {
-	var beatmapsets []Beatmapset
-	db.gorm.Preload(clause.Associations).Limit(limit).Order("last_updated desc").Find(&beatmapsets)
-	for _, beatmapset := range beatmapsets {
-		fn(beatmapset)
-	}
-	return
-}
-
-func (db *Db) IterTrackedBeatmapsetsBy(mapperId int, fn func(beatmapset Beatmapset) error) (err error) {
-	var beatmapsets []Beatmapset
-	db.gorm.Where("mapper_id = ?", mapperId).Order("last_updated desc").Find(&beatmapsets)
-	for _, beatmapset := range beatmapsets {
-		fn(beatmapset)
-	}
 	return
 }
 
 // Loop over channels that are tracking this specific mapper
-func (db *Db) IterTrackingChannels(mapperId int, fn func(channel DiscordChannel) error) (err error) {
+func (db *Db) IterTrackingChannels(mapperId int, fn func(channelId string) error) (err error) {
-	var channels []DiscordChannel
+	err = db.DB.View(func(tx *bolt.Tx) error {
-	db.gorm.Model(&User{ID: mapperId}).Association("TrackingChannels").Find(&channels)
+		mapper := getMapper(tx, mapperId)
-	for _, channel := range channels {
+		if mapper == nil {
-		fn(channel)
+			return nil
-	}
+		}
+
+		trackers := mapper.Bucket([]byte("trackers"))
+		if trackers == nil {
+			return nil
+		}
+
+		c := trackers.Cursor()
+		for k, _ := c.First(); k != nil; k, _ = c.Next() {
+			channelId := string(k)
+			err := fn(channelId)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
 
 	return
 }
 
-// Loop over tracked mappers for this channel
+// Loop over tracked mappers
-func (db *Db) IterChannelTrackedMappers(channelId string, fn func(user User) error) (err error) {
+func (db *Db) IterTrackedMappers(fn func(userId int) error) (err error) {
-	var mappers []User
+	err = db.DB.View(func(tx *bolt.Tx) error {
-	db.gorm.Model(&DiscordChannel{ID: channelId}).Association("TrackedMappers").Find(&mappers)
+		mappers := tx.Bucket([]byte("mapper"))
-	for _, mapper := range mappers {
+		if mappers == nil {
-		fn(mapper)
+			return nil
-	}
+		}
+
+		c := mappers.Cursor()
+		for k, _ := c.First(); k != nil; k, _ = c.Next() {
+			userId, err := strconv.Atoi(string(k))
+			if err != nil {
+				return err
+			}
+
+			err = fn(userId)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
 
 	return
 }
 
-// Loop over all tracked mappers
+// Get a list of channels that are tracking this mapper
-func (db *Db) IterAllTrackedMappers(fn func(user User) error) (err error) {
+func (db *Db) GetMapperTrackers(userId int) (trackersList []string) {
-	var mappers []User
+	trackersList = make([]string, 0)
-	db.gorm.Find(&mappers)
+	db.DB.View(func(tx *bolt.Tx) error {
-	for _, mapper := range mappers {
+		mapper, err := getMapperMut(tx, userId)
-		fn(mapper)
-	}
-
-	return
-}
-
-func (db *Db) UpdateMapperLastEvent(userId int, eventId int) (err error) {
-	if eventId == -1 {
-		var events []osuapi.Event
-		events, err = db.api.GetUserEvents(userId, 1, 0)
 		if err != nil {
-			err = errors.Wrap(err, "couldn't get user events from API")
+			return err
-			return
 		}
 
-		if len(events) > 0 {
+		trackers := mapper.Bucket([]byte("trackers"))
-			eventId = events[0].ID
+		if trackers == nil {
+			return nil
 		}
-	}
 
-	db.gorm.Model(&User{}).Where("id = ?", userId).Update("latest_event_id", eventId)
+		c := trackers.Cursor()
-	return nil
+		for k, _ := c.First(); k != nil; k, _ = c.Next() {
+			channelId := string(k)
+			trackersList = append(trackersList, channelId)
+		}
+
+		return nil
+	})
+	return
+}
+
+// Update the latest event of a mapper to the given one
+func (db *Db) UpdateMapperLatestEvent(userId int, eventId int) (err error) {
+	err = db.DB.Update(func(tx *bolt.Tx) error {
+		mapper, err := getMapperMut(tx, userId)
+		if err != nil {
+			return err
+		}
+
+		err = mapper.Put(LATEST_EVENT, []byte(strconv.Itoa(eventId)))
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+	return
 }
 
 // Get the latest event ID of this mapper, if they have one
 func (db *Db) MapperLastEvent(userId int) (has bool, id int) {
-	var user User
+	has = false
-	db.gorm.Select("latest_event_id").First(&user)
+	id = -1
-	return true, user.LatestEventID
+	db.DB.View(func(tx *bolt.Tx) error {
+		mapper := getMapper(tx, userId)
+		if mapper == nil {
+			return nil
+		}
+
+		lastEventId := mapper.Get(LATEST_EVENT)
+		if lastEventId == nil {
+			return nil
+		}
+
+		var err error
+		id, err = strconv.Atoi(string(lastEventId))
+		if err != nil {
+			return nil
+		}
+
+		has = true
+		return nil
+	})
+
+	return
 }
 
 // Start tracking a new mapper (if they're not already tracked)
 func (db *Db) ChannelTrackMapper(channelId string, mapperId int, priority int) (err error) {
-	err = db.gorm.Model(&DiscordChannel{ID: channelId}).Association("TrackedMappers").Append(db.GetUser(mapperId))
+	events, err := db.api.GetUserEvents(mapperId, 1, 0)
 	if err != nil {
-		err = errors.Wrap(err, "could not add tracking for channel "+channelId)
 		return
 	}
 
-	err = db.UpdateMapperLastEvent(mapperId, -1)
+	err = db.Batch(func(tx *bolt.Tx) error {
-	if err != nil {
+		{
-		err = errors.Wrap(err, "could not update mapper latest event")
+			mapper, err := getMapperMut(tx, mapperId)
-		return
+			if err != nil {
-	}
+				return err
+			}
 
-	return nil
+			if len(events) > 0 {
+				latestEventId := strconv.Itoa(events[0].ID)
+				mapper.Put(LATEST_EVENT, []byte(latestEventId))
+			}
+
+			trackers, err := mapper.CreateBucketIfNotExists([]byte("trackers"))
+			if err != nil {
+				return err
+			}
+
+			err = trackers.Put([]byte(channelId), []byte(strconv.Itoa(priority)))
+			if err != nil {
+				return err
+			}
+		}
+		{
+			channels, err := tx.CreateBucketIfNotExists([]byte("channels"))
+			if err != nil {
+				return err
+			}
+
+			channel, err := channels.CreateBucketIfNotExists([]byte(channelId))
+			if err != nil {
+				return err
+			}
+
+			tracks, err := channel.CreateBucketIfNotExists([]byte("tracks"))
+			if err != nil {
+				return err
+			}
+
+			err = tracks.Put([]byte(strconv.Itoa(mapperId)), []byte(strconv.Itoa(priority)))
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	return
 }
 
 func (db *Db) Close() {
+	db.DB.Close()
 }
 
-func (db *Db) GetUser(userId int) (user *User, err error) {
+func getMapper(tx *bolt.Tx, userId int) (mapper *bolt.Bucket) {
-	// TODO: cache user info for some time?
+	mappers := tx.Bucket([]byte("mapper"))
+	if mappers == nil {
+		return nil
+	}
 
-	apiUser, err := db.api.GetUser(strconv.Itoa(userId))
+	mapper = mappers.Bucket([]byte(strconv.Itoa(userId)))
+	if mapper == nil {
+		return nil
+	}
+
+	return
+}
+
+func getMapperMut(tx *bolt.Tx, userId int) (mapper *bolt.Bucket, err error) {
+	mappers, err := tx.CreateBucketIfNotExists([]byte("mapper"))
 	if err != nil {
-		err = errors.Wrap(err, "could not retrieve user from the API")
 		return
 	}
 
-	user = &User{
+	mapper, err = mappers.CreateBucketIfNotExists([]byte(strconv.Itoa(userId)))
-		ID:       userId,
+	if err != nil {
-		Username: apiUser.Username,
+		return
-		Country:  apiUser.CountryCode,
 	}
-	db.gorm.Clauses(clause.OnConflict{UpdateAll: true}).Create(user)
 
 	return
 }
-
-type Stats struct {
-	TotalMaps  int64
-	TotalUsers int64
-}
-
-func (db *Db) GetStats() (stats Stats) {
-	db.gorm.Model(&Beatmapset{}).Count(&stats.TotalMaps)
-	db.gorm.Model(&Beatmapset{}).Group("mapper_id").Count(&stats.TotalUsers)
-	return
-}

db/user.go

@@ -1,13 +0,0 @@
-package db
-
-import "gorm.io/gorm"
-
-type User struct {
-	gorm.Model
-	ID                 int `gorm:"primaryKey"`
-	Username           string
-	Country            string
-	LatestEventID      int              `gorm:"latest_event_id"`
-	TrackingChannels   []DiscordChannel `gorm:"many2many:tracked_mappers"`
-	TrackedBeatmapsets []Beatmapset     `gorm:"foreignKey:MapperID"`
-}

discord/bot.go

@@ -53,12 +53,6 @@ func NewBot(config *config.Config, db *db.Db, api *osuapi.Osuapi) (bot *Bot, err
 	return
 }
 
-func (bot *Bot) NotifyError(format string, args ...interface{}) {
-	msg := fmt.Sprintf(format, args...)
-	channel, _ := bot.UserChannelCreate("100443064228646912")
-	bot.ChannelMessageSend(channel.ID, msg)
-}
-
 func (bot *Bot) errWrap(fn interface{}) interface{} {
 	val := reflect.ValueOf(fn)
 	origType := reflect.TypeOf(fn)
@@ -72,7 +66,10 @@ func (bot *Bot) errWrap(fn interface{}) interface{} {
 		if len(res) > 0 && !res[0].IsNil() {
 			err := res[0].Interface().(error)
 			if err != nil {
-				bot.NotifyError("error: %s", err)
+				msg := fmt.Sprintf("error: %s", err)
+				channel, _ := bot.UserChannelCreate("100443064228646912")
+				id, _ := bot.ChannelMessageSend(channel.ID, msg)
+				log.Println(id, msg)
 			}
 		}
 		return []reflect.Value{}
@@ -103,7 +100,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 
 		// try to open a repo for this beatmap
 		var repo *git.Repository
-		repoDir := path.Join(bot.config.Repos, strconv.Itoa(beatmapSet.UserID), strconv.Itoa(beatmapSet.ID))
+		repoDir := path.Join(bot.config.Repos, strconv.Itoa(beatmapSet.ID))
 		if _, err := os.Stat(repoDir); os.IsNotExist(err) {
 			os.MkdirAll(repoDir, 0777)
 		}
@@ -155,7 +152,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			},
 		)
 		if err != nil {
-			err = fmt.Errorf("couldn't create commit for %d: %w", beatmapSet.ID, err)
+			err = fmt.Errorf("couldn't create commit for %s: %w", beatmapSet.ID, err)
 			return
 		}
 
@@ -171,7 +168,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			err = fmt.Errorf("couldn't retrieve commit parent: %w", err)
 			return
 		} else {
-			patch, err = parent.Patch(commit)
+			patch, err = commit.Patch(parent)
 			if err != nil {
 				err = fmt.Errorf("couldn't retrieve patch: %w", err)
 				return
@@ -179,26 +176,16 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			foundPatch = true
 		}
 
-		fmt.Println("saving", beatmapSet)
-		bot.db.SaveBeatmap(db.Beatmapset{
-			ID:          beatmapSet.ID,
-			MapperID:    beatmapSet.UserID,
-			Artist:      beatmapSet.Artist,
-			Title:       beatmapSet.Title,
-			LastUpdated: eventTime,
-		})
-		bot.db.GetUser(beatmapSet.UserID)
-
 		embed := &discordgo.MessageEmbed{
-			URL:       fmt.Sprintf("%s/map/%d/%d/versions", bot.config.Web.ServedAt, beatmapSet.UserID, beatmapSet.ID),
+			URL:       fmt.Sprintf("https://osu.ppy.sh/s/%d", beatmapSet.ID),
 			Title:     fmt.Sprintf("Update: %s - %s", beatmapSet.Artist, beatmapSet.Title),
 			Timestamp: eventTime.Format(time.RFC3339),
 			Author: &discordgo.MessageEmbedAuthor{
-				URL:  "https://osu.ppy.sh/u/" + strconv.Itoa(beatmapSet.UserID),
+				URL:  "https://osu.ppy.sh/u/" + strconv.Itoa(beatmapSet.UserId),
 				Name: beatmapSet.Creator,
 				IconURL: fmt.Sprintf(
 					"https://a.ppy.sh/%d?%d.png",
-					beatmapSet.UserID,
+					beatmapSet.UserId,
 					time.Now().Unix(),
 				),
 			},
@@ -224,6 +211,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			_, err = bot.ChannelMessageSendEmbed(channelId, embed)
 			if err != nil {
 				err = fmt.Errorf("failed to send to %s: %w", channelId, err)
+				return
 			}
 		}
 	}
@@ -311,16 +299,12 @@ func (bot *Bot) newMessageHandler(s *discordgo.Session, m *discordgo.MessageCrea
 			return
 		}
 
+		// go func() {
+		// 	time.Sleep(refreshInterval)
+		// 	bot.requests <- mapperId
+		// }()
+
 		bot.ChannelMessageSend(m.ChannelID, fmt.Sprintf("subscribed to %+v", mapper))
-
-	case "list":
-		mappers := make([]string, 0)
-		bot.db.IterChannelTrackedMappers(m.ChannelID, func(mapper db.User) error {
-			mappers = append(mappers, mapper.Username)
-			return nil
-		})
-
-		bot.ChannelMessageSend(m.ChannelID, "tracking: "+strings.Join(mappers, ", "))
 	}
 
 	return

go.mod

@@ -4,18 +4,9 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
 	github.com/bwmarrin/discordgo v0.22.0
-	github.com/dustin/go-humanize v1.0.0
-	github.com/foolin/goview v0.3.0
-	github.com/gin-contrib/static v0.0.0-20200916080430-d45d9a37d28e
-	github.com/gin-gonic/contrib v0.0.0-20201005132743-ca038bbf2944
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-git/go-git/v5 v5.2.0
-	github.com/gorilla/sessions v1.2.1 // indirect
+	go.etcd.io/bbolt v1.3.5
-	github.com/kofalt/go-memoize v0.0.0-20200917044458-9b55a8d73e1c
-	github.com/pkg/errors v0.8.1
 	golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520
-	gorm.io/driver/sqlite v1.1.4
-	gorm.io/gorm v1.21.12
 )

go.sum

@@ -1,80 +1,41 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
-github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
-github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
-github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
-github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04=
-github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw=
 github.com/bwmarrin/discordgo v0.22.0 h1:uBxY1HmlVCsW1IuaPjpCGT6A2DBwRn0nvOguQIxDdFM=
 github.com/bwmarrin/discordgo v0.22.0/go.mod h1:c1WtWUGN6nREDmzIpyTp/iD3VYt4Fpx+bVyfBG7JE+M=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
-github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/foolin/goview v0.3.0 h1:q5wKwXKEFb20dMRfYd59uj5qGCo7q4L9eVHHUjmMWrg=
-github.com/foolin/goview v0.3.0/go.mod h1:OC1VHC4FfpWymhShj8L1Tc3qipFmrmm+luAEdTvkos4=
-github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
-github.com/gin-contrib/static v0.0.0-20200916080430-d45d9a37d28e h1:8bZpGwoPxkaivQPrAbWl+7zjjUcbFUnYp7yQcx2r2N0=
-github.com/gin-contrib/static v0.0.0-20200916080430-d45d9a37d28e/go.mod h1:VhW/Ch/3FhimwZb8Oj+qJmdMmoB8r7lmJ5auRjm50oQ=
-github.com/gin-gonic/contrib v0.0.0-20201005132743-ca038bbf2944 h1:CUXsTZuAAdpQinpKgInZqKTOfn/jkIA9DLnozeybVRQ=
-github.com/gin-gonic/contrib v0.0.0-20201005132743-ca038bbf2944/go.mod h1:iqneQ2Df3omzIVTkIfn7c1acsVnMGiSLn4XF5Blh3Yg=
-github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
-github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
-github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
 github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
+github.com/go-git/go-git v1.0.0 h1:YcN9iDGDoXuIw0vHls6rINwV416HYa0EB2X+RBsyYp4=
+github.com/go-git/go-git v3.2.0+incompatible h1:puJR2KQIDtZm7uQoF5h+BUFHiU2naU4tfwau90nRIoo=
 github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
 github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI=
 github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
-github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
-github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
-github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
-github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
-github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
-github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
-github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
@@ -82,121 +43,64 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
-github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
-github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
-github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
-github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
-github.com/kofalt/go-memoize v0.0.0-20200917044458-9b55a8d73e1c h1:iQm590VHHIeY1j9OFi2qBPgIeDqxz+PmHdDK/fUcN7s=
-github.com/kofalt/go-memoize v0.0.0-20200917044458-9b55a8d73e1c/go.mod h1:IvB2BCBCdgZFN9ZSgInoUlL1sAu0Xbvqfd7D+qqzTeo=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
-github.com/labstack/echo/v4 v4.1.6/go.mod h1:kU/7PwzgNxZH4das4XNsSpBSOD09XIF5YEPzjpkGnGE=
-github.com/labstack/gommon v0.2.9/go.mod h1:E8ZTmW9vw5az5/ZyHWCp0Lw4OH2ecsaBP1C/NKavGG4=
-github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
-github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
-github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
-github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
-github.com/smartystreets/gunit v1.4.2 h1:tyWYZffdPhQPfK5VsMQXfauwnJkqg7Tv5DLuQVYxq3Q=
-github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520 h1:Bx6FllMpG4NWDOfhMBz1VR2QYNp/SAOHPIAsaVmxfPo=
 golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190608022120-eacb66d2a7c3/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
-gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
-gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
-gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
-gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
-gorm.io/gorm v1.21.12 h1:3fQM0Eiz7jcJEhPggHEpoYnsGZqynMzverL77DV40RM=
-gorm.io/gorm v1.21.12/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=

main.go

@@ -17,8 +17,6 @@ import (
 
 var exit_chan = make(chan int)
 
-var GitCommit string
-
 func main() {
 	configPath := flag.String("config", "config.toml", "Path to the config file (defaults to config.toml)")
 	flag.Parse()
@@ -30,7 +28,7 @@ func main() {
 
 	api := osuapi.New(&config)
 
-	db, err := db.OpenDb(config, api)
+	db, err := db.OpenDb(config.DatabasePath, api)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -41,8 +39,8 @@ func main() {
 		log.Fatal(err)
 	}
 
-	go scrape.RunScraper(&config, bot, db, api)
+	go scrape.RunScraper(bot, db, api)
-	go web.RunWeb(&config, api, db, GitCommit)
+	go web.RunWeb(&config)
 
 	signal_chan := make(chan os.Signal, 1)
 	signal.Notify(signal_chan,

osuapi/beatmapsets.go

@@ -1,103 +0,0 @@
-package osuapi
-
-import (
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/url"
-	"os"
-
-	"github.com/pkg/errors"
-)
-
-func (api *Osuapi) SearchBeatmaps(rankStatus string) (beatmapSearch BeatmapSearch, err error) {
-	values := url.Values{}
-	values.Set("s", rankStatus)
-	query := values.Encode()
-	url := "/beatmapsets/search?" + query
-	err = api.Request("GET", url, &beatmapSearch)
-	if err != nil {
-		errors.Wrap(err, "failed to search beatmaps")
-		return
-	}
-
-	return
-}
-
-func (api *Osuapi) DownloadSingleBeatmap(beatmapId int, path string) (err error) {
-	url := fmt.Sprintf("https://osu.ppy.sh/osu/%d", beatmapId)
-	resp, err := api.httpClient.Get(url)
-	if err != nil {
-		err = errors.Wrap(err, "could not fetch beatmap")
-		return
-	}
-
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
-	if err != nil {
-		err = errors.Wrap(err, "could not open file for downloading beatmap")
-		return
-	}
-
-	_, err = io.Copy(file, resp.Body)
-	if err != nil {
-		return
-	}
-	return
-}
-
-func (api *Osuapi) GetBeatmapSet(beatmapSetId int) (beatmapSet Beatmapset, err error) {
-	url := fmt.Sprintf("/beatmapsets/%d", beatmapSetId)
-	err = api.Request("GET", url, &beatmapSet)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (api *Osuapi) BeatmapsetDownload(beatmapSetId int) (path string, err error) {
-	url := fmt.Sprintf("/beatmapsets/%d/download", beatmapSetId)
-	resp, err := api.Request0("GET", url)
-	if err != nil {
-		return
-	}
-
-	file, err := ioutil.TempFile(os.TempDir(), "beatmapsetDownload")
-	if err != nil {
-		return
-	}
-
-	_, err = io.Copy(file, resp.Body)
-	if err != nil {
-		return
-	}
-	file.Close()
-
-	path = file.Name()
-	return
-}
-
-type GetBeatmapsetEventsOptions struct {
-	User  string
-	Types []string
-}
-
-func (api *Osuapi) GetBeatmapsetEvents(opts *GetBeatmapsetEventsOptions) (events []BeatmapsetEvent, err error) {
-	values := url.Values{}
-	values.Set("user", opts.User)
-	query := values.Encode()
-	for _, t := range opts.Types {
-		query += "&types[]=" + t
-	}
-	url := "/beatmapsets/events?" + query
-	fmt.Println("URL IS", url)
-
-	var reply BeatmapsetEvents
-	err = api.Request("GET", url, &reply)
-	if err != nil {
-		return
-	}
-
-	events = reply.Events
-	return
-}

osuapi/models.go

@@ -14,7 +14,7 @@ type Beatmapset struct {
 	Title         string `json:"title"`
 	TitleUnicode  string `json:"title_unicode"`
 	Creator       string `json:"creator"`
-	UserID        int    `json:"user_id"`
+	UserId        int    `json:"user_id"`
 
 	Covers      BeatmapCovers `json:"covers"`
 	Beatmaps    []Beatmap     `json:"beatmaps,omitempty"`
@@ -63,14 +63,3 @@ type EventUser struct {
 type BeatmapSearch struct {
 	Beatmapsets []Beatmapset `json:"beatmapsets"`
 }
-
-type BeatmapsetEvents struct {
-	Events []BeatmapsetEvent `json:"events"`
-}
-
-type BeatmapsetEvent struct {
-	ID         int        `json:"id"`
-	Type       string     `json:"type"`
-	Beatmapset Beatmapset `json:"beatmapset"`
-	UserID     int        `json:"user_id"`
-}

osuapi/osuapi.go

@@ -4,14 +4,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
+	"net/url"
+	"os"
 	"strings"
-	"sync"
 	"time"
 
-	"github.com/pkg/errors"
 	"golang.org/x/sync/semaphore"
 
 	"subscribe-bot/config"
@@ -25,9 +26,6 @@ type Osuapi struct {
 	token      string
 	expires    time.Time
 	config     *config.Config
-
-	tokenLock       sync.RWMutex
-	isFetchingToken bool
 }
 
 func New(config *config.Config) *Osuapi {
@@ -37,39 +35,17 @@ func New(config *config.Config) *Osuapi {
 
 	// want to cap at around 1000 requests a minute, OSU cap is 1200
 	lock := semaphore.NewWeighted(1000)
-
+	return &Osuapi{client, lock, "", time.Now(), config}
-	return &Osuapi{
-		httpClient: client,
-		lock:       lock,
-		expires:    time.Now(),
-		config:     config,
-	}
 }
 
 func (api *Osuapi) Token() (token string, err error) {
-	return api.tokenFetch(false)
+	if time.Now().Before(api.expires) {
-}
-
-func (api *Osuapi) tokenFetch(force bool) (token string, err error) {
-	if !force {
-		if time.Now().Before(api.expires) {
-			token = api.token
-			return
-		}
-	}
-
-	if api.isFetchingToken {
-		api.tokenLock.RLock()
 		token = api.token
-		api.tokenLock.RUnlock()
 		return
 	}
 
-	api.tokenLock.Lock()
-	api.isFetchingToken = true
-
 	data := fmt.Sprintf(
-		"client_id=%s&client_secret=%s&grant_type=client_credentials&scope=public",
+		"client_id=%d&client_secret=%s&grant_type=client_credentials&scope=public",
 		api.config.Oauth.ClientId,
 		api.config.Oauth.ClientSecret,
 	)
@@ -80,29 +56,24 @@ func (api *Osuapi) tokenFetch(force bool) (token string, err error) {
 		strings.NewReader(data),
 	)
 	if err != nil {
-		err = errors.Wrap(err, "failed to make POST request")
 		return
 	}
 
 	var osuToken OsuToken
 	respBody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		err = errors.Wrap(err, "failed to read response body")
 		return
 	}
 
 	err = json.Unmarshal(respBody, &osuToken)
 	if err != nil {
-		err = errors.Wrap(err, "failed to unmarshal response body as json")
 		return
 	}
 
 	log.Println("got new access token", osuToken.AccessToken[:12]+"...")
 	api.token = osuToken.AccessToken
 	api.expires = time.Now().Add(time.Duration(osuToken.ExpiresIn) * time.Second)
-
 	token = api.token
-	api.tokenLock.Unlock()
 	return
 }
 
@@ -116,26 +87,20 @@ func (api *Osuapi) Request0(action string, url string) (resp *http.Response, err
 
 	token, err := api.Token()
 	if err != nil {
-		err = errors.Wrap(err, "failed to fetch token")
 		return
 	}
 
 	req.Header.Add("Authorization", "Bearer "+token)
+	if err != nil {
+		return
+	}
 
 	resp, err = api.httpClient.Do(req)
 	if err != nil {
-		err = errors.Wrap(err, "http client failed")
 		return
 	}
 
 	if resp.StatusCode != 200 {
-		if resp.StatusCode == 401 {
-			// force fetch token again
-			api.tokenFetch(true)
-			log.Println("re-fetched token cus expired")
-			return api.Request0(action, url)
-		}
-
 		var respBody []byte
 		respBody, err = ioutil.ReadAll(resp.Body)
 		if err != nil {
@@ -157,17 +122,103 @@ func (api *Osuapi) Request0(action string, url string) (resp *http.Response, err
 func (api *Osuapi) Request(action string, url string, result interface{}) (err error) {
 	resp, err := api.Request0(action, url)
 	if err != nil {
-		return errors.Wrap(err, "base request failed")
+		return
 	}
 
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return errors.Wrap(err, "failed to read http response body")
+		return
 	}
 
 	err = json.Unmarshal(data, result)
 	if err != nil {
-		return errors.Wrap(err, "failed to unmarshal http response as json")
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) DownloadSingleBeatmap(beatmapId int, path string) (err error) {
+	url := fmt.Sprintf("https://osu.ppy.sh/osu/%d", beatmapId)
+	resp, err := api.httpClient.Get(url)
+
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return
+	}
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (api *Osuapi) GetBeatmapSet(beatmapSetId int) (beatmapSet Beatmapset, err error) {
+	url := fmt.Sprintf("/beatmapsets/%d", beatmapSetId)
+	err = api.Request("GET", url, &beatmapSet)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) BeatmapsetDownload(beatmapSetId int) (path string, err error) {
+	url := fmt.Sprintf("/beatmapsets/%d/download", beatmapSetId)
+	resp, err := api.Request0("GET", url)
+	if err != nil {
+		return
+	}
+
+	file, err := ioutil.TempFile(os.TempDir(), "beatmapsetDownload")
+	if err != nil {
+		return
+	}
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return
+	}
+	file.Close()
+
+	path = file.Name()
+	return
+}
+
+func (api *Osuapi) GetUser(userId string) (user User, err error) {
+	url := fmt.Sprintf("/users/%s", userId)
+	err = api.Request("GET", url, &user)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) GetUserEvents(userId int, limit int, offset int) (events []Event, err error) {
+	url := fmt.Sprintf(
+		"/users/%d/recent_activity?limit=%d&offset=%d",
+		userId,
+		limit,
+		offset,
+	)
+	err = api.Request("GET", url, &events)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) SearchBeatmaps(rankStatus string) (beatmapSearch BeatmapSearch, err error) {
+	values := url.Values{}
+	values.Set("s", rankStatus)
+	query := values.Encode()
+	url := "/beatmapsets/search?" + query
+	err = api.Request("GET", url, &beatmapSearch)
+	if err != nil {
+		return
 	}
 
 	return

osuapi/users.go

@@ -1,28 +0,0 @@
-package osuapi
-
-import "fmt"
-
-func (api *Osuapi) GetUser(userId string) (user User, err error) {
-	url := fmt.Sprintf("/users/%s", userId)
-	err = api.Request("GET", url, &user)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (api *Osuapi) GetUserEvents(userId int, limit int, offset int) (events []Event, err error) {
-	url := fmt.Sprintf(
-		"/users/%d/recent_activity?limit=%d&offset=%d",
-		userId,
-		limit,
-		offset,
-	)
-	err = api.Request("GET", url, &events)
-	if err != nil {
-		return
-	}
-
-	return
-}

scrape/nominated.go

@@ -1,16 +0,0 @@
-package scrape
-
-import (
-	"subscribe-bot/osuapi"
-)
-
-func (s *Scraper) scrapeNominatedMaps() {
-	events, _ := s.api.GetBeatmapsetEvents(&osuapi.GetBeatmapsetEventsOptions{
-		Types: []string{"nominate", "qualify"},
-	})
-
-	for _, event := range events {
-		(func(_ osuapi.BeatmapsetEvent) {})(event)
-		// fmt.Println(event)
-	}
-}

scrape/pending.go

@@ -1,145 +0,0 @@
-package scrape
-
-import (
-	"fmt"
-	"log"
-	"subscribe-bot/db"
-	"subscribe-bot/osuapi"
-	"time"
-)
-
-func (s *Scraper) scrapePendingMaps() {
-	// build a list of currently tracked mappers
-	trackedMappers := make(map[int]int)
-	s.db.IterAllTrackedMappers(func(mapper db.User) error {
-		trackedMappers[mapper.ID] = 1
-		return nil
-	})
-
-	// TODO: is this sorted for sure??
-	pendingSets, err := s.api.SearchBeatmaps("pending")
-	if err != nil {
-		log.Println("error fetching pending sets", err)
-		s.bot.NotifyError("failed to fetch pending sets", err)
-		return
-	}
-
-	allNewMaps := make(map[int][]osuapi.Beatmapset, 0)
-	var newLastUpdateTime = time.Unix(0, 0)
-	for _, beatmapSet := range pendingSets.Beatmapsets {
-		updatedTime, err := time.Parse(time.RFC3339, beatmapSet.LastUpdated)
-		if err != nil {
-			log.Println("error parsing last updated time", updatedTime)
-		}
-
-		if updatedTime.After(newLastUpdateTime) {
-			// update lastUpdateTime to latest updated map
-			newLastUpdateTime = updatedTime
-		}
-
-		if !updatedTime.After(lastUpdateTime) {
-			break
-		}
-
-		mapperId := beatmapSet.UserID
-
-		// if _, ok := trackedMappers[mapperId]; ok {
-		if _, ok2 := allNewMaps[mapperId]; !ok2 {
-			allNewMaps[mapperId] = make([]osuapi.Beatmapset, 0)
-		}
-
-		allNewMaps[mapperId] = append(allNewMaps[mapperId], beatmapSet)
-		// }
-	}
-
-	if len(allNewMaps) > 0 {
-		for mapperId, newMaps := range allNewMaps {
-			channels := make([]string, 0)
-			s.db.IterTrackingChannels(mapperId, func(channel db.DiscordChannel) error {
-				channels = append(channels, channel.ID)
-				return nil
-			})
-
-			err := s.bot.NotifyNewBeatmap(channels, newMaps)
-			if err != nil {
-				log.Println("error notifying new maps:", err)
-			}
-		}
-	}
-
-	lastUpdateTime = newLastUpdateTime
-	log.Println("last updated time", lastUpdateTime)
-}
-
-func getNewMaps(db *db.Db, api *osuapi.Osuapi, userId int) (newMaps []osuapi.Event, err error) {
-	// see if there's a last event
-	hasLastEvent, lastEventId := db.MapperLastEvent(userId)
-	newMaps = make([]osuapi.Event, 0)
-	var (
-		events            []osuapi.Event
-		newLatestEvent    = 0
-		updateLatestEvent = false
-	)
-	if hasLastEvent {
-		offset := 0
-
-	loop:
-		for {
-			events, err = api.GetUserEvents(userId, 50, offset)
-			if err != nil {
-				err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err)
-				return
-			}
-			if len(events) == 0 {
-				break
-			}
-
-			for _, event := range events {
-				if event.ID == lastEventId {
-					break loop
-				}
-
-				if event.ID > newLatestEvent {
-					updateLatestEvent = true
-					newLatestEvent = event.ID
-				}
-
-				if event.Type == "beatmapsetUpload" ||
-					event.Type == "beatmapsetRevive" ||
-					event.Type == "beatmapsetUpdate" {
-					newMaps = append(newMaps, event)
-				}
-			}
-
-			offset += len(events)
-		}
-	} else {
-		log.Printf("no last event id found for %d\n", userId)
-		events, err = api.GetUserEvents(userId, 50, 0)
-		if err != nil {
-			return
-		}
-
-		for _, event := range events {
-			if event.ID > newLatestEvent {
-				updateLatestEvent = true
-				newLatestEvent = event.ID
-			}
-
-			if event.Type == "beatmapsetUpload" ||
-				event.Type == "beatmapsetRevive" ||
-				event.Type == "beatmapsetUpdate" {
-				newMaps = append(newMaps, event)
-			}
-		}
-	}
-
-	if updateLatestEvent {
-		err = db.UpdateMapperLastEvent(userId, newLatestEvent)
-		if err != nil {
-			return
-		}
-	}
-
-	return
-}

scrape/scrape.go

@@ -1,6 +1,8 @@
 package scrape
 
 import (
+	"fmt"
+	"log"
 	"time"
 
 	"subscribe-bot/config"
@@ -11,26 +13,151 @@ import (
 
 var (
 	refreshInterval = 30 * time.Second
-	lastUpdateTime  time.Time
 	Ticker          = time.NewTicker(refreshInterval)
 )
 
-type Scraper struct {
-	config *config.Config
-	bot    *discord.Bot
-	db     *db.Db
-	api    *osuapi.Osuapi
-}
-
 func RunScraper(config *config.Config, bot *discord.Bot, db *db.Db, api *osuapi.Osuapi) {
-	lastUpdateTime = time.Now()
+	lastUpdateTime := time.Now()
-
-	scraper := Scraper{config, bot, db, api}
-
 	go func() {
 		for ; true; <-Ticker.C {
-			scraper.scrapePendingMaps()
+			// build a list of currently tracked mappers
-			scraper.scrapeNominatedMaps()
+			trackedMappers := make(map[int]int)
+			db.IterTrackedMappers(func(userId int) error {
+				trackedMappers[userId] = 1
+				return nil
+			})
+
+			// TODO: is this sorted for sure??
+			pendingSets, err := api.SearchBeatmaps("pending")
+			if err != nil {
+				log.Println("error fetching pending sets", err)
+			}
+
+			allNewMaps := make(map[int][]osuapi.Beatmapset, 0)
+			var newLastUpdateTime = time.Unix(0, 0)
+			for _, beatmapSet := range pendingSets.Beatmapsets {
+				updatedTime, err := time.Parse(time.RFC3339, beatmapSet.LastUpdated)
+				if err != nil {
+					log.Println("error parsing last updated time", updatedTime)
+				}
+
+				if updatedTime.After(newLastUpdateTime) {
+					// update lastUpdateTime to latest updated map
+					newLastUpdateTime = updatedTime
+				}
+
+				if !updatedTime.After(lastUpdateTime) {
+					break
+				}
+
+				mapperId := beatmapSet.UserId
+				if _, ok := trackedMappers[mapperId]; ok {
+					if _, ok2 := allNewMaps[mapperId]; !ok2 {
+						allNewMaps[mapperId] = make([]osuapi.Beatmapset, 0)
+					}
+
+					allNewMaps[mapperId] = append(allNewMaps[mapperId], beatmapSet)
+				}
+			}
+
+			if len(allNewMaps) > 0 {
+				for mapperId, newMaps := range allNewMaps {
+					channels := make([]string, 0)
+					db.IterTrackingChannels(mapperId, func(channelId string) error {
+						channels = append(channels, channelId)
+						return nil
+					})
+
+					err := bot.NotifyNewBeatmap(channels, newMaps)
+					if err != nil {
+						log.Println("error notifying new maps:", err)
+					}
+				}
+			}
+
+			lastUpdateTime = newLastUpdateTime
+			// this rings the terminal bell when it's updated so i don't have to stare
+			// at a blank screen for 30 seconds waiting for the feed to update
+			if config.Debug {
+				fmt.Print("\a")
+			}
+			log.Println("last updated time", lastUpdateTime)
 		}
 	}()
 }
+
+func getNewMaps(db *db.Db, api *osuapi.Osuapi, userId int) (newMaps []osuapi.Event, err error) {
+	// see if there's a last event
+	hasLastEvent, lastEventId := db.MapperLastEvent(userId)
+	newMaps = make([]osuapi.Event, 0)
+	var (
+		events            []osuapi.Event
+		newLatestEvent    = 0
+		updateLatestEvent = false
+	)
+	if hasLastEvent {
+		offset := 0
+
+	loop:
+		for {
+			events, err = api.GetUserEvents(userId, 50, offset)
+			if err != nil {
+				err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err)
+				return
+			}
+			if len(events) == 0 {
+				break
+			}
+
+			for _, event := range events {
+				if event.ID == lastEventId {
+					break loop
+				}
+
+				if event.ID > newLatestEvent {
+					updateLatestEvent = true
+					newLatestEvent = event.ID
+				}
+
+				if event.Type == "beatmapsetUpload" ||
+					event.Type == "beatmapsetRevive" ||
+					event.Type == "beatmapsetUpdate" {
+					newMaps = append(newMaps, event)
+				}
+			}
+
+			offset += len(events)
+		}
+	} else {
+		log.Printf("no last event id found for %d\n", userId)
+		events, err = api.GetUserEvents(userId, 50, 0)
+		if err != nil {
+			return
+		}
+
+		for _, event := range events {
+			if event.ID > newLatestEvent {
+				updateLatestEvent = true
+				newLatestEvent = event.ID
+			}
+
+			if event.Type == "beatmapsetUpload" ||
+				event.Type == "beatmapsetRevive" ||
+				event.Type == "beatmapsetUpdate" {
+				newMaps = append(newMaps, event)
+			}
+		}
+	}
+
+	// TODO: debug
+	// updateLatestEvent = false
+
+	if updateLatestEvent {
+		err = db.UpdateMapperLatestEvent(userId, newLatestEvent)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

web/auth.go

@@ -1,81 +0,0 @@
-package web
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/gin-gonic/contrib/sessions"
-	"github.com/gin-gonic/gin"
-)
-
-func (web *Web) logout(c *gin.Context) {
-	session := sessions.Default(c)
-	session.Delete("access_token")
-	session.Save()
-
-	c.Redirect(http.StatusTemporaryRedirect, "/")
-}
-
-func (web *Web) login(c *gin.Context) {
-	url := url.URL{
-		Scheme: "https",
-		Host:   "osu.ppy.sh",
-		Path:   "/oauth/authorize",
-	}
-	q := url.Query()
-	q.Set("client_id", web.config.Oauth.ClientId)
-	q.Set("redirect_uri", web.config.Web.ServedAt+"/login/callback")
-	q.Set("response_type", "code")
-	q.Set("scope", "identify public")
-	q.Set("state", "urmom")
-	url.RawQuery = q.Encode()
-	fmt.Println("redirecting to", url.String())
-	c.Redirect(http.StatusTemporaryRedirect, url.String())
-}
-
-func (web *Web) loginCallback(c *gin.Context) {
-	receivedCode := c.Query("code")
-
-	bodyQuery := url.Values{}
-	bodyQuery.Set("client_id", web.config.Oauth.ClientId)
-	bodyQuery.Set("client_secret", web.config.Oauth.ClientSecret)
-	bodyQuery.Set("code", receivedCode)
-	bodyQuery.Set("grant_type", "authorization_code")
-	bodyQuery.Set("redirect_uri", web.config.Web.ServedAt+"/login/callback")
-	body := strings.NewReader(bodyQuery.Encode())
-	resp, _ := web.hc.Post("https://osu.ppy.sh/oauth/token", "application/x-www-form-urlencoded", body)
-	respBody, _ := ioutil.ReadAll(resp.Body)
-	type OsuToken struct {
-		TokenType    string `json:"token_type"`
-		ExpiresIn    int    `json:"expires_in"`
-		AccessToken  string `json:"access_token"`
-		RefreshToken string `json:"refresh_token"`
-	}
-	var token OsuToken
-	_ = json.Unmarshal(respBody, &token)
-	fmt.Println("TOKEN", token)
-
-	session := sessions.Default(c)
-	session.Set("access_token", token.AccessToken)
-	session.Save()
-
-	c.Redirect(http.StatusTemporaryRedirect, "/")
-}
-
-func isLoggedIn(c *gin.Context) bool {
-	session := sessions.Default(c)
-	var accessToken string
-	loggedIn := false
-	accessTokenI := session.Get("access_token")
-	if accessTokenI != nil {
-		accessToken = accessTokenI.(string)
-		if len(accessToken) > 0 {
-			loggedIn = true
-		}
-	}
-	return loggedIn
-}

web/mapper.go

@@ -1,38 +0,0 @@
-package web
-
-import (
-	"net/http"
-	"strconv"
-	"subscribe-bot/db"
-
-	"github.com/gin-gonic/gin"
-	"github.com/pkg/errors"
-)
-
-func (web *Web) mapperIndex(c *gin.Context) (err error) {
-	userId := c.Param("userId")
-	mapperId, err := strconv.Atoi(userId)
-	if err != nil {
-		err = errors.Wrap(err, "id is not an integer")
-		return
-	}
-
-	mapper, err := web.db.GetUser(mapperId)
-	if err != nil {
-		err = errors.Wrap(err, "failed to get user from mapper-index")
-		return
-	}
-
-	beatmapsets := make([]db.Beatmapset, 0)
-	web.db.IterTrackedBeatmapsetsBy(mapperId, func(beatmapset db.Beatmapset) error {
-		beatmapsets = append(beatmapsets, beatmapset)
-		return nil
-	})
-
-	web.render(c, http.StatusOK, "mapper-index.html", gin.H{
-		"Mapper":      mapper,
-		"Beatmapsets": beatmapsets,
-	})
-
-	return
-}

web/render.go

@@ -1,30 +0,0 @@
-package web
-
-import (
-	"log"
-	"net/http"
-
-	"github.com/gin-gonic/gin"
-)
-
-func (web *Web) errorWrap(fn func(*gin.Context) error) func(*gin.Context) {
-	return func(c *gin.Context) {
-		err := fn(c)
-		if err != nil {
-			c.String(http.StatusInternalServerError, "error")
-			log.Println("fatal error", err)
-		}
-	}
-}
-
-func (web *Web) render(c *gin.Context, code int, tmpl string, obj gin.H) {
-	base := gin.H{
-		"IsLoggedIn": isLoggedIn(c),
-	}
-
-	for key, val := range obj {
-		base[key] = val
-	}
-
-	c.HTML(code, tmpl, base)
-}

web/repo.go

@@ -1,117 +0,0 @@
-package web
-
-import (
-	"archive/zip"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"path"
-	"strconv"
-	"time"
-
-	"github.com/dustin/go-humanize"
-	"github.com/gin-gonic/gin"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/go-git/go-git/v5/plumbing/object"
-)
-
-func (web *Web) mapVersions(c *gin.Context) (err error) {
-	userId := c.Param("userId")
-	mapId := c.Param("mapId")
-
-	id, _ := strconv.Atoi(mapId)
-	bs, _ := web.api.GetBeatmapSet(id)
-
-	repoDir := path.Join(web.config.Repos, userId, mapId)
-	repo, _ := git.PlainOpen(repoDir)
-
-	type Revision struct {
-		Date      time.Time
-		HumanDate string
-		Summary   string
-		Hash      string
-		HasParent bool
-	}
-
-	versions := make([]Revision, 0)
-	logIter, _ := repo.Log(&git.LogOptions{})
-	for i := 0; i < 20; i++ {
-		commit, err := logIter.Next()
-		if err == io.EOF {
-			break
-		}
-
-		stats, _ := commit.Stats()
-		_, err = commit.Parent(0)
-		hasParent := !errors.Is(err, object.ErrParentNotFound)
-
-		versions = append(versions, Revision{
-			Date:      commit.Author.When,
-			HumanDate: humanize.Time(commit.Author.When),
-			Summary:   stats.String(),
-			Hash:      commit.Hash.String(),
-			HasParent: hasParent,
-		})
-	}
-
-	web.render(c, http.StatusOK, "map-version.html", gin.H{
-		"Beatmapset": bs,
-		"LoggedIn":   isLoggedIn(c),
-		"Versions":   versions,
-	})
-	return
-}
-
-func (web *Web) mapPatch(c *gin.Context) (err error) {
-	userId := c.Param("userId")
-	mapId := c.Param("mapId")
-	hash := c.Param("hash")
-
-	repoDir := path.Join(web.config.Repos, userId, mapId)
-	repo, _ := git.PlainOpen(repoDir)
-
-	hashObj := plumbing.NewHash(hash)
-	commit, _ := repo.CommitObject(hashObj)
-	parent, _ := commit.Parent(0)
-	patch, _ := parent.Patch(commit)
-
-	c.String(http.StatusOK, "text/plain", patch.String())
-	return
-}
-
-func (web *Web) mapZip(c *gin.Context) (err error) {
-	userId := c.Param("userId")
-	mapId := c.Param("mapId")
-	hash := c.Param("hash")
-
-	repoDir := path.Join(web.config.Repos, userId, mapId)
-	repo, _ := git.PlainOpen(repoDir)
-
-	hashObj := plumbing.NewHash(hash)
-	commit, _ := repo.CommitObject(hashObj)
-	tree, _ := commit.Tree()
-
-	files := tree.Files()
-
-	c.Writer.Header().Set("Content-type", "application/octet-stream")
-	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s-%s.zip", mapId, hash))
-	c.Stream(func(w io.Writer) bool {
-		ar := zip.NewWriter(w)
-		for {
-			file, err := files.Next()
-			if err == io.EOF {
-				break
-			}
-
-			reader, _ := file.Reader()
-			fdest, _ := ar.Create(file.Name)
-			io.Copy(fdest, reader)
-		}
-		ar.Close()
-
-		return false
-	})
-	return
-}

web/static/main.css

@@ -1,53 +0,0 @@
-:root {
-  --primary-color: #09c;
-}
-
-body, html {
-  font-family: Arial, Helvetica, sans-serif;
-}
-
-.title a {
-  color: black;
-  text-decoration: none;
-}
-
-.nav-bar {
-  background-color: var(--primary-color);
-  padding: 5px;
-}
-
-.nav-bar a {
-  color: white;
-  padding: 7px;
-  margin: 0px 3px;
-}
-
-a {
-  color: var(--primary-color);
-  text-decoration: none;
-}
-
-a:hover {
-  text-decoration: underline;
-}
-
-/* containers */
-
-.container {
-  width: 90%;
-  margin-left: auto;
-  margin-right: auto;
-}
-
-@media only screen and (min-width: 33.75em) {  /* 540px */
-  .container {
-    width: 80%;
-  }
-}
-
-@media only screen and (min-width: 60em) { /* 960px */
-  .container {
-    width: 75%;
-    max-width: 60rem;
-  }
-}

web/templates/index.html

@@ -1,35 +0,0 @@
-{{ define "content" }}
-
-<h3>Stats</h3>
-
-Currently tracking <b>{{ .TotalMaps }}</b> maps from <b>{{ .TotalUsers }}</b> users.
-
-<h3>Last 10 Updated Beatmaps</h3>
-
-<table class="table-auto">
-    <thead>
-        <th>Links</th>
-        <th>Title</th>
-        <th>Mapper</th>
-        <th>Updated</th>
-    </thead>
-    <tbody>
-    {{ range .Beatmapsets }}
-        <tr>
-            <td>
-                <a href="https://osu.ppy.sh/s/{{ .ID }}" target="_blank">[osu]</a>
-                <a href="/map/{{ .MapperID }}/{{ .ID }}/versions">[versions]</a>
-            </td>
-            <td>{{ .Artist }} - {{ .Title }}</td>
-            <td>
-                <a href="/map/{{ .MapperID }}">{{ .Mapper.Username }}</a>
-            </td>
-            <td>
-                {{ .LastUpdated | humanize }}
-            </td>
-        </tr>
-    {{ end }}
-    </tbody>
-</table>
-
-{{ end }}

web/templates/map-version.html

@@ -1,36 +0,0 @@
-{{ define "content" }}
-
-<h3>versions of {{ .Beatmapset.Artist }} - {{ .Beatmapset.Title }}</h3>
-
-<p>
-    <a href="https://osu.ppy.sh/s/{{ .Beatmapset.ID }}" target="_blank">Map link</a>
-    &middot;
-    mapped by <a href="https://osu.ppy.sh/u/{{ .Beatmapset.UserID }}" target="_blank">{{ .Beatmapset.Creator }}</a>
-</p>
-
-<small>up to the latest 20 revisions, pagination coming later</small>
-
-<table>
-    <thead>
-        <th>Date</th>
-        <th>Links</th>
-        <th>Summary</th>
-    </thead>
-
-    <tbody>
-    {{ range .Versions }}
-        <tr>
-            <td><span title="{{ .Date }}">{{ .HumanDate }}</span></td>
-            <td>
-                <a href="zip/{{ .Hash }}" target="_blank">[zip]</a>
-                {{ if .HasParent }}
-                    <a href="patch/{{ .Hash }}" target="_blank">[patch]</a>
-                {{ end }}
-            </td>
-            <td><pre>{{ .Summary }}</pre></td>
-        </tr>
-    {{ end }}
-    </tbody>
-</table>
-
-{{ end }}

web/templates/mapper-index.html

@@ -1,27 +0,0 @@
-{{ define "content" }}
-
-<h3>Tracked Maps by <a href="https://osu.ppy.sh/u/{{ .Mapper.ID }}" target="_blank">{{ .Mapper.Username }}</a></h3>
-
-<table class="table-auto">
-    <thead>
-        <th>Links</th>
-        <th>Title</th>
-        <th>Updated</th>
-    </thead>
-    <tbody>
-    {{ range .Beatmapsets }}
-        <tr>
-            <td>
-                <a href="https://osu.ppy.sh/s/{{ .ID }}" target="_blank">[osu]</a>
-                <a href="/map/{{ .MapperID }}/{{ .ID }}/versions">[versions]</a>
-            </td>
-            <td>{{ .Artist }} - {{ .Title }}</td>
-            <td>
-                {{ .LastUpdated | humanize }}
-            </td>
-        </tr>
-    {{ end }}
-    </tbody>
-</table>
-
-{{ end }}

web/templates/master.html

@@ -1,45 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-    <head>
-        <title>subscribe-bot</title>
-
-        <meta name="viewport" content="width=device-width, initial-scale=1">
-
-        <link rel="stylesheet" href="/static/main.css" />
-    </head>
-
-    <body>
-        <div class="container">
-            <h1 class="title">
-                <a href="/">subscribe-bot</a>
-            </h1>
-
-            <small>logging in with your osu account does nothing</small>
-
-            <div class="nav-bar">
-                <a href="/">home</a>
-
-                {{ if .LoggedIn }}
-                    <a href="/logout">logout</a>
-                {{ else }}
-                    <a href="/login">login</a>
-                {{ end }}
-            </div>
-
-            {{ template "content" . }}
-
-            <footer>
-                <hr />
-                <small>
-                    Maintained by <a href="https://osu.ppy.sh/u/2688103" target="_blank">IOException</a>
-                    &middot;
-                    <a href="https://git.mzhang.io/osu/subscribe-bot" target="_blank">Source code</a>
-                    &middot;
-                    <a href="https://discord.gg/eqjVG2H" target="_blank">Discord</a>
-                    &middot;
-                    subscribe-bot v{{ GitCommit }}
-                </small>
-            </footer>
-        </div>
-    </body>
-</html>

web/web.go

@@ -2,98 +2,24 @@ package web
 
 import (
 	"fmt"
-	"html/template"
-	"net/http"
-	"time"
 
-	"github.com/dustin/go-humanize"
-	"github.com/foolin/goview"
-	"github.com/foolin/goview/supports/ginview"
-	"github.com/gin-contrib/static"
-	"github.com/gin-gonic/contrib/sessions"
 	"github.com/gin-gonic/gin"
-	"github.com/kofalt/go-memoize"
 
 	"subscribe-bot/config"
-	"subscribe-bot/db"
-	"subscribe-bot/osuapi"
 )
 
-const (
+func RunWeb(config *config.Config) {
-	USER_KEY = "user"
+	r := gin.Default()
-)
+	if !config.Debug {
-
-var (
-	cache = memoize.NewMemoizer(90*time.Second, 10*time.Minute)
-)
-
-type Web struct {
-	config  *config.Config
-	api     *osuapi.Osuapi
-	hc      *http.Client
-	db      *db.Db
-	version string
-}
-
-func RunWeb(config *config.Config, api *osuapi.Osuapi, db *db.Db, version string) {
-	hc := &http.Client{
-		Timeout: 10 * time.Second,
-	}
-
-	web := Web{config, api, hc, db, version}
-	web.Run()
-}
-
-func (web *Web) Run() {
-	if !web.config.Debug {
 		gin.SetMode(gin.ReleaseMode)
 	}
 
-	r := gin.Default()
+	r.GET("/ping", func(c *gin.Context) {
-	r.Use(gin.Recovery())
+		c.JSON(200, gin.H{
-	r.Use(static.Serve("/static", static.LocalFile("web/static", false)))
+			"message": "pong",
-	r.Use(sessions.Sessions("mysession", sessions.NewCookieStore([]byte(web.config.Web.SessionSecret))))
-
-	r.HTMLRender = ginview.New(goview.Config{
-		Root:         "web/templates",
-		Master:       "master.html",
-		DisableCache: web.config.Debug,
-		Funcs: template.FuncMap{
-			"GitCommit": func() string {
-				return web.version
-			},
-			"humanize": humanize.Time,
-		},
-	})
-
-	r.GET("/logout", web.logout)
-	r.GET("/login", web.login)
-	r.GET("/login/callback", web.loginCallback)
-
-	r.GET("/map/:userId", web.errorWrap(web.mapperIndex))
-	r.GET("/map/:userId/:mapId/versions", web.errorWrap(web.mapVersions))
-	r.GET("/map/:userId/:mapId/patch/:hash", web.errorWrap(web.mapPatch))
-	r.GET("/map/:userId/:mapId/zip/:hash", web.errorWrap(web.mapZip))
-
-	r.GET("/", func(c *gin.Context) {
-		beatmapSets := web.listRepos()
-		stats := web.db.GetStats()
-		web.render(c, http.StatusOK, "index.html", gin.H{
-			"Beatmapsets": beatmapSets,
-			"TotalMaps":   stats.TotalMaps,
-			"TotalUsers":  stats.TotalUsers,
 		})
 	})
 
-	addr := fmt.Sprintf("%s:%d", web.config.Web.Host, web.config.Web.Port)
+	addr := fmt.Sprintf("%s:%d", config.Web.Host, config.Web.Port)
 	r.Run(addr)
 }
-
-func (web *Web) listRepos() []db.Beatmapset {
-	beatmapSets := make([]db.Beatmapset, 0)
-	web.db.IterTrackedBeatmapsets(10, func(beatmapset db.Beatmapset) error {
-		beatmapSets = append(beatmapSets, beatmapset)
-		return nil
-	})
-	return beatmapSets
-}