swap target=blank

This reverts commit f002e7c6fb.

.gitignore

@@ -1,4 +1,5 @@
 /subscribe-bot
 /test.db
+/test.db-sqlite
 /config.toml
 /repos

Makefile

@@ -0,0 +1,12 @@
+GIT_COMMIT = $(shell git rev-parse --short HEAD)
+SOURCES = $(shell find . -type f -and -name "*.go" -or -name "*.html")
+
+all: subscribe-bot
+
+subscribe-bot: $(SOURCES)
+	go build -o $@ -ldflags "-X main.GitCommit=$(GIT_COMMIT)"
+
+checkFmt:
+	[ -z "$$(git ls-files | grep '\.go$$' | xargs gofmt -l)" ] || (exit 1)
+
+.PHONY: all checkFmt

README.md

@@ -4,8 +4,9 @@ subscribe-bot
 Subscribes to OSU map updates and stores versions for later review.
 
 Please don't run a separate bot, the official one is `subscribe-bot#8789`. If
-you want to contribute or test the bot, then here are instructions on how to
+you want to contribute or test the bot, instructions on how to run it are below.
-run it:
+
+Join the [Discord][2]
 
 How to run
 ----------
@@ -22,9 +23,19 @@ How to run
 1. Run the executable, passing `-config {path}` in case you want to use a
    different config file than `config.toml`.
 
+Architecture
+------------
+
+There's several independent services running within:
+
+- Discord bot, which posts updates to the relevant channels in discord
+- Web server, which hosts an HTTP server allowing you to view changes
+- Scraper, which actually polls the OSU API for new updates
+
 License
 -------
 
 [GPL3][1]
 
 [1]: https://www.gnu.org/licenses/gpl-3.0.en.html
+[2]: https://discord.gg/eqjVG2H

config/config.go

@@ -19,13 +19,15 @@ type Config struct {
 }
 
 type OauthConfig struct {
-	ClientId     int    `toml:"client_id"`
+	ClientId     string `toml:"client_id"`
 	ClientSecret string `toml:"client_secret"`
 }
 
 type WebConfig struct {
-	Host string `toml:"host"`
+	Host          string `toml:"host"`
-	Port int    `toml:"port"`
+	Port          int    `toml:"port"`
+	ServedAt      string `toml:"served_at"`
+	SessionSecret string `toml:"session_secret"`
 }
 
 func ReadConfig(path string) (config Config, err error) {

db/beatmap.go

@@ -0,0 +1,17 @@
+package db
+
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
+
+type Beatmapset struct {
+	gorm.Model
+	ID          int `gorm:"primaryKey"`
+	Artist      string
+	Title       string
+	MapperID    int       `gorm:"mapper_id"`
+	Mapper      User      `gorm:"foreignKey:MapperID"`
+	LastUpdated time.Time `gorm:"last_updated"`
+}

db/channel.go

@@ -0,0 +1,9 @@
+package db
+
+import "gorm.io/gorm"
+
+type DiscordChannel struct {
+	gorm.Model
+	ID             string `gorm:"primaryKey"`
+	TrackedMappers []User `gorm:"many2many:tracked_mappers"`
+}

db/config.go

@@ -0,0 +1,9 @@
+package db
+
+import "gorm.io/gorm"
+
+type Config struct {
+	gorm.Model
+	Key   string `gorm:"key;primaryKey"`
+	Value string `gorm:"value"`
+}

db/db.go

@@ -8,235 +8,175 @@ package db
 import (
 	"strconv"
 
-	bolt "go.etcd.io/bbolt"
+	"github.com/pkg/errors"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"gorm.io/gorm/logger"
 
+	"subscribe-bot/config"
 	"subscribe-bot/osuapi"
 )
 
 var (
 	LATEST_EVENT = []byte("latestEvent")
+	MAPPERS      = []byte("mapper")
+	CHANNELS     = []byte("channels")
 )
 
 type Db struct {
-	*bolt.DB
+	gorm *gorm.DB
-	api *osuapi.Osuapi
+	api  *osuapi.Osuapi
 }
 
-func OpenDb(path string, api *osuapi.Osuapi) (db *Db, err error) {
+func OpenDb(cfg config.Config, api *osuapi.Osuapi) (db *Db, err error) {
-	inner, err := bolt.Open(path, 0666, nil)
+	gormConfig := &gorm.Config{}
-	db = &Db{inner, api}
+	if cfg.Debug {
+		gormConfig.Logger = logger.Default.LogMode(logger.Info)
+	}
+
+	gorm, err := gorm.Open(sqlite.Open(cfg.DatabasePath), gormConfig)
+	if err != nil {
+		panic("failed to connect database")
+	}
+
+	// auto-migrate
+	gorm.AutoMigrate(&Config{})
+	gorm.AutoMigrate(&User{})
+	gorm.AutoMigrate(&Beatmapset{})
+	gorm.AutoMigrate(&DiscordChannel{})
+
+	db = &Db{gorm, api}
+	return
+}
+
+func (db *Db) SaveBeatmap(beatmap Beatmapset) {
+	db.gorm.Clauses(clause.OnConflict{UpdateAll: true}).Create(&beatmap)
+	return
+}
+
+func (db *Db) IterTrackedBeatmapsets(limit int, fn func(beatmapset Beatmapset) error) (err error) {
+	var beatmapsets []Beatmapset
+	db.gorm.Preload(clause.Associations).Limit(limit).Order("last_updated desc").Find(&beatmapsets)
+	for _, beatmapset := range beatmapsets {
+		fn(beatmapset)
+	}
+	return
+}
+
+func (db *Db) IterTrackedBeatmapsetsBy(mapperId int, fn func(beatmapset Beatmapset) error) (err error) {
+	var beatmapsets []Beatmapset
+	db.gorm.Where("mapper_id = ?", mapperId).Order("last_updated desc").Find(&beatmapsets)
+	for _, beatmapset := range beatmapsets {
+		fn(beatmapset)
+	}
 	return
 }
 
 // Loop over channels that are tracking this specific mapper
-func (db *Db) IterTrackingChannels(mapperId int, fn func(channelId string) error) (err error) {
+func (db *Db) IterTrackingChannels(mapperId int, fn func(channel DiscordChannel) error) (err error) {
-	err = db.DB.View(func(tx *bolt.Tx) error {
+	var channels []DiscordChannel
-		mapper := getMapper(tx, mapperId)
+	db.gorm.Model(&User{ID: mapperId}).Association("TrackingChannels").Find(&channels)
-		if mapper == nil {
+	for _, channel := range channels {
-			return nil
+		fn(channel)
-		}
+	}
-
-		trackers := mapper.Bucket([]byte("trackers"))
-		if trackers == nil {
-			return nil
-		}
-
-		c := trackers.Cursor()
-		for k, _ := c.First(); k != nil; k, _ = c.Next() {
-			channelId := string(k)
-			err := fn(channelId)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
-	})
 
 	return
 }
 
-// Loop over tracked mappers
+// Loop over tracked mappers for this channel
-func (db *Db) IterTrackedMappers(fn func(userId int) error) (err error) {
+func (db *Db) IterChannelTrackedMappers(channelId string, fn func(user User) error) (err error) {
-	err = db.DB.View(func(tx *bolt.Tx) error {
+	var mappers []User
-		mappers := tx.Bucket([]byte("mapper"))
+	db.gorm.Model(&DiscordChannel{ID: channelId}).Association("TrackedMappers").Find(&mappers)
-		if mappers == nil {
+	for _, mapper := range mappers {
-			return nil
+		fn(mapper)
-		}
+	}
-
-		c := mappers.Cursor()
-		for k, _ := c.First(); k != nil; k, _ = c.Next() {
-			userId, err := strconv.Atoi(string(k))
-			if err != nil {
-				return err
-			}
-
-			err = fn(userId)
-			if err != nil {
-				return err
-			}
-		}
-
-		return nil
-	})
 
 	return
 }
 
-// Get a list of channels that are tracking this mapper
+// Loop over all tracked mappers
-func (db *Db) GetMapperTrackers(userId int) (trackersList []string) {
+func (db *Db) IterAllTrackedMappers(fn func(user User) error) (err error) {
-	trackersList = make([]string, 0)
+	var mappers []User
-	db.DB.View(func(tx *bolt.Tx) error {
+	db.gorm.Find(&mappers)
-		mapper, err := getMapperMut(tx, userId)
+	for _, mapper := range mappers {
-		if err != nil {
+		fn(mapper)
-			return err
+	}
-		}
 
-		trackers := mapper.Bucket([]byte("trackers"))
-		if trackers == nil {
-			return nil
-		}
-
-		c := trackers.Cursor()
-		for k, _ := c.First(); k != nil; k, _ = c.Next() {
-			channelId := string(k)
-			trackersList = append(trackersList, channelId)
-		}
-
-		return nil
-	})
 	return
 }
 
-// Update the latest event of a mapper to the given one
+func (db *Db) UpdateMapperLastEvent(userId int, eventId int) (err error) {
-func (db *Db) UpdateMapperLatestEvent(userId int, eventId int) (err error) {
+	if eventId == -1 {
-	err = db.DB.Update(func(tx *bolt.Tx) error {
+		var events []osuapi.Event
-		mapper, err := getMapperMut(tx, userId)
+		events, err = db.api.GetUserEvents(userId, 1, 0)
 		if err != nil {
-			return err
+			err = errors.Wrap(err, "couldn't get user events from API")
+			return
 		}
 
-		err = mapper.Put(LATEST_EVENT, []byte(strconv.Itoa(eventId)))
+		if len(events) > 0 {
-		if err != nil {
+			eventId = events[0].ID
-			return err
 		}
+	}
 
-		return nil
+	db.gorm.Model(&User{}).Where("id = ?", userId).Update("latest_event_id", eventId)
-	})
+	return nil
-	return
 }
 
 // Get the latest event ID of this mapper, if they have one
 func (db *Db) MapperLastEvent(userId int) (has bool, id int) {
-	has = false
+	var user User
-	id = -1
+	db.gorm.Select("latest_event_id").First(&user)
-	db.DB.View(func(tx *bolt.Tx) error {
+	return true, user.LatestEventID
-		mapper := getMapper(tx, userId)
-		if mapper == nil {
-			return nil
-		}
-
-		lastEventId := mapper.Get(LATEST_EVENT)
-		if lastEventId == nil {
-			return nil
-		}
-
-		var err error
-		id, err = strconv.Atoi(string(lastEventId))
-		if err != nil {
-			return nil
-		}
-
-		has = true
-		return nil
-	})
-
-	return
 }
 
 // Start tracking a new mapper (if they're not already tracked)
 func (db *Db) ChannelTrackMapper(channelId string, mapperId int, priority int) (err error) {
-	events, err := db.api.GetUserEvents(mapperId, 1, 0)
+	err = db.gorm.Model(&DiscordChannel{ID: channelId}).Association("TrackedMappers").Append(db.GetUser(mapperId))
 	if err != nil {
+		err = errors.Wrap(err, "could not add tracking for channel "+channelId)
 		return
 	}
 
-	err = db.Batch(func(tx *bolt.Tx) error {
+	err = db.UpdateMapperLastEvent(mapperId, -1)
-		{
+	if err != nil {
-			mapper, err := getMapperMut(tx, mapperId)
+		err = errors.Wrap(err, "could not update mapper latest event")
-			if err != nil {
+		return
-				return err
+	}
-			}
 
-			if len(events) > 0 {
+	return nil
-				latestEventId := strconv.Itoa(events[0].ID)
-				mapper.Put(LATEST_EVENT, []byte(latestEventId))
-			}
-
-			trackers, err := mapper.CreateBucketIfNotExists([]byte("trackers"))
-			if err != nil {
-				return err
-			}
-
-			err = trackers.Put([]byte(channelId), []byte(strconv.Itoa(priority)))
-			if err != nil {
-				return err
-			}
-		}
-		{
-			channels, err := tx.CreateBucketIfNotExists([]byte("channels"))
-			if err != nil {
-				return err
-			}
-
-			channel, err := channels.CreateBucketIfNotExists([]byte(channelId))
-			if err != nil {
-				return err
-			}
-
-			tracks, err := channel.CreateBucketIfNotExists([]byte("tracks"))
-			if err != nil {
-				return err
-			}
-
-			err = tracks.Put([]byte(strconv.Itoa(mapperId)), []byte(strconv.Itoa(priority)))
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	})
-	return
 }
 
 func (db *Db) Close() {
-	db.DB.Close()
 }
 
-func getMapper(tx *bolt.Tx, userId int) (mapper *bolt.Bucket) {
+func (db *Db) GetUser(userId int) (user *User, err error) {
-	mappers := tx.Bucket([]byte("mapper"))
+	// TODO: cache user info for some time?
-	if mappers == nil {
+
-		return nil
+	apiUser, err := db.api.GetUser(strconv.Itoa(userId))
+	if err != nil {
+		err = errors.Wrap(err, "could not retrieve user from the API")
+		return
 	}
 
-	mapper = mappers.Bucket([]byte(strconv.Itoa(userId)))
+	user = &User{
-	if mapper == nil {
+		ID:       userId,
-		return nil
+		Username: apiUser.Username,
+		Country:  apiUser.CountryCode,
 	}
+	db.gorm.Clauses(clause.OnConflict{UpdateAll: true}).Create(user)
 
 	return
 }
 
-func getMapperMut(tx *bolt.Tx, userId int) (mapper *bolt.Bucket, err error) {
+type Stats struct {
-	mappers, err := tx.CreateBucketIfNotExists([]byte("mapper"))
+	TotalMaps  int64
-	if err != nil {
+	TotalUsers int64
-		return
+}
-	}
-
-	mapper, err = mappers.CreateBucketIfNotExists([]byte(strconv.Itoa(userId)))
-	if err != nil {
-		return
-	}
 
+func (db *Db) GetStats() (stats Stats) {
+	db.gorm.Model(&Beatmapset{}).Count(&stats.TotalMaps)
+	db.gorm.Model(&Beatmapset{}).Group("mapper_id").Count(&stats.TotalUsers)
 	return
 }

db/user.go

@@ -0,0 +1,13 @@
+package db
+
+import "gorm.io/gorm"
+
+type User struct {
+	gorm.Model
+	ID                 int `gorm:"primaryKey"`
+	Username           string
+	Country            string
+	LatestEventID      int              `gorm:"latest_event_id"`
+	TrackingChannels   []DiscordChannel `gorm:"many2many:tracked_mappers"`
+	TrackedBeatmapsets []Beatmapset     `gorm:"foreignKey:MapperID"`
+}

discord/bot.go

@@ -53,6 +53,12 @@ func NewBot(config *config.Config, db *db.Db, api *osuapi.Osuapi) (bot *Bot, err
 	return
 }
 
+func (bot *Bot) NotifyError(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	channel, _ := bot.UserChannelCreate("100443064228646912")
+	bot.ChannelMessageSend(channel.ID, msg)
+}
+
 func (bot *Bot) errWrap(fn interface{}) interface{} {
 	val := reflect.ValueOf(fn)
 	origType := reflect.TypeOf(fn)
@@ -66,10 +72,7 @@ func (bot *Bot) errWrap(fn interface{}) interface{} {
 		if len(res) > 0 && !res[0].IsNil() {
 			err := res[0].Interface().(error)
 			if err != nil {
-				msg := fmt.Sprintf("error: %s", err)
+				bot.NotifyError("error: %s", err)
-				channel, _ := bot.UserChannelCreate("100443064228646912")
-				id, _ := bot.ChannelMessageSend(channel.ID, msg)
-				log.Println(id, msg)
 			}
 		}
 		return []reflect.Value{}
@@ -100,7 +103,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 
 		// try to open a repo for this beatmap
 		var repo *git.Repository
-		repoDir := path.Join(bot.config.Repos, strconv.Itoa(beatmapSet.ID))
+		repoDir := path.Join(bot.config.Repos, strconv.Itoa(beatmapSet.UserID), strconv.Itoa(beatmapSet.ID))
 		if _, err := os.Stat(repoDir); os.IsNotExist(err) {
 			os.MkdirAll(repoDir, 0777)
 		}
@@ -152,7 +155,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			},
 		)
 		if err != nil {
-			err = fmt.Errorf("couldn't create commit for %s: %w", beatmapSet.ID, err)
+			err = fmt.Errorf("couldn't create commit for %d: %w", beatmapSet.ID, err)
 			return
 		}
 
@@ -168,7 +171,7 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			err = fmt.Errorf("couldn't retrieve commit parent: %w", err)
 			return
 		} else {
-			patch, err = commit.Patch(parent)
+			patch, err = parent.Patch(commit)
 			if err != nil {
 				err = fmt.Errorf("couldn't retrieve patch: %w", err)
 				return
@@ -176,16 +179,26 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			foundPatch = true
 		}
 
+		fmt.Println("saving", beatmapSet)
+		bot.db.SaveBeatmap(db.Beatmapset{
+			ID:          beatmapSet.ID,
+			MapperID:    beatmapSet.UserID,
+			Artist:      beatmapSet.Artist,
+			Title:       beatmapSet.Title,
+			LastUpdated: eventTime,
+		})
+		bot.db.GetUser(beatmapSet.UserID)
+
 		embed := &discordgo.MessageEmbed{
-			URL:       fmt.Sprintf("https://osu.ppy.sh/s/%d", beatmapSet.ID),
+			URL:       fmt.Sprintf("%s/map/%d/%d/versions", bot.config.Web.ServedAt, beatmapSet.UserID, beatmapSet.ID),
 			Title:     fmt.Sprintf("Update: %s - %s", beatmapSet.Artist, beatmapSet.Title),
 			Timestamp: eventTime.Format(time.RFC3339),
 			Author: &discordgo.MessageEmbedAuthor{
-				URL:  "https://osu.ppy.sh/u/" + strconv.Itoa(beatmapSet.UserId),
+				URL:  "https://osu.ppy.sh/u/" + strconv.Itoa(beatmapSet.UserID),
 				Name: beatmapSet.Creator,
 				IconURL: fmt.Sprintf(
 					"https://a.ppy.sh/%d?%d.png",
-					beatmapSet.UserId,
+					beatmapSet.UserID,
 					time.Now().Unix(),
 				),
 			},
@@ -211,7 +224,6 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			_, err = bot.ChannelMessageSendEmbed(channelId, embed)
 			if err != nil {
 				err = fmt.Errorf("failed to send to %s: %w", channelId, err)
-				return
 			}
 		}
 	}
@@ -299,12 +311,16 @@ func (bot *Bot) newMessageHandler(s *discordgo.Session, m *discordgo.MessageCrea
 			return
 		}
 
-		// go func() {
-		// 	time.Sleep(refreshInterval)
-		// 	bot.requests <- mapperId
-		// }()
-
 		bot.ChannelMessageSend(m.ChannelID, fmt.Sprintf("subscribed to %+v", mapper))
+
+	case "list":
+		mappers := make([]string, 0)
+		bot.db.IterChannelTrackedMappers(m.ChannelID, func(mapper db.User) error {
+			mappers = append(mappers, mapper.Username)
+			return nil
+		})
+
+		bot.ChannelMessageSend(m.ChannelID, "tracking: "+strings.Join(mappers, ", "))
 	}
 
 	return

go.mod

@@ -4,9 +4,18 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
+	github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff // indirect
 	github.com/bwmarrin/discordgo v0.22.0
+	github.com/dustin/go-humanize v1.0.0
+	github.com/foolin/goview v0.3.0
+	github.com/gin-contrib/static v0.0.0-20200916080430-d45d9a37d28e
+	github.com/gin-gonic/contrib v0.0.0-20201005132743-ca038bbf2944
 	github.com/gin-gonic/gin v1.6.3
 	github.com/go-git/go-git/v5 v5.2.0
-	go.etcd.io/bbolt v1.3.5
+	github.com/gorilla/sessions v1.2.1 // indirect
+	github.com/kofalt/go-memoize v0.0.0-20200917044458-9b55a8d73e1c
+	github.com/pkg/errors v0.8.1
 	golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520
+	gorm.io/driver/sqlite v1.1.4
+	gorm.io/gorm v1.21.12
 )

go.sum

@@ -1,41 +1,80 @@
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
+github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
+github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
 github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
+github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff h1:RmdPFa+slIr4SCBg4st/l/vZWVe9QJKMXGO60Bxbe04=
+github.com/boj/redistore v0.0.0-20180917114910-cd5dcc76aeff/go.mod h1:+RTT1BOk5P97fT2CiHkbFQwkK3mjsFAP6zCYV2aXtjw=
 github.com/bwmarrin/discordgo v0.22.0 h1:uBxY1HmlVCsW1IuaPjpCGT6A2DBwRn0nvOguQIxDdFM=
 github.com/bwmarrin/discordgo v0.22.0/go.mod h1:c1WtWUGN6nREDmzIpyTp/iD3VYt4Fpx+bVyfBG7JE+M=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/foolin/goview v0.3.0 h1:q5wKwXKEFb20dMRfYd59uj5qGCo7q4L9eVHHUjmMWrg=
+github.com/foolin/goview v0.3.0/go.mod h1:OC1VHC4FfpWymhShj8L1Tc3qipFmrmm+luAEdTvkos4=
+github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s=
 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
+github.com/gin-contrib/static v0.0.0-20200916080430-d45d9a37d28e h1:8bZpGwoPxkaivQPrAbWl+7zjjUcbFUnYp7yQcx2r2N0=
+github.com/gin-contrib/static v0.0.0-20200916080430-d45d9a37d28e/go.mod h1:VhW/Ch/3FhimwZb8Oj+qJmdMmoB8r7lmJ5auRjm50oQ=
+github.com/gin-gonic/contrib v0.0.0-20201005132743-ca038bbf2944 h1:CUXsTZuAAdpQinpKgInZqKTOfn/jkIA9DLnozeybVRQ=
+github.com/gin-gonic/contrib v0.0.0-20201005132743-ca038bbf2944/go.mod h1:iqneQ2Df3omzIVTkIfn7c1acsVnMGiSLn4XF5Blh3Yg=
+github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM=
+github.com/gin-gonic/gin v1.5.0/go.mod h1:Nd6IXA8m5kNZdNEHMBd93KT+mdY3+bewLgRvmCsR2Do=
 github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14=
 github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M=
+github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
 github.com/go-git/gcfg v1.5.0 h1:Q5ViNfGF8zFgyJWPqYwA7qGFoMTEiBmdlkcfRmpIMa4=
 github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E=
 github.com/go-git/go-billy/v5 v5.0.0 h1:7NQHvd9FVid8VL4qVUMm8XifBK+2xCoZ2lSk0agRrHM=
 github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0=
-github.com/go-git/go-git v1.0.0 h1:YcN9iDGDoXuIw0vHls6rINwV416HYa0EB2X+RBsyYp4=
+github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12 h1:PbKy9zOy4aAKrJ5pibIRpVO2BXnK1Tlcg+caKI7Ox5M=
-github.com/go-git/go-git v3.2.0+incompatible h1:puJR2KQIDtZm7uQoF5h+BUFHiU2naU4tfwau90nRIoo=
 github.com/go-git/go-git-fixtures/v4 v4.0.2-0.20200613231340-f56387b50c12/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw=
 github.com/go-git/go-git/v5 v5.2.0 h1:YPBLG/3UK1we1ohRkncLjaXWLW+HKp5QNM/jTli2JgI=
 github.com/go-git/go-git/v5 v5.2.0/go.mod h1:kh02eMX+wdqqxgNMEyq8YgwlIOsDOa9homkUq1PoTMs=
+github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.12.1/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0=
+github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8=
+github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
+github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ=
+github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/gorilla/sessions v1.1.1/go.mod h1:8KCfur6+4Mqcc6S0FEfKuN15Vl5MgXW92AE8ovaJD0w=
+github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI=
+github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
 github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
 github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=
@@ -43,64 +82,121 @@ github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
+github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/jinzhu/now v1.1.2 h1:eVKgfIdy9b6zbWBMgFpfDPoAMifwSZagU9HmEU6zgiI=
+github.com/jinzhu/now v1.1.2/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd h1:Coekwdh0v2wtGp9Gmz1Ze3eVRAWJMLokvN3QjdzCHLY=
 github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kofalt/go-memoize v0.0.0-20200917044458-9b55a8d73e1c h1:iQm590VHHIeY1j9OFi2qBPgIeDqxz+PmHdDK/fUcN7s=
+github.com/kofalt/go-memoize v0.0.0-20200917044458-9b55a8d73e1c/go.mod h1:IvB2BCBCdgZFN9ZSgInoUlL1sAu0Xbvqfd7D+qqzTeo=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/labstack/echo v3.3.10+incompatible/go.mod h1:0INS7j/VjnFxD4E2wkz67b8cVwCLbBmJyDaka6Cmk1s=
+github.com/labstack/echo/v4 v4.1.6/go.mod h1:kU/7PwzgNxZH4das4XNsSpBSOD09XIF5YEPzjpkGnGE=
+github.com/labstack/gommon v0.2.9/go.mod h1:E8ZTmW9vw5az5/ZyHWCp0Lw4OH2ecsaBP1C/NKavGG4=
+github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.14.5 h1:1IdxlwTNazvbKJQSxoJ5/9ECbEeaTTyeU7sEAZ5KKTQ=
+github.com/mattn/go-sqlite3 v1.14.5/go.mod h1:WVKg1VTActs4Qso6iwGbiFih2UIHo0ENGwNd0Lj+XmI=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
+github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
+github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
+github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
+github.com/smartystreets/gunit v1.4.2 h1:tyWYZffdPhQPfK5VsMQXfauwnJkqg7Tv5DLuQVYxq3Q=
+github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
-go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
-go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 h1:xMPOj6Pz6UipU1wXLkrtqpHbR0AVFnyPEQq/wRWz9lM=
 golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190607181551-461777fb6f67/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520 h1:Bx6FllMpG4NWDOfhMBz1VR2QYNp/SAOHPIAsaVmxfPo=
 golang.org/x/sync v0.0.0-20201008141435-b3e1573b7520/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190609082536-301114b31cce/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 h1:uYVVQ9WP/Ds2ROhcaGPeIdVq0RIXVLwsHlnvJ+cT1So=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190608022120-eacb66d2a7c3/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE=
+gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
+gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM=
+gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw=
+gorm.io/gorm v1.20.7/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw=
+gorm.io/gorm v1.21.12 h1:3fQM0Eiz7jcJEhPggHEpoYnsGZqynMzverL77DV40RM=
+gorm.io/gorm v1.21.12/go.mod h1:F+OptMscr0P2F2qU97WT1WimdH9GaQPoDW7AYd5i2Y0=

main.go

@@ -17,6 +17,8 @@ import (
 
 var exit_chan = make(chan int)
 
+var GitCommit string
+
 func main() {
 	configPath := flag.String("config", "config.toml", "Path to the config file (defaults to config.toml)")
 	flag.Parse()
@@ -28,7 +30,7 @@ func main() {
 
 	api := osuapi.New(&config)
 
-	db, err := db.OpenDb(config.DatabasePath, api)
+	db, err := db.OpenDb(config, api)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -39,8 +41,8 @@ func main() {
 		log.Fatal(err)
 	}
 
-	go scrape.RunScraper(bot, db, api)
+	go scrape.RunScraper(&config, bot, db, api)
-	go web.RunWeb(&config)
+	go web.RunWeb(&config, api, db, GitCommit)
 
 	signal_chan := make(chan os.Signal, 1)
 	signal.Notify(signal_chan,

osuapi/beatmapsets.go

@@ -0,0 +1,103 @@
+package osuapi
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/url"
+	"os"
+
+	"github.com/pkg/errors"
+)
+
+func (api *Osuapi) SearchBeatmaps(rankStatus string) (beatmapSearch BeatmapSearch, err error) {
+	values := url.Values{}
+	values.Set("s", rankStatus)
+	query := values.Encode()
+	url := "/beatmapsets/search?" + query
+	err = api.Request("GET", url, &beatmapSearch)
+	if err != nil {
+		errors.Wrap(err, "failed to search beatmaps")
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) DownloadSingleBeatmap(beatmapId int, path string) (err error) {
+	url := fmt.Sprintf("https://osu.ppy.sh/osu/%d", beatmapId)
+	resp, err := api.httpClient.Get(url)
+	if err != nil {
+		err = errors.Wrap(err, "could not fetch beatmap")
+		return
+	}
+
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		err = errors.Wrap(err, "could not open file for downloading beatmap")
+		return
+	}
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return
+	}
+	return
+}
+
+func (api *Osuapi) GetBeatmapSet(beatmapSetId int) (beatmapSet Beatmapset, err error) {
+	url := fmt.Sprintf("/beatmapsets/%d", beatmapSetId)
+	err = api.Request("GET", url, &beatmapSet)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) BeatmapsetDownload(beatmapSetId int) (path string, err error) {
+	url := fmt.Sprintf("/beatmapsets/%d/download", beatmapSetId)
+	resp, err := api.Request0("GET", url)
+	if err != nil {
+		return
+	}
+
+	file, err := ioutil.TempFile(os.TempDir(), "beatmapsetDownload")
+	if err != nil {
+		return
+	}
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		return
+	}
+	file.Close()
+
+	path = file.Name()
+	return
+}
+
+type GetBeatmapsetEventsOptions struct {
+	User  string
+	Types []string
+}
+
+func (api *Osuapi) GetBeatmapsetEvents(opts *GetBeatmapsetEventsOptions) (events []BeatmapsetEvent, err error) {
+	values := url.Values{}
+	values.Set("user", opts.User)
+	query := values.Encode()
+	for _, t := range opts.Types {
+		query += "&types[]=" + t
+	}
+	url := "/beatmapsets/events?" + query
+	fmt.Println("URL IS", url)
+
+	var reply BeatmapsetEvents
+	err = api.Request("GET", url, &reply)
+	if err != nil {
+		return
+	}
+
+	events = reply.Events
+	return
+}

osuapi/models.go

@@ -14,7 +14,7 @@ type Beatmapset struct {
 	Title         string `json:"title"`
 	TitleUnicode  string `json:"title_unicode"`
 	Creator       string `json:"creator"`
-	UserId        int    `json:"user_id"`
+	UserID        int    `json:"user_id"`
 
 	Covers      BeatmapCovers `json:"covers"`
 	Beatmaps    []Beatmap     `json:"beatmaps,omitempty"`
@@ -63,3 +63,14 @@ type EventUser struct {
 type BeatmapSearch struct {
 	Beatmapsets []Beatmapset `json:"beatmapsets"`
 }
+
+type BeatmapsetEvents struct {
+	Events []BeatmapsetEvent `json:"events"`
+}
+
+type BeatmapsetEvent struct {
+	ID         int        `json:"id"`
+	Type       string     `json:"type"`
+	Beatmapset Beatmapset `json:"beatmapset"`
+	UserID     int        `json:"user_id"`
+}

osuapi/osuapi.go

@@ -4,15 +4,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"log"
 	"net/http"
-	"net/url"
-	"os"
 	"strings"
+	"sync"
 	"time"
 
+	"github.com/pkg/errors"
 	"golang.org/x/sync/semaphore"
 
 	"subscribe-bot/config"
@@ -26,6 +25,9 @@ type Osuapi struct {
 	token      string
 	expires    time.Time
 	config     *config.Config
+
+	tokenLock       sync.RWMutex
+	isFetchingToken bool
 }
 
 func New(config *config.Config) *Osuapi {
@@ -35,17 +37,39 @@ func New(config *config.Config) *Osuapi {
 
 	// want to cap at around 1000 requests a minute, OSU cap is 1200
 	lock := semaphore.NewWeighted(1000)
-	return &Osuapi{client, lock, "", time.Now(), config}
+
+	return &Osuapi{
+		httpClient: client,
+		lock:       lock,
+		expires:    time.Now(),
+		config:     config,
+	}
 }
 
 func (api *Osuapi) Token() (token string, err error) {
-	if time.Now().Before(api.expires) {
+	return api.tokenFetch(false)
+}
+
+func (api *Osuapi) tokenFetch(force bool) (token string, err error) {
+	if !force {
+		if time.Now().Before(api.expires) {
+			token = api.token
+			return
+		}
+	}
+
+	if api.isFetchingToken {
+		api.tokenLock.RLock()
 		token = api.token
+		api.tokenLock.RUnlock()
 		return
 	}
 
+	api.tokenLock.Lock()
+	api.isFetchingToken = true
+
 	data := fmt.Sprintf(
-		"client_id=%d&client_secret=%s&grant_type=client_credentials&scope=public",
+		"client_id=%s&client_secret=%s&grant_type=client_credentials&scope=public",
 		api.config.Oauth.ClientId,
 		api.config.Oauth.ClientSecret,
 	)
@@ -56,24 +80,29 @@ func (api *Osuapi) Token() (token string, err error) {
 		strings.NewReader(data),
 	)
 	if err != nil {
+		err = errors.Wrap(err, "failed to make POST request")
 		return
 	}
 
 	var osuToken OsuToken
 	respBody, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
+		err = errors.Wrap(err, "failed to read response body")
 		return
 	}
 
 	err = json.Unmarshal(respBody, &osuToken)
 	if err != nil {
+		err = errors.Wrap(err, "failed to unmarshal response body as json")
 		return
 	}
 
 	log.Println("got new access token", osuToken.AccessToken[:12]+"...")
 	api.token = osuToken.AccessToken
 	api.expires = time.Now().Add(time.Duration(osuToken.ExpiresIn) * time.Second)
+
 	token = api.token
+	api.tokenLock.Unlock()
 	return
 }
 
@@ -87,20 +116,26 @@ func (api *Osuapi) Request0(action string, url string) (resp *http.Response, err
 
 	token, err := api.Token()
 	if err != nil {
+		err = errors.Wrap(err, "failed to fetch token")
 		return
 	}
 
 	req.Header.Add("Authorization", "Bearer "+token)
-	if err != nil {
-		return
-	}
 
 	resp, err = api.httpClient.Do(req)
 	if err != nil {
+		err = errors.Wrap(err, "http client failed")
 		return
 	}
 
 	if resp.StatusCode != 200 {
+		if resp.StatusCode == 401 {
+			// force fetch token again
+			api.tokenFetch(true)
+			log.Println("re-fetched token cus expired")
+			return api.Request0(action, url)
+		}
+
 		var respBody []byte
 		respBody, err = ioutil.ReadAll(resp.Body)
 		if err != nil {
@@ -122,103 +157,17 @@ func (api *Osuapi) Request0(action string, url string) (resp *http.Response, err
 func (api *Osuapi) Request(action string, url string, result interface{}) (err error) {
 	resp, err := api.Request0(action, url)
 	if err != nil {
-		return
+		return errors.Wrap(err, "base request failed")
 	}
 
 	data, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return
+		return errors.Wrap(err, "failed to read http response body")
 	}
 
 	err = json.Unmarshal(data, result)
 	if err != nil {
-		return
+		return errors.Wrap(err, "failed to unmarshal http response as json")
-	}
-
-	return
-}
-
-func (api *Osuapi) DownloadSingleBeatmap(beatmapId int, path string) (err error) {
-	url := fmt.Sprintf("https://osu.ppy.sh/osu/%d", beatmapId)
-	resp, err := api.httpClient.Get(url)
-
-	file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0644)
-	if err != nil {
-		return
-	}
-
-	_, err = io.Copy(file, resp.Body)
-	if err != nil {
-		return
-	}
-	return
-}
-
-func (api *Osuapi) GetBeatmapSet(beatmapSetId int) (beatmapSet Beatmapset, err error) {
-	url := fmt.Sprintf("/beatmapsets/%d", beatmapSetId)
-	err = api.Request("GET", url, &beatmapSet)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (api *Osuapi) BeatmapsetDownload(beatmapSetId int) (path string, err error) {
-	url := fmt.Sprintf("/beatmapsets/%d/download", beatmapSetId)
-	resp, err := api.Request0("GET", url)
-	if err != nil {
-		return
-	}
-
-	file, err := ioutil.TempFile(os.TempDir(), "beatmapsetDownload")
-	if err != nil {
-		return
-	}
-
-	_, err = io.Copy(file, resp.Body)
-	if err != nil {
-		return
-	}
-	file.Close()
-
-	path = file.Name()
-	return
-}
-
-func (api *Osuapi) GetUser(userId string) (user User, err error) {
-	url := fmt.Sprintf("/users/%s", userId)
-	err = api.Request("GET", url, &user)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (api *Osuapi) GetUserEvents(userId int, limit int, offset int) (events []Event, err error) {
-	url := fmt.Sprintf(
-		"/users/%d/recent_activity?limit=%d&offset=%d",
-		userId,
-		limit,
-		offset,
-	)
-	err = api.Request("GET", url, &events)
-	if err != nil {
-		return
-	}
-
-	return
-}
-
-func (api *Osuapi) SearchBeatmaps(rankStatus string) (beatmapSearch BeatmapSearch, err error) {
-	values := url.Values{}
-	values.Set("s", rankStatus)
-	query := values.Encode()
-	url := "/beatmapsets/search?" + query
-	err = api.Request("GET", url, &beatmapSearch)
-	if err != nil {
-		return
 	}
 
 	return

osuapi/users.go

@@ -0,0 +1,28 @@
+package osuapi
+
+import "fmt"
+
+func (api *Osuapi) GetUser(userId string) (user User, err error) {
+	url := fmt.Sprintf("/users/%s", userId)
+	err = api.Request("GET", url, &user)
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (api *Osuapi) GetUserEvents(userId int, limit int, offset int) (events []Event, err error) {
+	url := fmt.Sprintf(
+		"/users/%d/recent_activity?limit=%d&offset=%d",
+		userId,
+		limit,
+		offset,
+	)
+	err = api.Request("GET", url, &events)
+	if err != nil {
+		return
+	}
+
+	return
+}

scrape/nominated.go

@@ -0,0 +1,16 @@
+package scrape
+
+import (
+	"subscribe-bot/osuapi"
+)
+
+func (s *Scraper) scrapeNominatedMaps() {
+	events, _ := s.api.GetBeatmapsetEvents(&osuapi.GetBeatmapsetEventsOptions{
+		Types: []string{"nominate", "qualify"},
+	})
+
+	for _, event := range events {
+		(func(_ osuapi.BeatmapsetEvent) {})(event)
+		// fmt.Println(event)
+	}
+}

scrape/pending.go

@@ -0,0 +1,145 @@
+package scrape
+
+import (
+	"fmt"
+	"log"
+	"subscribe-bot/db"
+	"subscribe-bot/osuapi"
+	"time"
+)
+
+func (s *Scraper) scrapePendingMaps() {
+	// build a list of currently tracked mappers
+	trackedMappers := make(map[int]int)
+	s.db.IterAllTrackedMappers(func(mapper db.User) error {
+		trackedMappers[mapper.ID] = 1
+		return nil
+	})
+
+	// TODO: is this sorted for sure??
+	pendingSets, err := s.api.SearchBeatmaps("pending")
+	if err != nil {
+		log.Println("error fetching pending sets", err)
+		s.bot.NotifyError("failed to fetch pending sets", err)
+		return
+	}
+
+	allNewMaps := make(map[int][]osuapi.Beatmapset, 0)
+	var newLastUpdateTime = time.Unix(0, 0)
+	for _, beatmapSet := range pendingSets.Beatmapsets {
+		updatedTime, err := time.Parse(time.RFC3339, beatmapSet.LastUpdated)
+		if err != nil {
+			log.Println("error parsing last updated time", updatedTime)
+		}
+
+		if updatedTime.After(newLastUpdateTime) {
+			// update lastUpdateTime to latest updated map
+			newLastUpdateTime = updatedTime
+		}
+
+		if !updatedTime.After(lastUpdateTime) {
+			break
+		}
+
+		mapperId := beatmapSet.UserID
+
+		// if _, ok := trackedMappers[mapperId]; ok {
+		if _, ok2 := allNewMaps[mapperId]; !ok2 {
+			allNewMaps[mapperId] = make([]osuapi.Beatmapset, 0)
+		}
+
+		allNewMaps[mapperId] = append(allNewMaps[mapperId], beatmapSet)
+		// }
+	}
+
+	if len(allNewMaps) > 0 {
+		for mapperId, newMaps := range allNewMaps {
+			channels := make([]string, 0)
+			s.db.IterTrackingChannels(mapperId, func(channel db.DiscordChannel) error {
+				channels = append(channels, channel.ID)
+				return nil
+			})
+
+			err := s.bot.NotifyNewBeatmap(channels, newMaps)
+			if err != nil {
+				log.Println("error notifying new maps:", err)
+			}
+		}
+	}
+
+	lastUpdateTime = newLastUpdateTime
+	log.Println("last updated time", lastUpdateTime)
+}
+
+func getNewMaps(db *db.Db, api *osuapi.Osuapi, userId int) (newMaps []osuapi.Event, err error) {
+	// see if there's a last event
+	hasLastEvent, lastEventId := db.MapperLastEvent(userId)
+	newMaps = make([]osuapi.Event, 0)
+	var (
+		events            []osuapi.Event
+		newLatestEvent    = 0
+		updateLatestEvent = false
+	)
+	if hasLastEvent {
+		offset := 0
+
+	loop:
+		for {
+			events, err = api.GetUserEvents(userId, 50, offset)
+			if err != nil {
+				err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err)
+				return
+			}
+			if len(events) == 0 {
+				break
+			}
+
+			for _, event := range events {
+				if event.ID == lastEventId {
+					break loop
+				}
+
+				if event.ID > newLatestEvent {
+					updateLatestEvent = true
+					newLatestEvent = event.ID
+				}
+
+				if event.Type == "beatmapsetUpload" ||
+					event.Type == "beatmapsetRevive" ||
+					event.Type == "beatmapsetUpdate" {
+					newMaps = append(newMaps, event)
+				}
+			}
+
+			offset += len(events)
+		}
+	} else {
+		log.Printf("no last event id found for %d\n", userId)
+		events, err = api.GetUserEvents(userId, 50, 0)
+		if err != nil {
+			return
+		}
+
+		for _, event := range events {
+			if event.ID > newLatestEvent {
+				updateLatestEvent = true
+				newLatestEvent = event.ID
+			}
+
+			if event.Type == "beatmapsetUpload" ||
+				event.Type == "beatmapsetRevive" ||
+				event.Type == "beatmapsetUpdate" {
+				newMaps = append(newMaps, event)
+			}
+		}
+	}
+
+	if updateLatestEvent {
+		err = db.UpdateMapperLastEvent(userId, newLatestEvent)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}

scrape/scrape.go

@@ -1,8 +1,6 @@
 package scrape
 
 import (
-	"fmt"
-	"log"
 	"time"
 
 	"subscribe-bot/config"
@@ -13,151 +11,26 @@ import (
 
 var (
 	refreshInterval = 30 * time.Second
+	lastUpdateTime  time.Time
 	Ticker          = time.NewTicker(refreshInterval)
 )
 
+type Scraper struct {
+	config *config.Config
+	bot    *discord.Bot
+	db     *db.Db
+	api    *osuapi.Osuapi
+}
+
 func RunScraper(config *config.Config, bot *discord.Bot, db *db.Db, api *osuapi.Osuapi) {
-	lastUpdateTime := time.Now()
+	lastUpdateTime = time.Now()
+
+	scraper := Scraper{config, bot, db, api}
+
 	go func() {
 		for ; true; <-Ticker.C {
-			// build a list of currently tracked mappers
+			scraper.scrapePendingMaps()
-			trackedMappers := make(map[int]int)
+			scraper.scrapeNominatedMaps()
-			db.IterTrackedMappers(func(userId int) error {
-				trackedMappers[userId] = 1
-				return nil
-			})
-
-			// TODO: is this sorted for sure??
-			pendingSets, err := api.SearchBeatmaps("pending")
-			if err != nil {
-				log.Println("error fetching pending sets", err)
-			}
-
-			allNewMaps := make(map[int][]osuapi.Beatmapset, 0)
-			var newLastUpdateTime = time.Unix(0, 0)
-			for _, beatmapSet := range pendingSets.Beatmapsets {
-				updatedTime, err := time.Parse(time.RFC3339, beatmapSet.LastUpdated)
-				if err != nil {
-					log.Println("error parsing last updated time", updatedTime)
-				}
-
-				if updatedTime.After(newLastUpdateTime) {
-					// update lastUpdateTime to latest updated map
-					newLastUpdateTime = updatedTime
-				}
-
-				if !updatedTime.After(lastUpdateTime) {
-					break
-				}
-
-				mapperId := beatmapSet.UserId
-				if _, ok := trackedMappers[mapperId]; ok {
-					if _, ok2 := allNewMaps[mapperId]; !ok2 {
-						allNewMaps[mapperId] = make([]osuapi.Beatmapset, 0)
-					}
-
-					allNewMaps[mapperId] = append(allNewMaps[mapperId], beatmapSet)
-				}
-			}
-
-			if len(allNewMaps) > 0 {
-				for mapperId, newMaps := range allNewMaps {
-					channels := make([]string, 0)
-					db.IterTrackingChannels(mapperId, func(channelId string) error {
-						channels = append(channels, channelId)
-						return nil
-					})
-
-					err := bot.NotifyNewBeatmap(channels, newMaps)
-					if err != nil {
-						log.Println("error notifying new maps:", err)
-					}
-				}
-			}
-
-			lastUpdateTime = newLastUpdateTime
-			// this rings the terminal bell when it's updated so i don't have to stare
-			// at a blank screen for 30 seconds waiting for the feed to update
-			if config.Debug {
-				fmt.Print("\a")
-			}
-			log.Println("last updated time", lastUpdateTime)
 		}
 	}()
 }
-
-func getNewMaps(db *db.Db, api *osuapi.Osuapi, userId int) (newMaps []osuapi.Event, err error) {
-	// see if there's a last event
-	hasLastEvent, lastEventId := db.MapperLastEvent(userId)
-	newMaps = make([]osuapi.Event, 0)
-	var (
-		events            []osuapi.Event
-		newLatestEvent    = 0
-		updateLatestEvent = false
-	)
-	if hasLastEvent {
-		offset := 0
-
-	loop:
-		for {
-			events, err = api.GetUserEvents(userId, 50, offset)
-			if err != nil {
-				err = fmt.Errorf("couldn't load events for user %d, offset %d: %w", userId, offset, err)
-				return
-			}
-			if len(events) == 0 {
-				break
-			}
-
-			for _, event := range events {
-				if event.ID == lastEventId {
-					break loop
-				}
-
-				if event.ID > newLatestEvent {
-					updateLatestEvent = true
-					newLatestEvent = event.ID
-				}
-
-				if event.Type == "beatmapsetUpload" ||
-					event.Type == "beatmapsetRevive" ||
-					event.Type == "beatmapsetUpdate" {
-					newMaps = append(newMaps, event)
-				}
-			}
-
-			offset += len(events)
-		}
-	} else {
-		log.Printf("no last event id found for %d\n", userId)
-		events, err = api.GetUserEvents(userId, 50, 0)
-		if err != nil {
-			return
-		}
-
-		for _, event := range events {
-			if event.ID > newLatestEvent {
-				updateLatestEvent = true
-				newLatestEvent = event.ID
-			}
-
-			if event.Type == "beatmapsetUpload" ||
-				event.Type == "beatmapsetRevive" ||
-				event.Type == "beatmapsetUpdate" {
-				newMaps = append(newMaps, event)
-			}
-		}
-	}
-
-	// TODO: debug
-	// updateLatestEvent = false
-
-	if updateLatestEvent {
-		err = db.UpdateMapperLatestEvent(userId, newLatestEvent)
-		if err != nil {
-			return
-		}
-	}
-
-	return
-}

web/auth.go

@@ -0,0 +1,81 @@
+package web
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/gin-gonic/contrib/sessions"
+	"github.com/gin-gonic/gin"
+)
+
+func (web *Web) logout(c *gin.Context) {
+	session := sessions.Default(c)
+	session.Delete("access_token")
+	session.Save()
+
+	c.Redirect(http.StatusTemporaryRedirect, "/")
+}
+
+func (web *Web) login(c *gin.Context) {
+	url := url.URL{
+		Scheme: "https",
+		Host:   "osu.ppy.sh",
+		Path:   "/oauth/authorize",
+	}
+	q := url.Query()
+	q.Set("client_id", web.config.Oauth.ClientId)
+	q.Set("redirect_uri", web.config.Web.ServedAt+"/login/callback")
+	q.Set("response_type", "code")
+	q.Set("scope", "identify public")
+	q.Set("state", "urmom")
+	url.RawQuery = q.Encode()
+	fmt.Println("redirecting to", url.String())
+	c.Redirect(http.StatusTemporaryRedirect, url.String())
+}
+
+func (web *Web) loginCallback(c *gin.Context) {
+	receivedCode := c.Query("code")
+
+	bodyQuery := url.Values{}
+	bodyQuery.Set("client_id", web.config.Oauth.ClientId)
+	bodyQuery.Set("client_secret", web.config.Oauth.ClientSecret)
+	bodyQuery.Set("code", receivedCode)
+	bodyQuery.Set("grant_type", "authorization_code")
+	bodyQuery.Set("redirect_uri", web.config.Web.ServedAt+"/login/callback")
+	body := strings.NewReader(bodyQuery.Encode())
+	resp, _ := web.hc.Post("https://osu.ppy.sh/oauth/token", "application/x-www-form-urlencoded", body)
+	respBody, _ := ioutil.ReadAll(resp.Body)
+	type OsuToken struct {
+		TokenType    string `json:"token_type"`
+		ExpiresIn    int    `json:"expires_in"`
+		AccessToken  string `json:"access_token"`
+		RefreshToken string `json:"refresh_token"`
+	}
+	var token OsuToken
+	_ = json.Unmarshal(respBody, &token)
+	fmt.Println("TOKEN", token)
+
+	session := sessions.Default(c)
+	session.Set("access_token", token.AccessToken)
+	session.Save()
+
+	c.Redirect(http.StatusTemporaryRedirect, "/")
+}
+
+func isLoggedIn(c *gin.Context) bool {
+	session := sessions.Default(c)
+	var accessToken string
+	loggedIn := false
+	accessTokenI := session.Get("access_token")
+	if accessTokenI != nil {
+		accessToken = accessTokenI.(string)
+		if len(accessToken) > 0 {
+			loggedIn = true
+		}
+	}
+	return loggedIn
+}

web/mapper.go

@@ -0,0 +1,38 @@
+package web
+
+import (
+	"net/http"
+	"strconv"
+	"subscribe-bot/db"
+
+	"github.com/gin-gonic/gin"
+	"github.com/pkg/errors"
+)
+
+func (web *Web) mapperIndex(c *gin.Context) (err error) {
+	userId := c.Param("userId")
+	mapperId, err := strconv.Atoi(userId)
+	if err != nil {
+		err = errors.Wrap(err, "id is not an integer")
+		return
+	}
+
+	mapper, err := web.db.GetUser(mapperId)
+	if err != nil {
+		err = errors.Wrap(err, "failed to get user from mapper-index")
+		return
+	}
+
+	beatmapsets := make([]db.Beatmapset, 0)
+	web.db.IterTrackedBeatmapsetsBy(mapperId, func(beatmapset db.Beatmapset) error {
+		beatmapsets = append(beatmapsets, beatmapset)
+		return nil
+	})
+
+	web.render(c, http.StatusOK, "mapper-index.html", gin.H{
+		"Mapper":      mapper,
+		"Beatmapsets": beatmapsets,
+	})
+
+	return
+}

web/render.go

@@ -0,0 +1,30 @@
+package web
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/gin-gonic/gin"
+)
+
+func (web *Web) errorWrap(fn func(*gin.Context) error) func(*gin.Context) {
+	return func(c *gin.Context) {
+		err := fn(c)
+		if err != nil {
+			c.String(http.StatusInternalServerError, "error")
+			log.Println("fatal error", err)
+		}
+	}
+}
+
+func (web *Web) render(c *gin.Context, code int, tmpl string, obj gin.H) {
+	base := gin.H{
+		"IsLoggedIn": isLoggedIn(c),
+	}
+
+	for key, val := range obj {
+		base[key] = val
+	}
+
+	c.HTML(code, tmpl, base)
+}

web/repo.go

@@ -0,0 +1,117 @@
+package web
+
+import (
+	"archive/zip"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/gin-gonic/gin"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+func (web *Web) mapVersions(c *gin.Context) (err error) {
+	userId := c.Param("userId")
+	mapId := c.Param("mapId")
+
+	id, _ := strconv.Atoi(mapId)
+	bs, _ := web.api.GetBeatmapSet(id)
+
+	repoDir := path.Join(web.config.Repos, userId, mapId)
+	repo, _ := git.PlainOpen(repoDir)
+
+	type Revision struct {
+		Date      time.Time
+		HumanDate string
+		Summary   string
+		Hash      string
+		HasParent bool
+	}
+
+	versions := make([]Revision, 0)
+	logIter, _ := repo.Log(&git.LogOptions{})
+	for i := 0; i < 20; i++ {
+		commit, err := logIter.Next()
+		if err == io.EOF {
+			break
+		}
+
+		stats, _ := commit.Stats()
+		_, err = commit.Parent(0)
+		hasParent := !errors.Is(err, object.ErrParentNotFound)
+
+		versions = append(versions, Revision{
+			Date:      commit.Author.When,
+			HumanDate: humanize.Time(commit.Author.When),
+			Summary:   stats.String(),
+			Hash:      commit.Hash.String(),
+			HasParent: hasParent,
+		})
+	}
+
+	web.render(c, http.StatusOK, "map-version.html", gin.H{
+		"Beatmapset": bs,
+		"LoggedIn":   isLoggedIn(c),
+		"Versions":   versions,
+	})
+	return
+}
+
+func (web *Web) mapPatch(c *gin.Context) (err error) {
+	userId := c.Param("userId")
+	mapId := c.Param("mapId")
+	hash := c.Param("hash")
+
+	repoDir := path.Join(web.config.Repos, userId, mapId)
+	repo, _ := git.PlainOpen(repoDir)
+
+	hashObj := plumbing.NewHash(hash)
+	commit, _ := repo.CommitObject(hashObj)
+	parent, _ := commit.Parent(0)
+	patch, _ := parent.Patch(commit)
+
+	c.String(http.StatusOK, "text/plain", patch.String())
+	return
+}
+
+func (web *Web) mapZip(c *gin.Context) (err error) {
+	userId := c.Param("userId")
+	mapId := c.Param("mapId")
+	hash := c.Param("hash")
+
+	repoDir := path.Join(web.config.Repos, userId, mapId)
+	repo, _ := git.PlainOpen(repoDir)
+
+	hashObj := plumbing.NewHash(hash)
+	commit, _ := repo.CommitObject(hashObj)
+	tree, _ := commit.Tree()
+
+	files := tree.Files()
+
+	c.Writer.Header().Set("Content-type", "application/octet-stream")
+	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s-%s.zip", mapId, hash))
+	c.Stream(func(w io.Writer) bool {
+		ar := zip.NewWriter(w)
+		for {
+			file, err := files.Next()
+			if err == io.EOF {
+				break
+			}
+
+			reader, _ := file.Reader()
+			fdest, _ := ar.Create(file.Name)
+			io.Copy(fdest, reader)
+		}
+		ar.Close()
+
+		return false
+	})
+	return
+}

web/static/main.css

@@ -0,0 +1,53 @@
+:root {
+  --primary-color: #09c;
+}
+
+body, html {
+  font-family: Arial, Helvetica, sans-serif;
+}
+
+.title a {
+  color: black;
+  text-decoration: none;
+}
+
+.nav-bar {
+  background-color: var(--primary-color);
+  padding: 5px;
+}
+
+.nav-bar a {
+  color: white;
+  padding: 7px;
+  margin: 0px 3px;
+}
+
+a {
+  color: var(--primary-color);
+  text-decoration: none;
+}
+
+a:hover {
+  text-decoration: underline;
+}
+
+/* containers */
+
+.container {
+  width: 90%;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+@media only screen and (min-width: 33.75em) {  /* 540px */
+  .container {
+    width: 80%;
+  }
+}
+
+@media only screen and (min-width: 60em) { /* 960px */
+  .container {
+    width: 75%;
+    max-width: 60rem;
+  }
+}

web/templates/index.html

@@ -0,0 +1,35 @@
+{{ define "content" }}
+
+<h3>Stats</h3>
+
+Currently tracking <b>{{ .TotalMaps }}</b> maps from <b>{{ .TotalUsers }}</b> users.
+
+<h3>Last 10 Updated Beatmaps</h3>
+
+<table class="table-auto">
+    <thead>
+        <th>Links</th>
+        <th>Title</th>
+        <th>Mapper</th>
+        <th>Updated</th>
+    </thead>
+    <tbody>
+    {{ range .Beatmapsets }}
+        <tr>
+            <td>
+                <a href="https://osu.ppy.sh/s/{{ .ID }}" target="_blank">[osu]</a>
+                <a href="/map/{{ .MapperID }}/{{ .ID }}/versions">[versions]</a>
+            </td>
+            <td>{{ .Artist }} - {{ .Title }}</td>
+            <td>
+                <a href="/map/{{ .MapperID }}">{{ .Mapper.Username }}</a>
+            </td>
+            <td>
+                {{ .LastUpdated | humanize }}
+            </td>
+        </tr>
+    {{ end }}
+    </tbody>
+</table>
+
+{{ end }}

web/templates/map-version.html

@@ -0,0 +1,36 @@
+{{ define "content" }}
+
+<h3>versions of {{ .Beatmapset.Artist }} - {{ .Beatmapset.Title }}</h3>
+
+<p>
+    <a href="https://osu.ppy.sh/s/{{ .Beatmapset.ID }}" target="_blank">Map link</a>
+    &middot;
+    mapped by <a href="https://osu.ppy.sh/u/{{ .Beatmapset.UserID }}" target="_blank">{{ .Beatmapset.Creator }}</a>
+</p>
+
+<small>up to the latest 20 revisions, pagination coming later</small>
+
+<table>
+    <thead>
+        <th>Date</th>
+        <th>Links</th>
+        <th>Summary</th>
+    </thead>
+
+    <tbody>
+    {{ range .Versions }}
+        <tr>
+            <td><span title="{{ .Date }}">{{ .HumanDate }}</span></td>
+            <td>
+                <a href="zip/{{ .Hash }}" target="_blank">[zip]</a>
+                {{ if .HasParent }}
+                    <a href="patch/{{ .Hash }}" target="_blank">[patch]</a>
+                {{ end }}
+            </td>
+            <td><pre>{{ .Summary }}</pre></td>
+        </tr>
+    {{ end }}
+    </tbody>
+</table>
+
+{{ end }}

web/templates/mapper-index.html

@@ -0,0 +1,27 @@
+{{ define "content" }}
+
+<h3>Tracked Maps by <a href="https://osu.ppy.sh/u/{{ .Mapper.ID }}" target="_blank">{{ .Mapper.Username }}</a></h3>
+
+<table class="table-auto">
+    <thead>
+        <th>Links</th>
+        <th>Title</th>
+        <th>Updated</th>
+    </thead>
+    <tbody>
+    {{ range .Beatmapsets }}
+        <tr>
+            <td>
+                <a href="https://osu.ppy.sh/s/{{ .ID }}" target="_blank">[osu]</a>
+                <a href="/map/{{ .MapperID }}/{{ .ID }}/versions">[versions]</a>
+            </td>
+            <td>{{ .Artist }} - {{ .Title }}</td>
+            <td>
+                {{ .LastUpdated | humanize }}
+            </td>
+        </tr>
+    {{ end }}
+    </tbody>
+</table>
+
+{{ end }}

web/templates/master.html

@@ -0,0 +1,45 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <title>subscribe-bot</title>
+
+        <meta name="viewport" content="width=device-width, initial-scale=1">
+
+        <link rel="stylesheet" href="/static/main.css" />
+    </head>
+
+    <body>
+        <div class="container">
+            <h1 class="title">
+                <a href="/">subscribe-bot</a>
+            </h1>
+
+            <small>logging in with your osu account does nothing</small>
+
+            <div class="nav-bar">
+                <a href="/">home</a>
+
+                {{ if .LoggedIn }}
+                    <a href="/logout">logout</a>
+                {{ else }}
+                    <a href="/login">login</a>
+                {{ end }}
+            </div>
+
+            {{ template "content" . }}
+
+            <footer>
+                <hr />
+                <small>
+                    Maintained by <a href="https://osu.ppy.sh/u/2688103" target="_blank">IOException</a>
+                    &middot;
+                    <a href="https://git.mzhang.io/osu/subscribe-bot" target="_blank">Source code</a>
+                    &middot;
+                    <a href="https://discord.gg/eqjVG2H" target="_blank">Discord</a>
+                    &middot;
+                    subscribe-bot v{{ GitCommit }}
+                </small>
+            </footer>
+        </div>
+    </body>
+</html>

web/web.go

@@ -2,24 +2,98 @@ package web
 
 import (
 	"fmt"
+	"html/template"
+	"net/http"
+	"time"
 
+	"github.com/dustin/go-humanize"
+	"github.com/foolin/goview"
+	"github.com/foolin/goview/supports/ginview"
+	"github.com/gin-contrib/static"
+	"github.com/gin-gonic/contrib/sessions"
 	"github.com/gin-gonic/gin"
+	"github.com/kofalt/go-memoize"
 
 	"subscribe-bot/config"
+	"subscribe-bot/db"
+	"subscribe-bot/osuapi"
 )
 
-func RunWeb(config *config.Config) {
+const (
-	r := gin.Default()
+	USER_KEY = "user"
-	if !config.Debug {
+)
+
+var (
+	cache = memoize.NewMemoizer(90*time.Second, 10*time.Minute)
+)
+
+type Web struct {
+	config  *config.Config
+	api     *osuapi.Osuapi
+	hc      *http.Client
+	db      *db.Db
+	version string
+}
+
+func RunWeb(config *config.Config, api *osuapi.Osuapi, db *db.Db, version string) {
+	hc := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+
+	web := Web{config, api, hc, db, version}
+	web.Run()
+}
+
+func (web *Web) Run() {
+	if !web.config.Debug {
 		gin.SetMode(gin.ReleaseMode)
 	}
 
-	r.GET("/ping", func(c *gin.Context) {
+	r := gin.Default()
-		c.JSON(200, gin.H{
+	r.Use(gin.Recovery())
-			"message": "pong",
+	r.Use(static.Serve("/static", static.LocalFile("web/static", false)))
+	r.Use(sessions.Sessions("mysession", sessions.NewCookieStore([]byte(web.config.Web.SessionSecret))))
+
+	r.HTMLRender = ginview.New(goview.Config{
+		Root:         "web/templates",
+		Master:       "master.html",
+		DisableCache: web.config.Debug,
+		Funcs: template.FuncMap{
+			"GitCommit": func() string {
+				return web.version
+			},
+			"humanize": humanize.Time,
+		},
+	})
+
+	r.GET("/logout", web.logout)
+	r.GET("/login", web.login)
+	r.GET("/login/callback", web.loginCallback)
+
+	r.GET("/map/:userId", web.errorWrap(web.mapperIndex))
+	r.GET("/map/:userId/:mapId/versions", web.errorWrap(web.mapVersions))
+	r.GET("/map/:userId/:mapId/patch/:hash", web.errorWrap(web.mapPatch))
+	r.GET("/map/:userId/:mapId/zip/:hash", web.errorWrap(web.mapZip))
+
+	r.GET("/", func(c *gin.Context) {
+		beatmapSets := web.listRepos()
+		stats := web.db.GetStats()
+		web.render(c, http.StatusOK, "index.html", gin.H{
+			"Beatmapsets": beatmapSets,
+			"TotalMaps":   stats.TotalMaps,
+			"TotalUsers":  stats.TotalUsers,
 		})
 	})
 
-	addr := fmt.Sprintf("%s:%d", config.Web.Host, config.Web.Port)
+	addr := fmt.Sprintf("%s:%d", web.config.Web.Host, web.config.Web.Port)
 	r.Run(addr)
 }
+
+func (web *Web) listRepos() []db.Beatmapset {
+	beatmapSets := make([]db.Beatmapset, 0)
+	web.db.IterTrackedBeatmapsets(10, func(beatmapset db.Beatmapset) error {
+		beatmapSets = append(beatmapSets, beatmapset)
+		return nil
+	})
+	return beatmapSets
+}