More efficient home page, and limit to 10

db/beatmap.go

@@ -1,6 +1,10 @@
 package db
 
-import "gorm.io/gorm"
+import (
+	"time"
+
+	"gorm.io/gorm"
+)
 
 type Beatmapset struct {
 	gorm.Model
@@ -8,5 +12,6 @@ type Beatmapset struct {
 	Artist      string
 	Title       string
 	MapperID    int
-	Mapper   User `gorm:"foreignKey:MapperID;references:ID"`
+	Mapper      User      `gorm:"foreignKey:MapperID"`
+	LastUpdated time.Time `gorm:"last_updated"`
 }

db/db.go

@@ -6,13 +6,16 @@ package db
 // channel/<channel_id>/tracks/<mapper_id> -> priority
 
 import (
+	"fmt"
 	"strconv"
 
 	"github.com/pkg/errors"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
+	"gorm.io/gorm/logger"
 
+	"subscribe-bot/config"
 	"subscribe-bot/osuapi"
 )
 
@@ -27,8 +30,13 @@ type Db struct {
 	api  *osuapi.Osuapi
 }
 
-func OpenDb(path string, api *osuapi.Osuapi) (db *Db, err error) {
+func OpenDb(cfg config.Config, api *osuapi.Osuapi) (db *Db, err error) {
-	gorm, err := gorm.Open(sqlite.Open(path), &gorm.Config{})
+	gormConfig := &gorm.Config{}
+	if cfg.Debug {
+		gormConfig.Logger = logger.Default.LogMode(logger.Info)
+	}
+
+	gorm, err := gorm.Open(sqlite.Open(cfg.DatabasePath), gormConfig)
 	if err != nil {
 		panic("failed to connect database")
 	}
@@ -43,6 +51,21 @@ func OpenDb(path string, api *osuapi.Osuapi) (db *Db, err error) {
 	return
 }
 
+func (db *Db) SaveBeatmap(beatmap Beatmapset) {
+	db.gorm.Clauses(clause.OnConflict{UpdateAll: true}).Create(&beatmap)
+	return
+}
+
+func (db *Db) IterTrackedBeatmapsets(limit int, fn func(beatmapset Beatmapset) error) (err error) {
+	var beatmapsets []Beatmapset
+	db.gorm.Preload(clause.Associations).Limit(limit).Order("last_updated desc").Find(&beatmapsets)
+	fmt.Println("HELLOSU", beatmapsets)
+	for _, beatmapset := range beatmapsets {
+		fn(beatmapset)
+	}
+	return
+}
+
 // Loop over channels that are tracking this specific mapper
 func (db *Db) IterTrackingChannels(mapperId int, fn func(channel DiscordChannel) error) (err error) {
 	var channels []DiscordChannel
@@ -103,7 +126,7 @@ func (db *Db) MapperLastEvent(userId int) (has bool, id int) {
 
 // Start tracking a new mapper (if they're not already tracked)
 func (db *Db) ChannelTrackMapper(channelId string, mapperId int, priority int) (err error) {
-	err = db.gorm.Model(&DiscordChannel{ID: channelId}).Association("TrackedMappers").Append(db.getUser(mapperId))
+	err = db.gorm.Model(&DiscordChannel{ID: channelId}).Association("TrackedMappers").Append(db.GetUser(mapperId))
 	if err != nil {
 		err = errors.Wrap(err, "could not add tracking for channel "+channelId)
 		return
@@ -121,7 +144,7 @@ func (db *Db) ChannelTrackMapper(channelId string, mapperId int, priority int) (
 func (db *Db) Close() {
 }
 
-func (db *Db) getUser(userId int) (user *User, err error) {
+func (db *Db) GetUser(userId int) (user *User, err error) {
 	// TODO: cache user info for some time?
 
 	apiUser, err := db.api.GetUser(strconv.Itoa(userId))

db/user.go

@@ -9,4 +9,5 @@ type User struct {
 	Country            string
 	LatestEventID      int              `gorm:"latest_event_id"`
 	TrackingChannels   []DiscordChannel `gorm:"many2many:tracked_mappers"`
+	TrackedBeatmapsets []Beatmapset     `gorm:"foreignKey:MapperID"`
 }

discord/bot.go

@@ -179,6 +179,16 @@ func (bot *Bot) NotifyNewBeatmap(channels []string, newMaps []osuapi.Beatmapset)
 			foundPatch = true
 		}
 
+		fmt.Println("saving", beatmapSet)
+		bot.db.SaveBeatmap(db.Beatmapset{
+			ID:          beatmapSet.ID,
+			MapperID:    beatmapSet.UserID,
+			Artist:      beatmapSet.Artist,
+			Title:       beatmapSet.Title,
+			LastUpdated: eventTime,
+		})
+		bot.db.GetUser(beatmapSet.UserID)
+
 		embed := &discordgo.MessageEmbed{
 			URL:       fmt.Sprintf("%s/map/%d/%d/versions", bot.config.Web.ServedAt, beatmapSet.UserID, beatmapSet.ID),
 			Title:     fmt.Sprintf("Update: %s - %s", beatmapSet.Artist, beatmapSet.Title),

main.go

@@ -30,7 +30,7 @@ func main() {
 
 	api := osuapi.New(&config)
 
-	db, err := db.OpenDb(config.DatabasePath, api)
+	db, err := db.OpenDb(config, api)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -42,7 +42,7 @@ func main() {
 	}
 
 	go scrape.RunScraper(&config, bot, db, api)
-	go web.RunWeb(&config, api, GitCommit)
+	go web.RunWeb(&config, api, db, GitCommit)
 
 	signal_chan := make(chan os.Signal, 1)
 	signal.Notify(signal_chan,

web/templates/index.html

@@ -1,23 +1,27 @@
 {{ define "content" }}
 
-<p>Maps:</p>
+<p>Last 10 Updated Beatmaps:</p>
 
 <table class="table-auto">
     <thead>
         <th>Links</th>
         <th>Title</th>
         <th>Mapper</th>
+        <th>Updated</th>
     </thead>
     <tbody>
     {{ range .Beatmapsets }}
         <tr>
             <td>
                 <a href="https://osu.ppy.sh/s/{{ .ID }}" target="_blank">osu</a>
-                <a href="/map/{{ .UserID }}/{{ .ID }}/versions">versions</a>
+                <a href="/map/{{ .MapperID }}/{{ .ID }}/versions">versions</a>
             </td>
             <td>{{ .Artist }} - {{ .Title }}</td>
             <td>
-                <a href="https://osu.ppy.sh/u/{{ .UserID }}" target="_blank">{{ .Creator }}</a>
+                <a href="https://osu.ppy.sh/u/{{ .MapperID }}" target="_blank">{{ .Mapper.Username }}</a>
+            </td>
+            <td>
+                {{ .LastUpdated | humanize }}
             </td>
         </tr>
     {{ end }}

web/web.go

@@ -3,14 +3,10 @@ package web
 import (
 	"fmt"
 	"html/template"
-	"io/ioutil"
 	"net/http"
-	"os"
-	"path"
-	"strconv"
-	"sync"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/foolin/goview"
 	"github.com/foolin/goview/supports/ginview"
 	"github.com/gin-contrib/static"
@@ -19,6 +15,7 @@ import (
 	"github.com/kofalt/go-memoize"
 
 	"subscribe-bot/config"
+	"subscribe-bot/db"
 	"subscribe-bot/osuapi"
 )
 
@@ -34,15 +31,16 @@ type Web struct {
 	config  *config.Config
 	api     *osuapi.Osuapi
 	hc      *http.Client
+	db      *db.Db
 	version string
 }
 
-func RunWeb(config *config.Config, api *osuapi.Osuapi, version string) {
+func RunWeb(config *config.Config, api *osuapi.Osuapi, db *db.Db, version string) {
 	hc := &http.Client{
 		Timeout: 10 * time.Second,
 	}
 
-	web := Web{config, api, hc, version}
+	web := Web{config, api, hc, db, version}
 	web.Run()
 }
 
@@ -64,6 +62,7 @@ func (web *Web) Run() {
 			"GitCommit": func() string {
 				return web.version
 			},
+			"humanize": humanize.Time,
 		},
 	})
 
@@ -101,41 +100,11 @@ func isLoggedIn(c *gin.Context) bool {
 	return loggedIn
 }
 
-func (web *Web) listRepos() []osuapi.Beatmapset {
+func (web *Web) listRepos() []db.Beatmapset {
-	expensive := func() (interface{}, error) {
+	beatmapSets := make([]db.Beatmapset, 0)
-		repos := make([]int, 0)
+	web.db.IterTrackedBeatmapsets(10, func(beatmapset db.Beatmapset) error {
-		reposDir := web.config.Repos
+		beatmapSets = append(beatmapSets, beatmapset)
-		users, _ := ioutil.ReadDir(reposDir)
+		return nil
-
+	})
-		for _, user := range users {
+	return beatmapSets
-			userDir := path.Join(reposDir, user.Name())
-			var maps []os.FileInfo
-			maps, _ = ioutil.ReadDir(userDir)
-
-			for _, mapId := range maps {
-				mapDir := path.Join(userDir, mapId.Name())
-				fmt.Println(mapDir)
-
-				id, _ := strconv.Atoi(mapId.Name())
-				repos = append(repos, id)
-			}
-		}
-
-		beatmapSets := make([]osuapi.Beatmapset, len(repos))
-		var wg sync.WaitGroup
-		for i, repo := range repos {
-			wg.Add(1)
-			go func(i int, repo int) {
-				bs, _ := web.api.GetBeatmapSet(repo)
-				beatmapSets[i] = bs
-				wg.Done()
-			}(i, repo)
-		}
-		wg.Wait()
-
-		return beatmapSets, nil
-	}
-
-	result, _, _ := cache.Memoize("key1", expensive)
-	return result.([]osuapi.Beatmapset)
 }