marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-05-31 10:58:35 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

jspf/scrobblerlog: return results in batches

Browse source 
This allows the importer to start working while export is still in progress
This commit is contained in:
Philipp Wolfer 2025-05-23 07:48:42 +02:00
parent c7af90b585
commit a8ce2be5d7
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
2 changed files with 21 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

15
internal/backends/jspf/jspf.go
View file

@ -36,6 +36,7 @@ const (
artistMBIDPrefix = "https://musicbrainz.org/artist/"
recordingMBIDPrefix = "https://musicbrainz.org/recording/"
releaseMBIDPrefix = "https://musicbrainz.org/release/"
batchSize = 1000
)
type JSPFBackend struct {
@ -107,7 +108,7 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti
return
}
listens := make(models.ListensList, 0, len(b.playlist.Tracks))
listens := make(models.ListensList, 0, batchSize)
p.Export.Total = int64(len(b.playlist.Tracks))
for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) {
listen, err := trackAsListen(track)
@ -115,6 +116,11 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti
listens = append(listens, *listen)
p.Export.TotalItems += 1
}
if len(listens) >= batchSize {
results <- models.ListensResult{Items: listens}
listens = listens[:0]
}
}
sort.Sort(listens)
@ -150,7 +156,7 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time
return
}
loves := make(models.LovesList, 0, len(b.playlist.Tracks))
loves := make(models.LovesList, 0, batchSize)
p.Export.Total = int64(len(b.playlist.Tracks))
for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) {
love, err := trackAsLove(track)
@ -158,6 +164,11 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time
loves = append(loves, *love)
p.Export.TotalItems += 1
}
if len(loves) >= batchSize {
results <- models.LovesResult{Items: loves}
loves = loves[:0]
}
}
sort.Sort(loves)

9
internal/backends/scrobblerlog/scrobblerlog.go
View file

@ -30,6 +30,8 @@ import (
"go.uploadedlobster.com/scotty/pkg/scrobblerlog"
)
const batchSize = 1000
type ScrobblerLogBackend struct {
filePath string
ignoreSkipped bool
@ -152,7 +154,7 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp
return
}
listens := make(models.ListensList, 0, len(b.log.Records))
listens := make(models.ListensList, 0, batchSize)
client := strings.Split(b.log.Client, " ")[0]
p.Export.Total = int64(len(b.log.Records))
for _, record := range models.IterExportProgress(b.log.Records, &p, progress) {
@ -161,6 +163,11 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp
listens = append(listens, recordToListen(record, client))
p.Export.TotalItems += 1
}
if len(listens) >= batchSize {
results <- models.ListensResult{Items: listens}
listens = listens[:0]
}
}
sort.Sort(listens)