marc/scotty
1
0
Fork 0
mirror of https://git.sr.ht/~phw/scotty synced 2025-06-05 12:58:33 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

jspf/scrobblerlog: return results in batches

Browse source 
This allows the importer to start working while export is still in progress
This commit is contained in:
Philipp Wolfer 2025-05-23 07:48:42 +02:00
parent c7af90b585
commit a8ce2be5d7
No known key found for this signature in database
GPG key ID: 8FDF744D4919943B
2 changed files with 21 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

9
internal/backends/scrobblerlog/scrobblerlog.go
View file

@ -30,6 +30,8 @@ import (
"go.uploadedlobster.com/scotty/pkg/scrobblerlog"
)
const batchSize = 1000
type ScrobblerLogBackend struct {
filePath string
ignoreSkipped bool
@ -152,7 +154,7 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp
return
}
listens := make(models.ListensList, 0, len(b.log.Records))
listens := make(models.ListensList, 0, batchSize)
client := strings.Split(b.log.Client, " ")[0]
p.Export.Total = int64(len(b.log.Records))
for _, record := range models.IterExportProgress(b.log.Records, &p, progress) {
@ -161,6 +163,11 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp
listens = append(listens, recordToListen(record, client))
p.Export.TotalItems += 1
}
if len(listens) >= batchSize {
results <- models.ListensResult{Items: listens}
listens = listens[:0]
}
}
sort.Sort(listens)