diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index e2bcde1..8da585f 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -36,6 +36,7 @@ const ( artistMBIDPrefix = "https://musicbrainz.org/artist/" recordingMBIDPrefix = "https://musicbrainz.org/recording/" releaseMBIDPrefix = "https://musicbrainz.org/release/" + batchSize = 1000 ) type JSPFBackend struct { @@ -107,7 +108,7 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti return } - listens := make(models.ListensList, 0, len(b.playlist.Tracks)) + listens := make(models.ListensList, 0, batchSize) p.Export.Total = int64(len(b.playlist.Tracks)) for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { listen, err := trackAsListen(track) @@ -115,6 +116,11 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti listens = append(listens, *listen) p.Export.TotalItems += 1 } + + if len(listens) >= batchSize { + results <- models.ListensResult{Items: listens} + listens = listens[:0] + } } sort.Sort(listens) @@ -150,7 +156,7 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time return } - loves := make(models.LovesList, 0, len(b.playlist.Tracks)) + loves := make(models.LovesList, 0, batchSize) p.Export.Total = int64(len(b.playlist.Tracks)) for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { love, err := trackAsLove(track) @@ -158,6 +164,11 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time loves = append(loves, *love) p.Export.TotalItems += 1 } + + if len(loves) >= batchSize { + results <- models.LovesResult{Items: loves} + loves = loves[:0] + } } sort.Sort(loves) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d42f3c..84080ae 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -30,6 +30,8 @@ import ( "go.uploadedlobster.com/scotty/pkg/scrobblerlog" ) +const batchSize = 1000 + type ScrobblerLogBackend struct { filePath string ignoreSkipped bool @@ -152,7 +154,7 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp return } - listens := make(models.ListensList, 0, len(b.log.Records)) + listens := make(models.ListensList, 0, batchSize) client := strings.Split(b.log.Client, " ")[0] p.Export.Total = int64(len(b.log.Records)) for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { @@ -161,6 +163,11 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens = append(listens, recordToListen(record, client)) p.Export.TotalItems += 1 } + + if len(listens) >= batchSize { + results <- models.ListensResult{Items: listens} + listens = listens[:0] + } } sort.Sort(listens)