From c7af90b585bfe6ca59dbfc0ec4ea42eb49263909 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 07:47:52 +0200 Subject: [PATCH] More granular progress report for JSPF and scrobblerlog --- CHANGES.md | 1 + internal/backends/jspf/jspf.go | 24 ++++++------- .../backends/scrobblerlog/scrobblerlog.go | 13 ++++--- internal/models/models.go | 36 +++++++++++++++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 64f854f..8dc9838 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ - The import progress shows total items processed instead of time estimate - Fix program hanging endlessly if import fails (#11) - If import fails still store the last successfully imported timestamp + - More granular progress updates for JSPF and scrobblerlog - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 354640e..e2bcde1 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -108,21 +108,22 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti } listens := make(models.ListensList, 0, len(b.playlist.Tracks)) - for _, track := range b.playlist.Tracks { + p.Export.Total = int64(len(b.playlist.Tracks)) + for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { listen, err := trackAsListen(track) if err == nil && listen != nil && listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, *listen) + p.Export.TotalItems += 1 } } + sort.Sort(listens) - p.Export.Total = int64(len(listens)) - p.Export.Complete() - progress <- p results <- models.ListensResult{Items: listens} } func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - for _, listen := range export.Items { + p := models.TransferProgress{}.FromImportResult(importResult, false) + for _, listen := range models.IterImportProgress(export.Items, &p, progress) { if err := ctx.Err(); err != nil { return importResult, err } @@ -133,7 +134,6 @@ func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensRe importResult.UpdateTimestamp(listen.ListenedAt) } - progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } @@ -151,21 +151,22 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time } loves := make(models.LovesList, 0, len(b.playlist.Tracks)) - for _, track := range b.playlist.Tracks { + p.Export.Total = int64(len(b.playlist.Tracks)) + for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { love, err := trackAsLove(track) if err == nil && love != nil && love.Created.After(oldestTimestamp) { loves = append(loves, *love) + p.Export.TotalItems += 1 } } + sort.Sort(loves) - p.Export.Total = int64(len(loves)) - p.Export.Complete() - progress <- p results <- models.LovesResult{Items: loves} } func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - for _, love := range export.Items { + p := models.TransferProgress{}.FromImportResult(importResult, false) + for _, love := range models.IterImportProgress(export.Items, &p, progress) { if err := ctx.Err(); err != nil { return importResult, err } @@ -176,7 +177,6 @@ func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult importResult.UpdateTimestamp(love.Created) } - progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d331ce..6d42f3c 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -154,22 +154,23 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens := make(models.ListensList, 0, len(b.log.Records)) client := strings.Split(b.log.Client, " ")[0] - for _, record := range b.log.Records { + p.Export.Total = int64(len(b.log.Records)) + for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { listen := recordToListen(record, client) if listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, recordToListen(record, client)) + p.Export.TotalItems += 1 } } + sort.Sort(listens) - p.Export.Total = int64(len(listens)) - p.Export.Complete() - progress <- p results <- models.ListensResult{Items: listens} } func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + p := models.TransferProgress{}.FromImportResult(importResult, false) records := make([]scrobblerlog.Record, len(export.Items)) - for i, listen := range export.Items { + for i, listen := range models.IterImportProgress(export.Items, &p, progress) { records[i] = listenToRecord(listen) } lastTimestamp, err := b.log.Append(b.file, records) @@ -179,8 +180,6 @@ func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.L importResult.UpdateTimestamp(lastTimestamp) importResult.ImportCount += len(export.Items) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) - return importResult, nil } diff --git a/internal/models/models.go b/internal/models/models.go index 081266d..09b4d6b 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -22,6 +22,7 @@ THE SOFTWARE. package models import ( + "iter" "strings" "time" @@ -244,3 +245,38 @@ func (p *Progress) Complete() { func (p *Progress) Abort() { p.Aborted = true } + +func IterExportProgress[T any]( + items []T, t *TransferProgress, c chan TransferProgress, +) iter.Seq2[int, T] { + return iterProgress(items, t, t.Export, c, true) +} + +func IterImportProgress[T any]( + items []T, t *TransferProgress, c chan TransferProgress, +) iter.Seq2[int, T] { + return iterProgress(items, t, t.Import, c, false) +} + +func iterProgress[T any]( + items []T, t *TransferProgress, + p *Progress, c chan TransferProgress, + autocomplete bool, +) iter.Seq2[int, T] { + // Report progress in 1% steps + steps := len(items) / 100 + return func(yield func(int, T) bool) { + for i, item := range items { + yield(i, item) + p.Elapsed++ + if i%steps == 0 { + c <- *t + } + } + + if autocomplete { + p.Complete() + c <- *t + } + } +}