Release 0.6.0

This reverts commit a8ce2be5d7.

CHANGES.md

@@ -1,6 +1,6 @@
 # Scotty Changelog
 
-## 0.6.0 - WIP
+## 0.6.0 - 2025-05-23
 - Fully reworked progress report
   - Cancel both export and import on error
   - Show progress bars as aborted on export / import error
@@ -8,11 +8,12 @@
   - The import progress shows total items processed instead of time estimate
   - Fix program hanging endlessly if import fails (#11)
   - If import fails still store the last successfully imported timestamp
+  - More granular progress updates for JSPF and scrobblerlog
 - JSPF: implemented export as loves and listens
 - JSPF: write track duration
 - JSPF: read username and recording MSID
-- JSPF: add MusicBrainz playlist extension in append mode, if it does not exist
-  in the existing JSPF file
+- JSPF: add MusicBrainz playlist extension in append mode, if it does not
+  exist in the existing JSPF file
 - scrobblerlog: fix timezone not being set from config (#6)
 - scrobblerlog: fix listen export not considering latest timestamp
 - Funkwhale: fix progress abort on error

internal/backends/jspf/jspf.go

@@ -108,21 +108,22 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti
 	}
 
 	listens := make(models.ListensList, 0, len(b.playlist.Tracks))
-	for _, track := range b.playlist.Tracks {
+	p.Export.Total = int64(len(b.playlist.Tracks))
+	for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) {
 		listen, err := trackAsListen(track)
 		if err == nil && listen != nil && listen.ListenedAt.After(oldestTimestamp) {
 			listens = append(listens, *listen)
+			p.Export.TotalItems += 1
 		}
 	}
+
 	sort.Sort(listens)
-	p.Export.Total = int64(len(listens))
-	p.Export.Complete()
-	progress <- p
 	results <- models.ListensResult{Items: listens}
 }
 
 func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
-	for _, listen := range export.Items {
+	p := models.TransferProgress{}.FromImportResult(importResult, false)
+	for _, listen := range models.IterImportProgress(export.Items, &p, progress) {
 		if err := ctx.Err(); err != nil {
 			return importResult, err
 		}
@@ -133,7 +134,6 @@ func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensRe
 		importResult.UpdateTimestamp(listen.ListenedAt)
 	}
 
-	progress <- models.TransferProgress{}.FromImportResult(importResult, false)
 	return importResult, nil
 }
 
@@ -151,21 +151,22 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time
 	}
 
 	loves := make(models.LovesList, 0, len(b.playlist.Tracks))
-	for _, track := range b.playlist.Tracks {
+	p.Export.Total = int64(len(b.playlist.Tracks))
+	for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) {
 		love, err := trackAsLove(track)
 		if err == nil && love != nil && love.Created.After(oldestTimestamp) {
 			loves = append(loves, *love)
+			p.Export.TotalItems += 1
 		}
 	}
+
 	sort.Sort(loves)
-	p.Export.Total = int64(len(loves))
-	p.Export.Complete()
-	progress <- p
 	results <- models.LovesResult{Items: loves}
 }
 
 func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
-	for _, love := range export.Items {
+	p := models.TransferProgress{}.FromImportResult(importResult, false)
+	for _, love := range models.IterImportProgress(export.Items, &p, progress) {
 		if err := ctx.Err(); err != nil {
 			return importResult, err
 		}
@@ -176,7 +177,6 @@ func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult
 		importResult.UpdateTimestamp(love.Created)
 	}
 
-	progress <- models.TransferProgress{}.FromImportResult(importResult, false)
 	return importResult, nil
 }
 

internal/backends/scrobblerlog/scrobblerlog.go

@@ -154,22 +154,23 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp
 
 	listens := make(models.ListensList, 0, len(b.log.Records))
 	client := strings.Split(b.log.Client, " ")[0]
-	for _, record := range b.log.Records {
+	p.Export.Total = int64(len(b.log.Records))
+	for _, record := range models.IterExportProgress(b.log.Records, &p, progress) {
 		listen := recordToListen(record, client)
 		if listen.ListenedAt.After(oldestTimestamp) {
 			listens = append(listens, recordToListen(record, client))
+			p.Export.TotalItems += 1
 		}
 	}
+
 	sort.Sort(listens)
-	p.Export.Total = int64(len(listens))
-	p.Export.Complete()
-	progress <- p
 	results <- models.ListensResult{Items: listens}
 }
 
 func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) {
+	p := models.TransferProgress{}.FromImportResult(importResult, false)
 	records := make([]scrobblerlog.Record, len(export.Items))
-	for i, listen := range export.Items {
+	for i, listen := range models.IterImportProgress(export.Items, &p, progress) {
 		records[i] = listenToRecord(listen)
 	}
 	lastTimestamp, err := b.log.Append(b.file, records)
@@ -179,8 +180,6 @@ func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.L
 
 	importResult.UpdateTimestamp(lastTimestamp)
 	importResult.ImportCount += len(export.Items)
-	progress <- models.TransferProgress{}.FromImportResult(importResult, false)
-
 	return importResult, nil
 }
 

internal/cli/progress.go

@@ -87,7 +87,10 @@ func (u *progressBarUpdater) update() {
 
 func (u *progressBarUpdater) updateExportProgress(progress *models.Progress) {
 	bar := u.exportBar
-	u.totalItems = progress.TotalItems
+	if progress.TotalItems != u.totalItems {
+		u.totalItems = progress.TotalItems
+		u.importBar.SetTotal(int64(u.totalItems), false)
+	}
 
 	if progress.Aborted {
 		bar.Abort(false)

internal/models/models.go

@@ -22,6 +22,7 @@ THE SOFTWARE.
 package models
 
 import (
+	"iter"
 	"strings"
 	"time"
 
@@ -244,3 +245,40 @@ func (p *Progress) Complete() {
 func (p *Progress) Abort() {
 	p.Aborted = true
 }
+
+func IterExportProgress[T any](
+	items []T, t *TransferProgress, c chan TransferProgress,
+) iter.Seq2[int, T] {
+	return iterProgress(items, t, t.Export, c, true)
+}
+
+func IterImportProgress[T any](
+	items []T, t *TransferProgress, c chan TransferProgress,
+) iter.Seq2[int, T] {
+	return iterProgress(items, t, t.Import, c, false)
+}
+
+func iterProgress[T any](
+	items []T, t *TransferProgress,
+	p *Progress, c chan TransferProgress,
+	autocomplete bool,
+) iter.Seq2[int, T] {
+	// Report progress in 1% steps
+	steps := max(len(items)/100, 1)
+	return func(yield func(int, T) bool) {
+		for i, item := range items {
+			if !yield(i, item) {
+				return
+			}
+			p.Elapsed++
+			if i%steps == 0 {
+				c <- *t
+			}
+		}
+
+		if autocomplete {
+			p.Complete()
+			c <- *t
+		}
+	}
+}

internal/version/version.go

@@ -17,7 +17,7 @@ package version
 
 const (
 	AppName    = "scotty"
-	AppVersion = "0.5.2"
+	AppVersion = "0.6.0"
 	AppURL     = "https://git.sr.ht/~phw/scotty/"
 )
 

pkg/scrobblerlog/parser.go

@@ -39,6 +39,7 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
+	"iter"
 	"strconv"
 	"strings"
 	"time"
@@ -91,53 +92,36 @@ type ScrobblerLog struct {
 // The reader must provide a valid scrobbler log file with a valid header.
 // This function implicitly calls [ScrobblerLog.ReadHeader].
 func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error {
-	l.Records = make([]Record, 0)
-
-	reader := bufio.NewReader(data)
-	err := l.readHeader(reader)
+	tsvReader, err := l.initReader(data)
 	if err != nil {
 		return err
 	}
 
-	tsvReader := csv.NewReader(reader)
-	tsvReader.Comma = '\t'
-	// Row length is often flexible
-	tsvReader.FieldsPerRecord = -1
-
-	for {
-		// A row is:
-		// artistName	releaseName	trackName	trackNumber	duration	rating	timestamp	recordingMBID
-		row, err := tsvReader.Read()
-		if err == io.EOF {
-			break
-		} else if err != nil {
-			return err
-		}
-
-		// fmt.Printf("row: %v\n", row)
-
-		// We consider only the last field (recording MBID) optional
-		// This was added in the 1.1 file format.
-		if len(row) < 7 {
-			line, _ := tsvReader.FieldPos(0)
-			return fmt.Errorf("invalid record in scrobblerlog line %v", line)
-		}
-
-		record, err := l.rowToRecord(row)
+	for _, err := range l.iterRecords(tsvReader, ignoreSkipped) {
 		if err != nil {
 			return err
 		}
-
-		if ignoreSkipped && record.Rating == RatingSkipped {
-			continue
-		}
-
-		l.Records = append(l.Records, record)
 	}
 
 	return nil
 }
 
+// Parses a scrobbler log file from the given reader and returns an iterator over all records.
+//
+// The reader must provide a valid scrobbler log file with a valid header.
+// This function implicitly calls [ScrobblerLog.ReadHeader].
+func (l *ScrobblerLog) ParseIter(data io.Reader, ignoreSkipped bool) iter.Seq2[Record, error] {
+
+	tsvReader, err := l.initReader(data)
+	if err != nil {
+		return func(yield func(Record, error) bool) {
+			yield(Record{}, err)
+		}
+	}
+
+	return l.iterRecords(tsvReader, ignoreSkipped)
+}
+
 // Append writes the given records to the writer.
 //
 // The writer should be for an existing scrobbler log file or
@@ -177,6 +161,37 @@ func (l *ScrobblerLog) ReadHeader(reader io.Reader) error {
 	return l.readHeader(bufio.NewReader(reader))
 }
 
+// Writes the header of a scrobbler log file to the given writer.
+func (l *ScrobblerLog) WriteHeader(writer io.Writer) error {
+	headers := []string{
+		"#AUDIOSCROBBLER/1.1\n",
+		"#TZ/" + string(l.TZ) + "\n",
+		"#CLIENT/" + l.Client + "\n",
+	}
+	for _, line := range headers {
+		_, err := writer.Write([]byte(line))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (l *ScrobblerLog) initReader(data io.Reader) (*csv.Reader, error) {
+	reader := bufio.NewReader(data)
+	err := l.readHeader(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	tsvReader := csv.NewReader(reader)
+	tsvReader.Comma = '\t'
+	// Row length is often flexible
+	tsvReader.FieldsPerRecord = -1
+
+	return tsvReader, nil
+}
+
 func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error {
 	// Skip header
 	for i := 0; i < 3; i++ {
@@ -215,37 +230,64 @@ func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error {
 	return nil
 }
 
-// Writes the header of a scrobbler log file to the given writer.
-func (l *ScrobblerLog) WriteHeader(writer io.Writer) error {
-	headers := []string{
-		"#AUDIOSCROBBLER/1.1\n",
-		"#TZ/" + string(l.TZ) + "\n",
-		"#CLIENT/" + l.Client + "\n",
-	}
-	for _, line := range headers {
-		_, err := writer.Write([]byte(line))
-		if err != nil {
-			return err
+func (l *ScrobblerLog) iterRecords(reader *csv.Reader, ignoreSkipped bool) iter.Seq2[Record, error] {
+	return func(yield func(Record, error) bool) {
+		l.Records = make([]Record, 0)
+		for {
+			record, err := l.parseRow(reader)
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				yield(Record{}, err)
+				break
+			}
+
+			if ignoreSkipped && record.Rating == RatingSkipped {
+				continue
+			}
+
+			l.Records = append(l.Records, *record)
+			if !yield(*record, nil) {
+				break
+			}
 		}
 	}
-	return nil
 }
 
-func (l ScrobblerLog) rowToRecord(row []string) (Record, error) {
-	var record Record
+func (l *ScrobblerLog) parseRow(reader *csv.Reader) (*Record, error) {
+	// A row is:
+	// artistName	releaseName	trackName	trackNumber	duration	rating	timestamp	recordingMBID
+	row, err := reader.Read()
+	if err != nil {
+		return nil, err
+	}
+
+	// fmt.Printf("row: %v\n", row)
+
+	// We consider only the last field (recording MBID) optional
+	// This was added in the 1.1 file format.
+	if len(row) < 7 {
+		line, _ := reader.FieldPos(0)
+		return nil, fmt.Errorf("invalid record in scrobblerlog line %v", line)
+	}
+
+	return l.rowToRecord(row)
+}
+
+func (l ScrobblerLog) rowToRecord(row []string) (*Record, error) {
 	trackNumber, err := strconv.Atoi(row[3])
 	if err != nil {
-		return record, err
+		return nil, err
 	}
 
 	duration, err := strconv.Atoi(row[4])
 	if err != nil {
-		return record, err
+		return nil, err
 	}
 
 	timestamp, err := strconv.ParseInt(row[6], 10, 64)
 	if err != nil {
-		return record, err
+		return nil, err
 	}
 
 	var timezone *time.Location = nil
@@ -253,7 +295,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (Record, error) {
 		timezone = l.FallbackTimezone
 	}
 
-	record = Record{
+	record := Record{
 		ArtistName:  row[0],
 		AlbumName:   row[1],
 		TrackName:   row[2],
@@ -267,7 +309,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (Record, error) {
 		record.MusicBrainzRecordingID = mbtypes.MBID(row[7])
 	}
 
-	return record, nil
+	return &record, nil
 }
 
 // Convert a Unix timestamp to a [time.Time] object, but treat the timestamp

pkg/scrobblerlog/parser_test.go

@@ -44,7 +44,14 @@ Kraftwerk	Trans-Europe Express	The Hall of Mirrors	2	474	S	1260358000	385ba9e9-6
 Teeth Agency	You Don't Have To Live In Pain	Wolfs Jam	2	107	L	1260359404	1262beaf-19f8-4534-b9ed-7eef9ca8e83f
 `
 
-func TestParser(t *testing.T) {
+var testScrobblerLogInvalid = `#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox sansaclipplus $Revision$
+Özcan Deniz	Ses ve Ayrilik	Sevdanin rengi (sipacik) byMrTurkey	5	306	L	1260342084
+Özcan Deniz	Hediye 2@V@7	Bir Dudaktan	1	210	L
+`
+
+func TestParse(t *testing.T) {
 	assert := assert.New(t)
 	data := bytes.NewBufferString(testScrobblerLog)
 	result := scrobblerlog.ScrobblerLog{}
@@ -68,7 +75,7 @@ func TestParser(t *testing.T) {
 		record4.MusicBrainzRecordingID)
 }
 
-func TestParserIgnoreSkipped(t *testing.T) {
+func TestParseIgnoreSkipped(t *testing.T) {
 	assert := assert.New(t)
 	data := bytes.NewBufferString(testScrobblerLog)
 	result := scrobblerlog.ScrobblerLog{}
@@ -81,7 +88,7 @@ func TestParserIgnoreSkipped(t *testing.T) {
 		record4.MusicBrainzRecordingID)
 }
 
-func TestParserFallbackTimezone(t *testing.T) {
+func TestParseFallbackTimezone(t *testing.T) {
 	assert := assert.New(t)
 	data := bytes.NewBufferString(testScrobblerLog)
 	result := scrobblerlog.ScrobblerLog{
@@ -96,6 +103,29 @@ func TestParserFallbackTimezone(t *testing.T) {
 	)
 }
 
+func TestParseInvalid(t *testing.T) {
+	assert := assert.New(t)
+	data := bytes.NewBufferString(testScrobblerLogInvalid)
+	result := scrobblerlog.ScrobblerLog{}
+	err := result.Parse(data, true)
+	assert.ErrorContains(err, "invalid record in scrobblerlog line 2")
+}
+
+func TestParseIter(t *testing.T) {
+	assert := assert.New(t)
+	data := bytes.NewBufferString(testScrobblerLog)
+	result := scrobblerlog.ScrobblerLog{}
+	records := make([]scrobblerlog.Record, 0)
+	for record, err := range result.ParseIter(data, false) {
+		require.NoError(t, err)
+		records = append(records, record)
+	}
+
+	assert.Len(records, 5)
+	record1 := result.Records[0]
+	assert.Equal("Ses ve Ayrilik", record1.AlbumName)
+}
+
 func TestAppend(t *testing.T) {
 	assert := assert.New(t)
 	data := make([]byte, 0, 10)