diff --git a/CHANGES.md b/CHANGES.md index 64f854f..486d0ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Scotty Changelog -## 0.6.0 - WIP +## 0.6.0 - 2025-05-23 - Fully reworked progress report - Cancel both export and import on error - Show progress bars as aborted on export / import error @@ -8,11 +8,12 @@ - The import progress shows total items processed instead of time estimate - Fix program hanging endlessly if import fails (#11) - If import fails still store the last successfully imported timestamp + - More granular progress updates for JSPF and scrobblerlog - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID -- JSPF: add MusicBrainz playlist extension in append mode, if it does not exist - in the existing JSPF file +- JSPF: add MusicBrainz playlist extension in append mode, if it does not + exist in the existing JSPF file - scrobblerlog: fix timezone not being set from config (#6) - scrobblerlog: fix listen export not considering latest timestamp - Funkwhale: fix progress abort on error diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 354640e..e2bcde1 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -108,21 +108,22 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti } listens := make(models.ListensList, 0, len(b.playlist.Tracks)) - for _, track := range b.playlist.Tracks { + p.Export.Total = int64(len(b.playlist.Tracks)) + for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { listen, err := trackAsListen(track) if err == nil && listen != nil && listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, *listen) + p.Export.TotalItems += 1 } } + sort.Sort(listens) - p.Export.Total = int64(len(listens)) - p.Export.Complete() - progress <- p results <- models.ListensResult{Items: listens} } func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - for _, listen := range export.Items { + p := models.TransferProgress{}.FromImportResult(importResult, false) + for _, listen := range models.IterImportProgress(export.Items, &p, progress) { if err := ctx.Err(); err != nil { return importResult, err } @@ -133,7 +134,6 @@ func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensRe importResult.UpdateTimestamp(listen.ListenedAt) } - progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } @@ -151,21 +151,22 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time } loves := make(models.LovesList, 0, len(b.playlist.Tracks)) - for _, track := range b.playlist.Tracks { + p.Export.Total = int64(len(b.playlist.Tracks)) + for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { love, err := trackAsLove(track) if err == nil && love != nil && love.Created.After(oldestTimestamp) { loves = append(loves, *love) + p.Export.TotalItems += 1 } } + sort.Sort(loves) - p.Export.Total = int64(len(loves)) - p.Export.Complete() - progress <- p results <- models.LovesResult{Items: loves} } func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - for _, love := range export.Items { + p := models.TransferProgress{}.FromImportResult(importResult, false) + for _, love := range models.IterImportProgress(export.Items, &p, progress) { if err := ctx.Err(); err != nil { return importResult, err } @@ -176,7 +177,6 @@ func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult importResult.UpdateTimestamp(love.Created) } - progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d331ce..6d42f3c 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -154,22 +154,23 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens := make(models.ListensList, 0, len(b.log.Records)) client := strings.Split(b.log.Client, " ")[0] - for _, record := range b.log.Records { + p.Export.Total = int64(len(b.log.Records)) + for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { listen := recordToListen(record, client) if listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, recordToListen(record, client)) + p.Export.TotalItems += 1 } } + sort.Sort(listens) - p.Export.Total = int64(len(listens)) - p.Export.Complete() - progress <- p results <- models.ListensResult{Items: listens} } func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + p := models.TransferProgress{}.FromImportResult(importResult, false) records := make([]scrobblerlog.Record, len(export.Items)) - for i, listen := range export.Items { + for i, listen := range models.IterImportProgress(export.Items, &p, progress) { records[i] = listenToRecord(listen) } lastTimestamp, err := b.log.Append(b.file, records) @@ -179,8 +180,6 @@ func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.L importResult.UpdateTimestamp(lastTimestamp) importResult.ImportCount += len(export.Items) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) - return importResult, nil } diff --git a/internal/cli/progress.go b/internal/cli/progress.go index db862a1..d17594c 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -87,7 +87,10 @@ func (u *progressBarUpdater) update() { func (u *progressBarUpdater) updateExportProgress(progress *models.Progress) { bar := u.exportBar - u.totalItems = progress.TotalItems + if progress.TotalItems != u.totalItems { + u.totalItems = progress.TotalItems + u.importBar.SetTotal(int64(u.totalItems), false) + } if progress.Aborted { bar.Abort(false) diff --git a/internal/models/models.go b/internal/models/models.go index 081266d..78d9965 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -22,6 +22,7 @@ THE SOFTWARE. package models import ( + "iter" "strings" "time" @@ -244,3 +245,40 @@ func (p *Progress) Complete() { func (p *Progress) Abort() { p.Aborted = true } + +func IterExportProgress[T any]( + items []T, t *TransferProgress, c chan TransferProgress, +) iter.Seq2[int, T] { + return iterProgress(items, t, t.Export, c, true) +} + +func IterImportProgress[T any]( + items []T, t *TransferProgress, c chan TransferProgress, +) iter.Seq2[int, T] { + return iterProgress(items, t, t.Import, c, false) +} + +func iterProgress[T any]( + items []T, t *TransferProgress, + p *Progress, c chan TransferProgress, + autocomplete bool, +) iter.Seq2[int, T] { + // Report progress in 1% steps + steps := max(len(items)/100, 1) + return func(yield func(int, T) bool) { + for i, item := range items { + if !yield(i, item) { + return + } + p.Elapsed++ + if i%steps == 0 { + c <- *t + } + } + + if autocomplete { + p.Complete() + c <- *t + } + } +} diff --git a/internal/version/version.go b/internal/version/version.go index b38a40f..f3bc081 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -17,7 +17,7 @@ package version const ( AppName = "scotty" - AppVersion = "0.5.2" + AppVersion = "0.6.0" AppURL = "https://git.sr.ht/~phw/scotty/" ) diff --git a/pkg/scrobblerlog/parser.go b/pkg/scrobblerlog/parser.go index 8bad56d..48fadcf 100644 --- a/pkg/scrobblerlog/parser.go +++ b/pkg/scrobblerlog/parser.go @@ -39,6 +39,7 @@ import ( "encoding/csv" "fmt" "io" + "iter" "strconv" "strings" "time" @@ -91,53 +92,36 @@ type ScrobblerLog struct { // The reader must provide a valid scrobbler log file with a valid header. // This function implicitly calls [ScrobblerLog.ReadHeader]. func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { - l.Records = make([]Record, 0) - - reader := bufio.NewReader(data) - err := l.readHeader(reader) + tsvReader, err := l.initReader(data) if err != nil { return err } - tsvReader := csv.NewReader(reader) - tsvReader.Comma = '\t' - // Row length is often flexible - tsvReader.FieldsPerRecord = -1 - - for { - // A row is: - // artistName releaseName trackName trackNumber duration rating timestamp recordingMBID - row, err := tsvReader.Read() - if err == io.EOF { - break - } else if err != nil { - return err - } - - // fmt.Printf("row: %v\n", row) - - // We consider only the last field (recording MBID) optional - // This was added in the 1.1 file format. - if len(row) < 7 { - line, _ := tsvReader.FieldPos(0) - return fmt.Errorf("invalid record in scrobblerlog line %v", line) - } - - record, err := l.rowToRecord(row) + for _, err := range l.iterRecords(tsvReader, ignoreSkipped) { if err != nil { return err } - - if ignoreSkipped && record.Rating == RatingSkipped { - continue - } - - l.Records = append(l.Records, record) } return nil } +// Parses a scrobbler log file from the given reader and returns an iterator over all records. +// +// The reader must provide a valid scrobbler log file with a valid header. +// This function implicitly calls [ScrobblerLog.ReadHeader]. +func (l *ScrobblerLog) ParseIter(data io.Reader, ignoreSkipped bool) iter.Seq2[Record, error] { + + tsvReader, err := l.initReader(data) + if err != nil { + return func(yield func(Record, error) bool) { + yield(Record{}, err) + } + } + + return l.iterRecords(tsvReader, ignoreSkipped) +} + // Append writes the given records to the writer. // // The writer should be for an existing scrobbler log file or @@ -177,6 +161,37 @@ func (l *ScrobblerLog) ReadHeader(reader io.Reader) error { return l.readHeader(bufio.NewReader(reader)) } +// Writes the header of a scrobbler log file to the given writer. +func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { + headers := []string{ + "#AUDIOSCROBBLER/1.1\n", + "#TZ/" + string(l.TZ) + "\n", + "#CLIENT/" + l.Client + "\n", + } + for _, line := range headers { + _, err := writer.Write([]byte(line)) + if err != nil { + return err + } + } + return nil +} + +func (l *ScrobblerLog) initReader(data io.Reader) (*csv.Reader, error) { + reader := bufio.NewReader(data) + err := l.readHeader(reader) + if err != nil { + return nil, err + } + + tsvReader := csv.NewReader(reader) + tsvReader.Comma = '\t' + // Row length is often flexible + tsvReader.FieldsPerRecord = -1 + + return tsvReader, nil +} + func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { // Skip header for i := 0; i < 3; i++ { @@ -215,37 +230,64 @@ func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { return nil } -// Writes the header of a scrobbler log file to the given writer. -func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { - headers := []string{ - "#AUDIOSCROBBLER/1.1\n", - "#TZ/" + string(l.TZ) + "\n", - "#CLIENT/" + l.Client + "\n", - } - for _, line := range headers { - _, err := writer.Write([]byte(line)) - if err != nil { - return err +func (l *ScrobblerLog) iterRecords(reader *csv.Reader, ignoreSkipped bool) iter.Seq2[Record, error] { + return func(yield func(Record, error) bool) { + l.Records = make([]Record, 0) + for { + record, err := l.parseRow(reader) + if err == io.EOF { + break + } else if err != nil { + yield(Record{}, err) + break + } + + if ignoreSkipped && record.Rating == RatingSkipped { + continue + } + + l.Records = append(l.Records, *record) + if !yield(*record, nil) { + break + } } } - return nil } -func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { - var record Record +func (l *ScrobblerLog) parseRow(reader *csv.Reader) (*Record, error) { + // A row is: + // artistName releaseName trackName trackNumber duration rating timestamp recordingMBID + row, err := reader.Read() + if err != nil { + return nil, err + } + + // fmt.Printf("row: %v\n", row) + + // We consider only the last field (recording MBID) optional + // This was added in the 1.1 file format. + if len(row) < 7 { + line, _ := reader.FieldPos(0) + return nil, fmt.Errorf("invalid record in scrobblerlog line %v", line) + } + + return l.rowToRecord(row) +} + +func (l ScrobblerLog) rowToRecord(row []string) (*Record, error) { trackNumber, err := strconv.Atoi(row[3]) if err != nil { - return record, err + return nil, err } duration, err := strconv.Atoi(row[4]) if err != nil { - return record, err + return nil, err } timestamp, err := strconv.ParseInt(row[6], 10, 64) if err != nil { - return record, err + return nil, err } var timezone *time.Location = nil @@ -253,7 +295,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { timezone = l.FallbackTimezone } - record = Record{ + record := Record{ ArtistName: row[0], AlbumName: row[1], TrackName: row[2], @@ -267,7 +309,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { record.MusicBrainzRecordingID = mbtypes.MBID(row[7]) } - return record, nil + return &record, nil } // Convert a Unix timestamp to a [time.Time] object, but treat the timestamp diff --git a/pkg/scrobblerlog/parser_test.go b/pkg/scrobblerlog/parser_test.go index 8dc30e5..26990f9 100644 --- a/pkg/scrobblerlog/parser_test.go +++ b/pkg/scrobblerlog/parser_test.go @@ -44,7 +44,14 @@ Kraftwerk Trans-Europe Express The Hall of Mirrors 2 474 S 1260358000 385ba9e9-6 Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262beaf-19f8-4534-b9ed-7eef9ca8e83f ` -func TestParser(t *testing.T) { +var testScrobblerLogInvalid = `#AUDIOSCROBBLER/1.1 +#TZ/UNKNOWN +#CLIENT/Rockbox sansaclipplus $Revision$ +Özcan Deniz Ses ve Ayrilik Sevdanin rengi (sipacik) byMrTurkey 5 306 L 1260342084 +Özcan Deniz Hediye 2@V@7 Bir Dudaktan 1 210 L +` + +func TestParse(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{} @@ -68,7 +75,7 @@ func TestParser(t *testing.T) { record4.MusicBrainzRecordingID) } -func TestParserIgnoreSkipped(t *testing.T) { +func TestParseIgnoreSkipped(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{} @@ -81,7 +88,7 @@ func TestParserIgnoreSkipped(t *testing.T) { record4.MusicBrainzRecordingID) } -func TestParserFallbackTimezone(t *testing.T) { +func TestParseFallbackTimezone(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{ @@ -96,6 +103,29 @@ func TestParserFallbackTimezone(t *testing.T) { ) } +func TestParseInvalid(t *testing.T) { + assert := assert.New(t) + data := bytes.NewBufferString(testScrobblerLogInvalid) + result := scrobblerlog.ScrobblerLog{} + err := result.Parse(data, true) + assert.ErrorContains(err, "invalid record in scrobblerlog line 2") +} + +func TestParseIter(t *testing.T) { + assert := assert.New(t) + data := bytes.NewBufferString(testScrobblerLog) + result := scrobblerlog.ScrobblerLog{} + records := make([]scrobblerlog.Record, 0) + for record, err := range result.ParseIter(data, false) { + require.NoError(t, err) + records = append(records, record) + } + + assert.Len(records, 5) + record1 := result.Records[0] + assert.Equal("Ses ve Ayrilik", record1.AlbumName) +} + func TestAppend(t *testing.T) { assert := assert.New(t) data := make([]byte, 0, 10)