diff --git a/CHANGES.md b/CHANGES.md index 486d0ff..64f854f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Scotty Changelog -## 0.6.0 - 2025-05-23 +## 0.6.0 - WIP - Fully reworked progress report - Cancel both export and import on error - Show progress bars as aborted on export / import error @@ -8,12 +8,11 @@ - The import progress shows total items processed instead of time estimate - Fix program hanging endlessly if import fails (#11) - If import fails still store the last successfully imported timestamp - - More granular progress updates for JSPF and scrobblerlog - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID -- JSPF: add MusicBrainz playlist extension in append mode, if it does not - exist in the existing JSPF file +- JSPF: add MusicBrainz playlist extension in append mode, if it does not exist + in the existing JSPF file - scrobblerlog: fix timezone not being set from config (#6) - scrobblerlog: fix listen export not considering latest timestamp - Funkwhale: fix progress abort on error diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index e2bcde1..354640e 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -108,22 +108,21 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti } listens := make(models.ListensList, 0, len(b.playlist.Tracks)) - p.Export.Total = int64(len(b.playlist.Tracks)) - for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { + for _, track := range b.playlist.Tracks { listen, err := trackAsListen(track) if err == nil && listen != nil && listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, *listen) - p.Export.TotalItems += 1 } } - sort.Sort(listens) + p.Export.Total = int64(len(listens)) + p.Export.Complete() + progress <- p results <- models.ListensResult{Items: listens} } func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - p := models.TransferProgress{}.FromImportResult(importResult, false) - for _, listen := range models.IterImportProgress(export.Items, &p, progress) { + for _, listen := range export.Items { if err := ctx.Err(); err != nil { return importResult, err } @@ -134,6 +133,7 @@ func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensRe importResult.UpdateTimestamp(listen.ListenedAt) } + progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } @@ -151,22 +151,21 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time } loves := make(models.LovesList, 0, len(b.playlist.Tracks)) - p.Export.Total = int64(len(b.playlist.Tracks)) - for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { + for _, track := range b.playlist.Tracks { love, err := trackAsLove(track) if err == nil && love != nil && love.Created.After(oldestTimestamp) { loves = append(loves, *love) - p.Export.TotalItems += 1 } } - sort.Sort(loves) + p.Export.Total = int64(len(loves)) + p.Export.Complete() + progress <- p results <- models.LovesResult{Items: loves} } func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - p := models.TransferProgress{}.FromImportResult(importResult, false) - for _, love := range models.IterImportProgress(export.Items, &p, progress) { + for _, love := range export.Items { if err := ctx.Err(); err != nil { return importResult, err } @@ -177,6 +176,7 @@ func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult importResult.UpdateTimestamp(love.Created) } + progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d42f3c..6d331ce 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -154,23 +154,22 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens := make(models.ListensList, 0, len(b.log.Records)) client := strings.Split(b.log.Client, " ")[0] - p.Export.Total = int64(len(b.log.Records)) - for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { + for _, record := range b.log.Records { listen := recordToListen(record, client) if listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, recordToListen(record, client)) - p.Export.TotalItems += 1 } } - sort.Sort(listens) + p.Export.Total = int64(len(listens)) + p.Export.Complete() + progress <- p results <- models.ListensResult{Items: listens} } func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - p := models.TransferProgress{}.FromImportResult(importResult, false) records := make([]scrobblerlog.Record, len(export.Items)) - for i, listen := range models.IterImportProgress(export.Items, &p, progress) { + for i, listen := range export.Items { records[i] = listenToRecord(listen) } lastTimestamp, err := b.log.Append(b.file, records) @@ -180,6 +179,8 @@ func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.L importResult.UpdateTimestamp(lastTimestamp) importResult.ImportCount += len(export.Items) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) + return importResult, nil } diff --git a/internal/cli/progress.go b/internal/cli/progress.go index d17594c..db862a1 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -87,10 +87,7 @@ func (u *progressBarUpdater) update() { func (u *progressBarUpdater) updateExportProgress(progress *models.Progress) { bar := u.exportBar - if progress.TotalItems != u.totalItems { - u.totalItems = progress.TotalItems - u.importBar.SetTotal(int64(u.totalItems), false) - } + u.totalItems = progress.TotalItems if progress.Aborted { bar.Abort(false) diff --git a/internal/models/models.go b/internal/models/models.go index 78d9965..081266d 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -22,7 +22,6 @@ THE SOFTWARE. package models import ( - "iter" "strings" "time" @@ -245,40 +244,3 @@ func (p *Progress) Complete() { func (p *Progress) Abort() { p.Aborted = true } - -func IterExportProgress[T any]( - items []T, t *TransferProgress, c chan TransferProgress, -) iter.Seq2[int, T] { - return iterProgress(items, t, t.Export, c, true) -} - -func IterImportProgress[T any]( - items []T, t *TransferProgress, c chan TransferProgress, -) iter.Seq2[int, T] { - return iterProgress(items, t, t.Import, c, false) -} - -func iterProgress[T any]( - items []T, t *TransferProgress, - p *Progress, c chan TransferProgress, - autocomplete bool, -) iter.Seq2[int, T] { - // Report progress in 1% steps - steps := max(len(items)/100, 1) - return func(yield func(int, T) bool) { - for i, item := range items { - if !yield(i, item) { - return - } - p.Elapsed++ - if i%steps == 0 { - c <- *t - } - } - - if autocomplete { - p.Complete() - c <- *t - } - } -} diff --git a/internal/version/version.go b/internal/version/version.go index f3bc081..b38a40f 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -17,7 +17,7 @@ package version const ( AppName = "scotty" - AppVersion = "0.6.0" + AppVersion = "0.5.2" AppURL = "https://git.sr.ht/~phw/scotty/" ) diff --git a/pkg/scrobblerlog/parser.go b/pkg/scrobblerlog/parser.go index 48fadcf..8bad56d 100644 --- a/pkg/scrobblerlog/parser.go +++ b/pkg/scrobblerlog/parser.go @@ -39,7 +39,6 @@ import ( "encoding/csv" "fmt" "io" - "iter" "strconv" "strings" "time" @@ -92,36 +91,53 @@ type ScrobblerLog struct { // The reader must provide a valid scrobbler log file with a valid header. // This function implicitly calls [ScrobblerLog.ReadHeader]. func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { - tsvReader, err := l.initReader(data) + l.Records = make([]Record, 0) + + reader := bufio.NewReader(data) + err := l.readHeader(reader) if err != nil { return err } - for _, err := range l.iterRecords(tsvReader, ignoreSkipped) { + tsvReader := csv.NewReader(reader) + tsvReader.Comma = '\t' + // Row length is often flexible + tsvReader.FieldsPerRecord = -1 + + for { + // A row is: + // artistName releaseName trackName trackNumber duration rating timestamp recordingMBID + row, err := tsvReader.Read() + if err == io.EOF { + break + } else if err != nil { + return err + } + + // fmt.Printf("row: %v\n", row) + + // We consider only the last field (recording MBID) optional + // This was added in the 1.1 file format. + if len(row) < 7 { + line, _ := tsvReader.FieldPos(0) + return fmt.Errorf("invalid record in scrobblerlog line %v", line) + } + + record, err := l.rowToRecord(row) if err != nil { return err } + + if ignoreSkipped && record.Rating == RatingSkipped { + continue + } + + l.Records = append(l.Records, record) } return nil } -// Parses a scrobbler log file from the given reader and returns an iterator over all records. -// -// The reader must provide a valid scrobbler log file with a valid header. -// This function implicitly calls [ScrobblerLog.ReadHeader]. -func (l *ScrobblerLog) ParseIter(data io.Reader, ignoreSkipped bool) iter.Seq2[Record, error] { - - tsvReader, err := l.initReader(data) - if err != nil { - return func(yield func(Record, error) bool) { - yield(Record{}, err) - } - } - - return l.iterRecords(tsvReader, ignoreSkipped) -} - // Append writes the given records to the writer. // // The writer should be for an existing scrobbler log file or @@ -161,37 +177,6 @@ func (l *ScrobblerLog) ReadHeader(reader io.Reader) error { return l.readHeader(bufio.NewReader(reader)) } -// Writes the header of a scrobbler log file to the given writer. -func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { - headers := []string{ - "#AUDIOSCROBBLER/1.1\n", - "#TZ/" + string(l.TZ) + "\n", - "#CLIENT/" + l.Client + "\n", - } - for _, line := range headers { - _, err := writer.Write([]byte(line)) - if err != nil { - return err - } - } - return nil -} - -func (l *ScrobblerLog) initReader(data io.Reader) (*csv.Reader, error) { - reader := bufio.NewReader(data) - err := l.readHeader(reader) - if err != nil { - return nil, err - } - - tsvReader := csv.NewReader(reader) - tsvReader.Comma = '\t' - // Row length is often flexible - tsvReader.FieldsPerRecord = -1 - - return tsvReader, nil -} - func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { // Skip header for i := 0; i < 3; i++ { @@ -230,64 +215,37 @@ func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { return nil } -func (l *ScrobblerLog) iterRecords(reader *csv.Reader, ignoreSkipped bool) iter.Seq2[Record, error] { - return func(yield func(Record, error) bool) { - l.Records = make([]Record, 0) - for { - record, err := l.parseRow(reader) - if err == io.EOF { - break - } else if err != nil { - yield(Record{}, err) - break - } - - if ignoreSkipped && record.Rating == RatingSkipped { - continue - } - - l.Records = append(l.Records, *record) - if !yield(*record, nil) { - break - } +// Writes the header of a scrobbler log file to the given writer. +func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { + headers := []string{ + "#AUDIOSCROBBLER/1.1\n", + "#TZ/" + string(l.TZ) + "\n", + "#CLIENT/" + l.Client + "\n", + } + for _, line := range headers { + _, err := writer.Write([]byte(line)) + if err != nil { + return err } } + return nil } -func (l *ScrobblerLog) parseRow(reader *csv.Reader) (*Record, error) { - // A row is: - // artistName releaseName trackName trackNumber duration rating timestamp recordingMBID - row, err := reader.Read() - if err != nil { - return nil, err - } - - // fmt.Printf("row: %v\n", row) - - // We consider only the last field (recording MBID) optional - // This was added in the 1.1 file format. - if len(row) < 7 { - line, _ := reader.FieldPos(0) - return nil, fmt.Errorf("invalid record in scrobblerlog line %v", line) - } - - return l.rowToRecord(row) -} - -func (l ScrobblerLog) rowToRecord(row []string) (*Record, error) { +func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { + var record Record trackNumber, err := strconv.Atoi(row[3]) if err != nil { - return nil, err + return record, err } duration, err := strconv.Atoi(row[4]) if err != nil { - return nil, err + return record, err } timestamp, err := strconv.ParseInt(row[6], 10, 64) if err != nil { - return nil, err + return record, err } var timezone *time.Location = nil @@ -295,7 +253,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (*Record, error) { timezone = l.FallbackTimezone } - record := Record{ + record = Record{ ArtistName: row[0], AlbumName: row[1], TrackName: row[2], @@ -309,7 +267,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (*Record, error) { record.MusicBrainzRecordingID = mbtypes.MBID(row[7]) } - return &record, nil + return record, nil } // Convert a Unix timestamp to a [time.Time] object, but treat the timestamp diff --git a/pkg/scrobblerlog/parser_test.go b/pkg/scrobblerlog/parser_test.go index 26990f9..8dc30e5 100644 --- a/pkg/scrobblerlog/parser_test.go +++ b/pkg/scrobblerlog/parser_test.go @@ -44,14 +44,7 @@ Kraftwerk Trans-Europe Express The Hall of Mirrors 2 474 S 1260358000 385ba9e9-6 Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262beaf-19f8-4534-b9ed-7eef9ca8e83f ` -var testScrobblerLogInvalid = `#AUDIOSCROBBLER/1.1 -#TZ/UNKNOWN -#CLIENT/Rockbox sansaclipplus $Revision$ -Özcan Deniz Ses ve Ayrilik Sevdanin rengi (sipacik) byMrTurkey 5 306 L 1260342084 -Özcan Deniz Hediye 2@V@7 Bir Dudaktan 1 210 L -` - -func TestParse(t *testing.T) { +func TestParser(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{} @@ -75,7 +68,7 @@ func TestParse(t *testing.T) { record4.MusicBrainzRecordingID) } -func TestParseIgnoreSkipped(t *testing.T) { +func TestParserIgnoreSkipped(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{} @@ -88,7 +81,7 @@ func TestParseIgnoreSkipped(t *testing.T) { record4.MusicBrainzRecordingID) } -func TestParseFallbackTimezone(t *testing.T) { +func TestParserFallbackTimezone(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{ @@ -103,29 +96,6 @@ func TestParseFallbackTimezone(t *testing.T) { ) } -func TestParseInvalid(t *testing.T) { - assert := assert.New(t) - data := bytes.NewBufferString(testScrobblerLogInvalid) - result := scrobblerlog.ScrobblerLog{} - err := result.Parse(data, true) - assert.ErrorContains(err, "invalid record in scrobblerlog line 2") -} - -func TestParseIter(t *testing.T) { - assert := assert.New(t) - data := bytes.NewBufferString(testScrobblerLog) - result := scrobblerlog.ScrobblerLog{} - records := make([]scrobblerlog.Record, 0) - for record, err := range result.ParseIter(data, false) { - require.NoError(t, err) - records = append(records, record) - } - - assert.Len(records, 5) - record1 := result.Records[0] - assert.Equal("Ses ve Ayrilik", record1.AlbumName) -} - func TestAppend(t *testing.T) { assert := assert.New(t) data := make([]byte, 0, 10)