From 588a6cf96f41a31e427396eb885e80b20bfd0e65 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 1 May 2025 13:22:20 +0200 Subject: [PATCH 01/77] Document the scrobblerlog package --- pkg/scrobblerlog/parser.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/pkg/scrobblerlog/parser.go b/pkg/scrobblerlog/parser.go index d355c62..6b9d1ba 100644 --- a/pkg/scrobblerlog/parser.go +++ b/pkg/scrobblerlog/parser.go @@ -20,11 +20,18 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ -// Package to parse and writer .scrobbler.log files as written by Rockbox. +// Package to parse and write .scrobbler.log files as written by Rockbox. +// +// The parser supports reading version 1.1 and 1.0 of the scrobbler log file +// format. The latter is only supported if encoded in UTF-8. +// +// When written it always writes version 1.1 of the scrobbler log file format, +// which includes the MusicBrainz recording ID as the last field of each row. // // See // - https://www.rockbox.org/wiki/LastFMLog // - https://git.rockbox.org/cgit/rockbox.git/tree/apps/plugins/lastfm_scrobbler.c +// - https://web.archive.org/web/20110110053056/http://www.audioscrobbler.net/wiki/Portable_Player_Logging package scrobblerlog import ( @@ -79,6 +86,10 @@ type ScrobblerLog struct { FallbackTimezone *time.Location } +// Parses a scrobbler log file from the given reader. +// +// The reader must provide a valid scrobbler log file with a valid header. +// This function implicitly calls [ScrobblerLog.ReadHeader]. func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { l.Records = make([]Record, 0) @@ -106,6 +117,7 @@ func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { // fmt.Printf("row: %v\n", row) // We consider only the last field (recording MBID) optional + // This was added in the 1.1 file format. if len(row) < 7 { line, _ := tsvReader.FieldPos(0) return fmt.Errorf("invalid record in scrobblerlog line %v", line) @@ -126,6 +138,11 @@ func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { return nil } +// Append writes the given records to the writer. +// +// The writer should be for an existing scrobbler log file or +// [ScrobblerLog.WriteHeader] should be called before this function. +// Returns the last timestamp of the records written. func (l *ScrobblerLog) Append(data io.Writer, records []Record) (lastTimestamp time.Time, err error) { tsvWriter := csv.NewWriter(data) tsvWriter.Comma = '\t' @@ -153,6 +170,9 @@ func (l *ScrobblerLog) Append(data io.Writer, records []Record) (lastTimestamp t return } +// Parses just the header of a scrobbler log file from the given reader. +// +// This function sets [ScrobblerLog.TZ] and [ScrobblerLog.Client]. func (l *ScrobblerLog) ReadHeader(reader *bufio.Reader) error { // Skip header for i := 0; i < 3; i++ { @@ -191,6 +211,7 @@ func (l *ScrobblerLog) ReadHeader(reader *bufio.Reader) error { return nil } +// Writes the header of a scrobbler log file to the given writer. func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { headers := []string{ "#AUDIOSCROBBLER/1.1\n", From 443734e4c76c396a7c696aae6c4880344572640e Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 1 May 2025 13:48:21 +0200 Subject: [PATCH 02/77] jspf: write duration to exported JSPF --- internal/backends/jspf/jspf.go | 1 + pkg/jspf/models.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 3e6866d..f98349a 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -150,6 +150,7 @@ func trackAsTrack(t models.Track) jspf.Track { Album: t.ReleaseName, Creator: t.ArtistName(), TrackNum: t.TrackNumber, + Duration: t.Duration.Milliseconds(), Extension: map[string]any{}, } diff --git a/pkg/jspf/models.go b/pkg/jspf/models.go index d910367..d7a540c 100644 --- a/pkg/jspf/models.go +++ b/pkg/jspf/models.go @@ -57,7 +57,7 @@ type Track struct { Info string `json:"info,omitempty"` Album string `json:"album,omitempty"` TrackNum int `json:"trackNum,omitempty"` - Duration int `json:"duration,omitempty"` + Duration int64 `json:"duration,omitempty"` Links []Link `json:"link,omitempty"` Meta []Meta `json:"meta,omitempty"` Extension map[string]any `json:"extension,omitempty"` From cfc3cd522d693271c5adf6cefa5d32fb2b506845 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 1 May 2025 14:09:12 +0200 Subject: [PATCH 03/77] scrobblerlog: fix listen export not considering latest timestamp --- internal/backends/scrobblerlog/scrobblerlog.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 19ed30b..7955c15 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -151,9 +151,12 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c listens := make(models.ListensList, 0, len(b.log.Records)) client := strings.Split(b.log.Client, " ")[0] for _, record := range b.log.Records { - listens = append(listens, recordToListen(record, client)) + listen := recordToListen(record, client) + if listen.ListenedAt.After(oldestTimestamp) { + listens = append(listens, recordToListen(record, client)) + } } - sort.Sort(listens.NewerThan(oldestTimestamp)) + sort.Sort(listens) progress <- models.Progress{Total: int64(len(listens))}.Complete() results <- models.ListensResult{Items: listens} } From a645ec5c78aa3c0921405f6cad776ade85bf5dd6 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 1 May 2025 15:10:00 +0200 Subject: [PATCH 04/77] JSPF: Implemented export as loves and listens --- CHANGES.md | 4 + README.md | 2 +- internal/backends/jspf/jspf.go | 143 ++++++++++++++++++++++++++++++--- pkg/jspf/extensions.go | 33 +++++++- pkg/jspf/extensions_test.go | 31 ++++++- pkg/jspf/models.go | 54 ++++++------- 6 files changed, 225 insertions(+), 42 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5ac83ab..5dfd892 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Scotty Changelog +## 0.6.0 - WIP +- JSPF: Implemented export as loves and listens + + ## 0.5.2 - 2025-05-01 - ListenBrainz: fixed loves export not considering latest timestamp diff --git a/README.md b/README.md index 9f9f5c9..97e6120 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ Backend | Listens Export | Listens Import | Loves Export | Loves Import ----------------|----------------|----------------|--------------|------------- deezer | ✓ | ⨯ | ✓ | - funkwhale | ✓ | ⨯ | ✓ | - -jspf | - | ✓ | - | ✓ +jspf | ✓ | ✓ | ✓ | ✓ lastfm | ✓ | ✓ | ✓ | ✓ listenbrainz | ✓ | ✓ | ✓ | ✓ maloja | ✓ | ✓ | ⨯ | ⨯ diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index f98349a..a9e5c90 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -1,5 +1,5 @@ /* -Copyright © 2023-2024 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -18,15 +18,25 @@ Scotty. If not, see . package jspf import ( + "errors" "os" + "sort" + "strings" "time" + "go.uploadedlobster.com/mbtypes" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/pkg/jspf" ) +const ( + artistMBIDPrefix = "https://musicbrainz.org/artist/" + recordingMBIDPrefix = "https://musicbrainz.org/recording/" + releaseMBIDPrefix = "https://musicbrainz.org/release/" +) + type JSPFBackend struct { filePath string playlist jspf.Playlist @@ -68,7 +78,7 @@ func (b *JSPFBackend) InitConfig(config *config.ServiceConfig) error { Creator: config.GetString("username"), Identifier: config.GetString("identifier"), Tracks: make([]jspf.Track, 0), - Extension: map[string]any{ + Extension: jspf.ExtensionMap{ jspf.MusicBrainzPlaylistExtensionID: jspf.MusicBrainzPlaylistExtension{ LastModifiedAt: time.Now(), Public: true, @@ -86,6 +96,28 @@ func (b *JSPFBackend) FinishImport() error { return b.writeJSPF() } +func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { + defer close(results) + + err := b.readJSPF() + if err != nil { + progress <- models.Progress{}.Complete() + results <- models.ListensResult{Error: err} + return + } + + listens := make(models.ListensList, 0, len(b.playlist.Tracks)) + for _, track := range b.playlist.Tracks { + listen, err := trackAsListen(track) + if err == nil && listen != nil && listen.ListenedAt.After(oldestTimestamp) { + listens = append(listens, *listen) + } + } + sort.Sort(listens) + progress <- models.Progress{Total: int64(len(listens))}.Complete() + results <- models.ListensResult{Items: listens} +} + func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { for _, listen := range export.Items { track := listenAsTrack(listen) @@ -98,6 +130,28 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo return importResult, nil } +func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { + defer close(results) + + err := b.readJSPF() + if err != nil { + progress <- models.Progress{}.Complete() + results <- models.LovesResult{Error: err} + return + } + + loves := make(models.LovesList, 0, len(b.playlist.Tracks)) + for _, track := range b.playlist.Tracks { + love, err := trackAsLove(track) + if err == nil && love != nil && love.Created.After(oldestTimestamp) { + loves = append(loves, *love) + } + } + sort.Sort(loves) + progress <- models.Progress{Total: int64(len(loves))}.Complete() + results <- models.LovesResult{Items: loves} +} + func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { for _, love := range export.Items { track := loveAsTrack(love) @@ -112,22 +166,35 @@ func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models func listenAsTrack(l models.Listen) jspf.Track { l.FillAdditionalInfo() - track := trackAsTrack(l.Track) + track := trackAsJSPFTrack(l.Track) extension := makeMusicBrainzExtension(l.Track) extension.AddedAt = l.ListenedAt extension.AddedBy = l.UserName track.Extension[jspf.MusicBrainzTrackExtensionID] = extension if l.RecordingMBID != "" { - track.Identifier = append(track.Identifier, "https://musicbrainz.org/recording/"+string(l.RecordingMBID)) + track.Identifier = append(track.Identifier, recordingMBIDPrefix+string(l.RecordingMBID)) } return track } +func trackAsListen(t jspf.Track) (*models.Listen, error) { + track, addedAt, err := jspfTrackAsTrack(t) + if err != nil { + return nil, err + } + + listen := models.Listen{ + ListenedAt: *addedAt, + Track: *track, + } + return &listen, err +} + func loveAsTrack(l models.Love) jspf.Track { l.FillAdditionalInfo() - track := trackAsTrack(l.Track) + track := trackAsJSPFTrack(l.Track) extension := makeMusicBrainzExtension(l.Track) extension.AddedAt = l.Created extension.AddedBy = l.UserName @@ -138,25 +205,62 @@ func loveAsTrack(l models.Love) jspf.Track { recordingMBID = l.RecordingMBID } if recordingMBID != "" { - track.Identifier = append(track.Identifier, "https://musicbrainz.org/recording/"+string(recordingMBID)) + track.Identifier = append(track.Identifier, recordingMBIDPrefix+string(recordingMBID)) } return track } -func trackAsTrack(t models.Track) jspf.Track { +func trackAsLove(t jspf.Track) (*models.Love, error) { + track, addedAt, err := jspfTrackAsTrack(t) + if err != nil { + return nil, err + } + + love := models.Love{ + Created: *addedAt, + RecordingMBID: track.RecordingMBID, + Track: *track, + } + return &love, err +} + +func trackAsJSPFTrack(t models.Track) jspf.Track { track := jspf.Track{ Title: t.TrackName, Album: t.ReleaseName, Creator: t.ArtistName(), TrackNum: t.TrackNumber, Duration: t.Duration.Milliseconds(), - Extension: map[string]any{}, + Extension: jspf.ExtensionMap{}, } return track } +func jspfTrackAsTrack(t jspf.Track) (*models.Track, *time.Time, error) { + track := models.Track{ + ArtistNames: []string{t.Creator}, + ReleaseName: t.Album, + TrackName: t.Title, + TrackNumber: t.TrackNum, + Duration: time.Duration(t.Duration) * time.Millisecond, + } + + for _, id := range t.Identifier { + if strings.HasPrefix(id, recordingMBIDPrefix) { + track.RecordingMBID = mbtypes.MBID(id[len(recordingMBIDPrefix):]) + } + } + + addedAt, err := readMusicBrainzExtension(t, &track) + if err != nil { + return nil, nil, err + } + + return &track, addedAt, nil +} + func makeMusicBrainzExtension(t models.Track) jspf.MusicBrainzTrackExtension { extension := jspf.MusicBrainzTrackExtension{ AdditionalMetadata: t.AdditionalInfo, @@ -164,11 +268,11 @@ func makeMusicBrainzExtension(t models.Track) jspf.MusicBrainzTrackExtension { } for i, mbid := range t.ArtistMBIDs { - extension.ArtistIdentifiers[i] = "https://musicbrainz.org/artist/" + string(mbid) + extension.ArtistIdentifiers[i] = artistMBIDPrefix + string(mbid) } if t.ReleaseMBID != "" { - extension.ReleaseIdentifier = "https://musicbrainz.org/release/" + string(t.ReleaseMBID) + extension.ReleaseIdentifier = releaseMBIDPrefix + string(t.ReleaseMBID) } // The tracknumber tag would be redundant @@ -177,6 +281,25 @@ func makeMusicBrainzExtension(t models.Track) jspf.MusicBrainzTrackExtension { return extension } +func readMusicBrainzExtension(jspfTrack jspf.Track, outputTrack *models.Track) (*time.Time, error) { + ext := jspf.MusicBrainzTrackExtension{} + err := jspfTrack.Extension.Get(jspf.MusicBrainzTrackExtensionID, &ext) + if err != nil { + return nil, errors.New("missing MusicBrainz track extension") + } + + outputTrack.AdditionalInfo = ext.AdditionalMetadata + outputTrack.ReleaseMBID = mbtypes.MBID(ext.ReleaseIdentifier) + outputTrack.ArtistMBIDs = make([]mbtypes.MBID, len(ext.ArtistIdentifiers)) + for i, mbid := range ext.ArtistIdentifiers { + if strings.HasPrefix(mbid, artistMBIDPrefix) { + outputTrack.ArtistMBIDs[i] = mbtypes.MBID(mbid[len(artistMBIDPrefix):]) + } + } + + return &ext.AddedAt, nil +} + func (b *JSPFBackend) readJSPF() error { if b.append { file, err := os.Open(b.filePath) diff --git a/pkg/jspf/extensions.go b/pkg/jspf/extensions.go index 0f521c4..7cf99d3 100644 --- a/pkg/jspf/extensions.go +++ b/pkg/jspf/extensions.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,7 +22,28 @@ THE SOFTWARE. package jspf -import "time" +import ( + "encoding/json" + "fmt" + "time" +) + +// Represents a JSPF extension +type Extension any + +// A map of JSPF extensions +type ExtensionMap map[string]Extension + +// Parses the extension with the given ID and unmarshals it into "v". +// If the extensions is not found or the data cannot be unmarshalled, +// an error is returned. +func (e ExtensionMap) Get(id string, v any) error { + ext, ok := e[id] + if !ok { + return fmt.Errorf("extension %q not found", id) + } + return unmarshalExtension(ext, v) +} const ( // The identifier for the MusicBrainz / ListenBrainz JSPF playlist extension @@ -83,3 +104,11 @@ type MusicBrainzTrackExtension struct { // this document. AdditionalMetadata map[string]any `json:"additional_metadata,omitempty"` } + +func unmarshalExtension(ext Extension, v any) error { + asJson, err := json.Marshal(ext) + if err != nil { + return err + } + return json.Unmarshal(asJson, v) +} diff --git a/pkg/jspf/extensions_test.go b/pkg/jspf/extensions_test.go index 883301d..49d1bd5 100644 --- a/pkg/jspf/extensions_test.go +++ b/pkg/jspf/extensions_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -26,6 +26,7 @@ import ( "bytes" "fmt" "log" + "testing" "time" "go.uploadedlobster.com/scotty/pkg/jspf" @@ -38,7 +39,7 @@ func ExampleMusicBrainzTrackExtension() { Tracks: []jspf.Track{ { Title: "Oweynagat", - Extension: map[string]any{ + Extension: jspf.ExtensionMap{ jspf.MusicBrainzTrackExtensionID: jspf.MusicBrainzTrackExtension{ AddedAt: time.Date(2023, 11, 24, 07, 47, 50, 0, time.UTC), AddedBy: "scotty", @@ -72,3 +73,29 @@ func ExampleMusicBrainzTrackExtension() { // } // } } + +func TestExtensionMapGet(t *testing.T) { + ext := jspf.ExtensionMap{ + jspf.MusicBrainzTrackExtensionID: jspf.MusicBrainzTrackExtension{ + AddedAt: time.Date(2023, 11, 24, 07, 47, 50, 0, time.UTC), + AddedBy: "scotty", + }, + } + var trackExt jspf.MusicBrainzTrackExtension + err := ext.Get(jspf.MusicBrainzTrackExtensionID, &trackExt) + if err != nil { + t.Fatal(err) + } + if trackExt.AddedBy != "scotty" { + t.Fatalf("expected 'scotty', got '%s'", trackExt.AddedBy) + } +} + +func TestExtensionMapGetNotFound(t *testing.T) { + ext := jspf.ExtensionMap{} + var trackExt jspf.MusicBrainzTrackExtension + err := ext.Get(jspf.MusicBrainzTrackExtensionID, &trackExt) + if err == nil { + t.Fatal("expected ExtensionMap.Get to return an error") + } +} diff --git a/pkg/jspf/models.go b/pkg/jspf/models.go index d7a540c..829e922 100644 --- a/pkg/jspf/models.go +++ b/pkg/jspf/models.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -32,35 +32,35 @@ type JSPF struct { } type Playlist struct { - Title string `json:"title,omitempty"` - Creator string `json:"creator,omitempty"` - Annotation string `json:"annotation,omitempty"` - Info string `json:"info,omitempty"` - Location string `json:"location,omitempty"` - Identifier string `json:"identifier,omitempty"` - Image string `json:"image,omitempty"` - Date time.Time `json:"date,omitempty"` - License string `json:"license,omitempty"` - Attribution []Attribution `json:"attribution,omitempty"` - Links []Link `json:"link,omitempty"` - Meta []Meta `json:"meta,omitempty"` - Extension map[string]any `json:"extension,omitempty"` - Tracks []Track `json:"track"` + Title string `json:"title,omitempty"` + Creator string `json:"creator,omitempty"` + Annotation string `json:"annotation,omitempty"` + Info string `json:"info,omitempty"` + Location string `json:"location,omitempty"` + Identifier string `json:"identifier,omitempty"` + Image string `json:"image,omitempty"` + Date time.Time `json:"date,omitempty"` + License string `json:"license,omitempty"` + Attribution []Attribution `json:"attribution,omitempty"` + Links []Link `json:"link,omitempty"` + Meta []Meta `json:"meta,omitempty"` + Extension ExtensionMap `json:"extension,omitempty"` + Tracks []Track `json:"track"` } type Track struct { - Location []string `json:"location,omitempty"` - Identifier []string `json:"identifier,omitempty"` - Title string `json:"title,omitempty"` - Creator string `json:"creator,omitempty"` - Annotation string `json:"annotation,omitempty"` - Info string `json:"info,omitempty"` - Album string `json:"album,omitempty"` - TrackNum int `json:"trackNum,omitempty"` - Duration int64 `json:"duration,omitempty"` - Links []Link `json:"link,omitempty"` - Meta []Meta `json:"meta,omitempty"` - Extension map[string]any `json:"extension,omitempty"` + Location []string `json:"location,omitempty"` + Identifier []string `json:"identifier,omitempty"` + Title string `json:"title,omitempty"` + Creator string `json:"creator,omitempty"` + Annotation string `json:"annotation,omitempty"` + Info string `json:"info,omitempty"` + Album string `json:"album,omitempty"` + TrackNum int `json:"trackNum,omitempty"` + Duration int64 `json:"duration,omitempty"` + Links []Link `json:"link,omitempty"` + Meta []Meta `json:"meta,omitempty"` + Extension ExtensionMap `json:"extension,omitempty"` } type Attribution map[string]string From d757129bd710d0f05cd59321108db735508179c9 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 1 May 2025 15:20:37 +0200 Subject: [PATCH 05/77] jspf: also set username and recording MSID in exports --- README.md | 2 +- internal/backends/jspf/jspf.go | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 97e6120..c764730 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ You can help translate this project into your language with [Weblate](https://tr ## License -Scotty © 2023-2024 Philipp Wolfer +Scotty © 2023-2025 Philipp Wolfer Scotty is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index a9e5c90..6551f15 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -180,13 +180,14 @@ func listenAsTrack(l models.Listen) jspf.Track { } func trackAsListen(t jspf.Track) (*models.Listen, error) { - track, addedAt, err := jspfTrackAsTrack(t) + track, ext, err := jspfTrackAsTrack(t) if err != nil { return nil, err } listen := models.Listen{ - ListenedAt: *addedAt, + ListenedAt: ext.AddedAt, + UserName: ext.AddedBy, Track: *track, } return &listen, err @@ -212,16 +213,23 @@ func loveAsTrack(l models.Love) jspf.Track { } func trackAsLove(t jspf.Track) (*models.Love, error) { - track, addedAt, err := jspfTrackAsTrack(t) + track, ext, err := jspfTrackAsTrack(t) if err != nil { return nil, err } love := models.Love{ - Created: *addedAt, + Created: ext.AddedAt, + UserName: ext.AddedBy, RecordingMBID: track.RecordingMBID, Track: *track, } + + recordingMSID, ok := track.AdditionalInfo["recording_msid"].(string) + if ok { + love.RecordingMSID = mbtypes.MBID(recordingMSID) + } + return &love, err } @@ -238,7 +246,7 @@ func trackAsJSPFTrack(t models.Track) jspf.Track { return track } -func jspfTrackAsTrack(t jspf.Track) (*models.Track, *time.Time, error) { +func jspfTrackAsTrack(t jspf.Track) (*models.Track, *jspf.MusicBrainzTrackExtension, error) { track := models.Track{ ArtistNames: []string{t.Creator}, ReleaseName: t.Album, @@ -253,12 +261,12 @@ func jspfTrackAsTrack(t jspf.Track) (*models.Track, *time.Time, error) { } } - addedAt, err := readMusicBrainzExtension(t, &track) + ext, err := readMusicBrainzExtension(t, &track) if err != nil { return nil, nil, err } - return &track, addedAt, nil + return &track, ext, nil } func makeMusicBrainzExtension(t models.Track) jspf.MusicBrainzTrackExtension { @@ -281,7 +289,7 @@ func makeMusicBrainzExtension(t models.Track) jspf.MusicBrainzTrackExtension { return extension } -func readMusicBrainzExtension(jspfTrack jspf.Track, outputTrack *models.Track) (*time.Time, error) { +func readMusicBrainzExtension(jspfTrack jspf.Track, outputTrack *models.Track) (*jspf.MusicBrainzTrackExtension, error) { ext := jspf.MusicBrainzTrackExtension{} err := jspfTrack.Extension.Get(jspf.MusicBrainzTrackExtensionID, &ext) if err != nil { @@ -297,7 +305,7 @@ func readMusicBrainzExtension(jspfTrack jspf.Track, outputTrack *models.Track) ( } } - return &ext.AddedAt, nil + return &ext, nil } func (b *JSPFBackend) readJSPF() error { From bd7a35cd68fbc4794dafe5c47b64d3638b734141 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 2 May 2025 16:28:54 +0200 Subject: [PATCH 06/77] Update dependencies --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index ef1286c..8ed329b 100644 --- a/go.mod +++ b/go.mod @@ -15,12 +15,12 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/pelletier/go-toml/v2 v2.2.4 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 - github.com/spf13/cast v1.7.1 + github.com/spf13/cast v1.8.0 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d - github.com/vbauerster/mpb/v8 v8.9.3 + github.com/vbauerster/mpb/v8 v8.10.0 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.14.0 golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 @@ -66,7 +66,7 @@ require ( golang.org/x/tools v0.32.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.7 // indirect - modernc.org/libc v1.64.0 // indirect + modernc.org/libc v1.65.0 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.10.0 // indirect modernc.org/sqlite v1.37.0 // indirect diff --git a/go.sum b/go.sum index 8ade87a..4d3cb1b 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= +github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -125,8 +125,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4= -github.com/vbauerster/mpb/v8 v8.9.3 h1:PnMeF+sMvYv9u23l6DO6Q3+Mdj408mjLRXIzmUmU2Z8= -github.com/vbauerster/mpb/v8 v8.9.3/go.mod h1:hxS8Hz4C6ijnppDSIX6LjG8FYJSoPo9iIOcE53Zik0c= +github.com/vbauerster/mpb/v8 v8.10.0 h1:5ZYEWM4ovaZGAibjzW4PlQNb5k+JpzMqVwgNyk+K0M8= +github.com/vbauerster/mpb/v8 v8.10.0/go.mod h1:DYPFebxSahB+f7tuEUGauLQ7w8ij3wMr4clsVuJCV4I= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -216,8 +216,8 @@ modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.64.0 h1:U0k8BD2d3cD3e9I8RLcZgJBHAcsJzbXx5mKGSb5pyJA= -modernc.org/libc v1.64.0/go.mod h1:7m9VzGq7APssBTydds2zBcxGREwvIGpuUBaKTXdm2Qs= +modernc.org/libc v1.65.0 h1:e183gLDnAp9VJh6gWKdTy0CThL9Pt7MfcR/0bgb7Y1Y= +modernc.org/libc v1.65.0/go.mod h1:7m9VzGq7APssBTydds2zBcxGREwvIGpuUBaKTXdm2Qs= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4= From 8885e9cebcd90f5b3011af63bbe38d9c470a01b0 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 2 May 2025 21:35:14 +0200 Subject: [PATCH 07/77] Fix scrobblerlog timezone not being set from config --- internal/backends/scrobblerlog/scrobblerlog.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 7955c15..14ee24f 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -67,18 +67,19 @@ func (b *ScrobblerLogBackend) InitConfig(config *config.ServiceConfig) error { b.filePath = config.GetString("file-path") b.ignoreSkipped = config.GetBool("ignore-skipped", true) b.append = config.GetBool("append", true) - timezone := config.GetString("time-zone") - if timezone != "" { + b.log = scrobblerlog.ScrobblerLog{ + TZ: scrobblerlog.TimezoneUTC, + Client: "Rockbox unknown $Revision$", + } + + if timezone := config.GetString("time-zone"); timezone != "" { location, err := time.LoadLocation(timezone) if err != nil { return fmt.Errorf("Invalid time-zone %q: %w", timezone, err) } b.log.FallbackTimezone = location } - b.log = scrobblerlog.ScrobblerLog{ - TZ: scrobblerlog.TimezoneUTC, - Client: "Rockbox unknown $Revision$", - } + return nil } From b3136bde9a7f7d22a891c544a5b14231cd2226c4 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 11:43:26 +0200 Subject: [PATCH 08/77] jspf: add MB extension, if it does not exist --- internal/backends/jspf/jspf.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 6551f15..9d72765 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -77,14 +77,11 @@ func (b *JSPFBackend) InitConfig(config *config.ServiceConfig) error { Title: config.GetString("title"), Creator: config.GetString("username"), Identifier: config.GetString("identifier"), + Date: time.Now(), Tracks: make([]jspf.Track, 0), - Extension: jspf.ExtensionMap{ - jspf.MusicBrainzPlaylistExtensionID: jspf.MusicBrainzPlaylistExtension{ - LastModifiedAt: time.Now(), - Public: true, - }, - }, } + + b.addMusicBrainzPlaylistExtension() return nil } @@ -331,6 +328,7 @@ func (b *JSPFBackend) readJSPF() error { return err } b.playlist = playlist.Playlist + b.addMusicBrainzPlaylistExtension() } } @@ -350,3 +348,13 @@ func (b *JSPFBackend) writeJSPF() error { defer file.Close() return playlist.Write(file) } + +func (b *JSPFBackend) addMusicBrainzPlaylistExtension() { + if b.playlist.Extension == nil { + b.playlist.Extension = make(jspf.ExtensionMap, 1) + } + extension := jspf.MusicBrainzPlaylistExtension{Public: true} + b.playlist.Extension.Get(jspf.MusicBrainzPlaylistExtensionID, &extension) + extension.LastModifiedAt = time.Now() + b.playlist.Extension[jspf.MusicBrainzPlaylistExtensionID] = extension +} From 9480c69cbb42be234df13a7c6e52400370a3afe0 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 2 May 2025 08:36:59 +0200 Subject: [PATCH 09/77] Handle wait group for progress bar centrally This does not need to be exposed and caller only needs to wait for the Progress instance. --- internal/cli/progress.go | 5 +++-- internal/cli/transfer.go | 7 ++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 6d4421d..54ee4a8 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -28,7 +28,8 @@ import ( "go.uploadedlobster.com/scotty/internal/models" ) -func progressBar(wg *sync.WaitGroup, exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress { +func progressBar(exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress { + wg := &sync.WaitGroup{} p := mpb.New( mpb.WithWaitGroup(wg), mpb.WithOutput(color.Output), diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 0ba04b9..4777042 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Scotty is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -19,7 +19,6 @@ import ( "errors" "fmt" "strconv" - "sync" "time" "github.com/spf13/cobra" @@ -112,8 +111,7 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac // Prepare progress bars exportProgress := make(chan models.Progress) importProgress := make(chan models.Progress) - var wg sync.WaitGroup - progress := progressBar(&wg, exportProgress, importProgress) + progress := progressBar(exportProgress, importProgress) // Export from source exportChan := make(chan R, 1000) @@ -126,7 +124,6 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac if timestamp.After(result.LastTimestamp) { result.LastTimestamp = timestamp } - wg.Wait() progress.Wait() if result.Error != nil { printTimestamp("Import failed, last reported timestamp was %v (%s)", result.LastTimestamp) From 1c3364dad5b788b5ffa7c7b0451ccd2a469b17d8 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 2 May 2025 08:43:30 +0200 Subject: [PATCH 10/77] Close export results channel in generic implementation This removes the need for every implementation to handle this case. --- internal/backends/deezer/deezer.go | 4 ---- internal/backends/export.go | 8 +++++--- internal/backends/funkwhale/funkwhale.go | 4 ---- internal/backends/jspf/jspf.go | 4 ---- internal/backends/lastfm/lastfm.go | 4 ---- internal/backends/listenbrainz/listenbrainz.go | 3 --- internal/backends/maloja/maloja.go | 2 -- internal/backends/scrobblerlog/scrobblerlog.go | 1 - internal/backends/spotify/spotify.go | 4 ---- internal/backends/spotifyhistory/spotifyhistory.go | 2 -- internal/backends/subsonic/subsonic.go | 1 - 11 files changed, 5 insertions(+), 32 deletions(-) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index 756e271..1a5cb30 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -88,8 +88,6 @@ func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan totalDuration := startTime.Sub(oldestTimestamp) - defer close(results) - p := models.Progress{Total: int64(totalDuration.Seconds())} out: @@ -155,8 +153,6 @@ func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan m offset := math.MaxInt32 perPage := MaxItemsPerGet - defer close(results) - p := models.Progress{Total: int64(perPage)} var totalCount int diff --git a/internal/backends/export.go b/internal/backends/export.go index 44b8757..0346af2 100644 --- a/internal/backends/export.go +++ b/internal/backends/export.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Scotty is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -35,8 +35,9 @@ func (p ListensExportProcessor) ExportBackend() models.Backend { } func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { + defer close(results) + defer close(progress) p.Backend.ExportListens(oldestTimestamp, results, progress) - close(progress) } type LovesExportProcessor struct { @@ -48,6 +49,7 @@ func (p LovesExportProcessor) ExportBackend() models.Backend { } func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { + defer close(results) + defer close(progress) p.Backend.ExportLoves(oldestTimestamp, results, progress) - close(progress) } diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index 3e296c1..3619869 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -64,8 +64,6 @@ func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results c page := 1 perPage := MaxItemsPerGet - defer close(results) - // We need to gather the full list of listens in order to sort them listens := make(models.ListensList, 0, 2*perPage) p := models.Progress{Total: int64(perPage)} @@ -113,8 +111,6 @@ func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results cha page := 1 perPage := MaxItemsPerGet - defer close(results) - // We need to gather the full list of listens in order to sort them loves := make(models.LovesList, 0, 2*perPage) p := models.Progress{Total: int64(perPage)} diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 9d72765..826ea1b 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -94,8 +94,6 @@ func (b *JSPFBackend) FinishImport() error { } func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { - defer close(results) - err := b.readJSPF() if err != nil { progress <- models.Progress{}.Complete() @@ -128,8 +126,6 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo } func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { - defer close(results) - err := b.readJSPF() if err != nil { progress <- models.Progress{}.Complete() diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index 76fe9c7..444e5b0 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -93,8 +93,6 @@ func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan minTime := oldestTimestamp perPage := MaxItemsPerGet - defer close(results) - // We need to gather the full list of listens in order to sort them p := models.Progress{Total: int64(page)} @@ -258,8 +256,6 @@ func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan m page := 1 perPage := MaxItemsPerGet - defer close(results) - loves := make(models.LovesList, 0, 2*MaxItemsPerGet) p := models.Progress{Total: int64(perPage)} var totalCount int diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 6c7b747..e1ea53d 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -81,8 +81,6 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result totalDuration := startTime.Sub(minTime) - defer close(results) - p := models.Progress{Total: int64(totalDuration.Seconds())} for { @@ -195,7 +193,6 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo } func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { - defer close(results) exportChan := make(chan models.LovesResult) p := models.Progress{} diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index e9e3348..6bcdcc2 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -67,8 +67,6 @@ func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan page := 0 perPage := MaxItemsPerGet - defer close(results) - // We need to gather the full list of listens in order to sort them listens := make(models.ListensList, 0, 2*perPage) p := models.Progress{Total: int64(perPage)} diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 14ee24f..a355e3e 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -132,7 +132,6 @@ func (b *ScrobblerLogBackend) FinishImport() error { } func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { - defer close(results) file, err := os.Open(b.filePath) if err != nil { progress <- models.Progress{}.Complete() diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index 8c17903..8b6d9da 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -101,8 +101,6 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha totalDuration := startTime.Sub(oldestTimestamp) - defer close(results) - p := models.Progress{Total: int64(totalDuration.Seconds())} for { @@ -163,8 +161,6 @@ func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan offset := math.MaxInt32 perPage := MaxItemsPerGet - defer close(results) - p := models.Progress{Total: int64(perPage)} totalCount := 0 exportCount := 0 diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 1c986be..c150d3b 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -73,8 +73,6 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { - defer close(results) - files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) if err != nil { progress <- models.Progress{}.Complete() diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index d605324..a966c68 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -64,7 +64,6 @@ func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { - defer close(results) err := b.client.Authenticate(b.password) if err != nil { progress <- models.Progress{}.Complete() From 3b1adc9f1f81288efbea8174d8edc642c4173d9e Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 2 May 2025 09:25:07 +0200 Subject: [PATCH 11/77] Fix duplicate calls to handle import errors This fixes the import process hanging on error --- internal/backends/import.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/backends/import.go b/internal/backends/import.go index 6173a53..9938c10 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -41,7 +41,7 @@ func (p ListensImportProcessor) Process(results chan models.ListensResult, out c func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { if export.Error != nil { - return handleError(result, export.Error, progress), export.Error + return result, export.Error } if export.Total > 0 { @@ -51,7 +51,7 @@ func (p ListensImportProcessor) Import(export models.ListensResult, result model } importResult, err := p.Backend.ImportListens(export, result, progress) if err != nil { - return handleError(result, err, progress), err + return result, err } return importResult, nil } @@ -70,7 +70,7 @@ func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { if export.Error != nil { - return handleError(result, export.Error, progress), export.Error + return result, export.Error } if export.Total > 0 { @@ -80,7 +80,7 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im } importResult, err := p.Backend.ImportLoves(export, result, progress) if err != nil { - return handleError(importResult, err, progress), err + return result, err } return importResult, nil } From 069f0de2ee9db6cdf12a36c09ff5227e5ab18000 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 10:31:36 +0200 Subject: [PATCH 12/77] Call "FinishImport" even on error This gives the importer the chance to close connections and free resources to ensure already imported items are properly handled. --- internal/backends/import.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/backends/import.go b/internal/backends/import.go index 9938c10..0db5547 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -90,8 +90,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( defer close(progress) result := models.ImportResult{} - err := processor.ImportBackend().StartImport() - if err != nil { + if err := processor.ImportBackend().StartImport(); err != nil { out <- handleError(result, err, progress) return } @@ -99,6 +98,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( for exportResult := range results { importResult, err := processor.Import(exportResult, result, out, progress) if err != nil { + processor.ImportBackend().FinishImport() out <- handleError(result, err, progress) return } @@ -106,8 +106,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( progress <- models.Progress{}.FromImportResult(result) } - err = processor.ImportBackend().FinishImport() - if err != nil { + if err := processor.ImportBackend().FinishImport(); err != nil { out <- handleError(result, err, progress) return } From 55ac41b147a763651a4e5690ec4282800a6eff85 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 11:52:12 +0200 Subject: [PATCH 13/77] If import fails still save the last reported timestamp This allows continuing a partially failed import run. --- internal/backends/import.go | 11 ++++++----- internal/backends/maloja/maloja.go | 3 ++- internal/cli/transfer.go | 16 +++++++--------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/internal/backends/import.go b/internal/backends/import.go index 0db5547..c0d78bc 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -51,7 +51,7 @@ func (p ListensImportProcessor) Import(export models.ListensResult, result model } importResult, err := p.Backend.ImportListens(export, result, progress) if err != nil { - return result, err + return importResult, err } return importResult, nil } @@ -80,7 +80,7 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im } importResult, err := p.Backend.ImportLoves(export, result, progress) if err != nil { - return result, err + return importResult, err } return importResult, nil } @@ -89,6 +89,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( defer close(out) defer close(progress) result := models.ImportResult{} + p := models.Progress{} if err := processor.ImportBackend().StartImport(); err != nil { out <- handleError(result, err, progress) @@ -97,13 +98,13 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( for exportResult := range results { importResult, err := processor.Import(exportResult, result, out, progress) + result.Update(importResult) if err != nil { processor.ImportBackend().FinishImport() out <- handleError(result, err, progress) return } - result.Update(importResult) - progress <- models.Progress{}.FromImportResult(result) + progress <- p.FromImportResult(result) } if err := processor.ImportBackend().FinishImport(); err != nil { @@ -111,7 +112,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( return } - progress <- models.Progress{}.FromImportResult(result).Complete() + progress <- p.FromImportResult(result).Complete() out <- result } diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index 6bcdcc2..a22393b 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -105,6 +105,7 @@ out: } func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { + p := models.Progress{}.FromImportResult(importResult) for _, listen := range export.Items { scrobble := NewScrobble{ Title: listen.TrackName, @@ -125,7 +126,7 @@ func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResu importResult.UpdateTimestamp(listen.ListenedAt) importResult.ImportCount += 1 - progress <- models.Progress{}.FromImportResult(importResult) + progress <- p.FromImportResult(importResult) } return importResult, nil diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 4777042..83bad2a 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -121,16 +121,7 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac resultChan := make(chan models.ImportResult) go imp.Process(exportChan, resultChan, importProgress) result := <-resultChan - if timestamp.After(result.LastTimestamp) { - result.LastTimestamp = timestamp - } progress.Wait() - if result.Error != nil { - printTimestamp("Import failed, last reported timestamp was %v (%s)", result.LastTimestamp) - return result.Error - } - fmt.Println(i18n.Tr("Imported %v of %v %s into %v.", - result.ImportCount, result.TotalCount, c.entity, c.targetName)) // Update timestamp err = c.updateTimestamp(result, timestamp) @@ -138,6 +129,13 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac return err } + fmt.Println(i18n.Tr("Imported %v of %v %s into %v.", + result.ImportCount, result.TotalCount, c.entity, c.targetName)) + if result.Error != nil { + printTimestamp("Import failed, last reported timestamp was %v (%s)", result.LastTimestamp) + return result.Error + } + // Print errors if len(result.ImportLog) > 0 { fmt.Println() From 15d939e15098895c690bb4c055d84d23e800ceaa Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 12:02:17 +0200 Subject: [PATCH 14/77] Update changelog --- CHANGES.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 5dfd892..ba2685b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,15 @@ # Scotty Changelog ## 0.6.0 - WIP -- JSPF: Implemented export as loves and listens +- Fix program hanging endlessly if import fails (#11) +- If import fails still store the last successfully imported timestamp +- JSPF: implemented export as loves and listens +- JSPF: write track duration +- JSPF: read username and recording MSID +- JSPF: add MusicBrainz playlist extension in append mode, if it does not exist + in the existing JSPF file +- scrobblerlog: fix timezone not being set from config (#6) +- scrobblerlog: fix listen export not considering latest timestamp ## 0.5.2 - 2025-05-01 @@ -20,9 +28,9 @@ - ListenBrainz: log missing recording MBID on love import - Subsonic: support OpenSubsonic fields for recording MBID and genres (#5) - Subsonic: fixed progress for loves export -- scrobblerlog: add "time-zone" config option (#6). +- scrobblerlog: add "time-zone" config option (#6) - scrobblerlog: fixed progress for listen export -- scrobblerlog: renamed setting `include-skipped` to `ignore-skipped`. +- scrobblerlog: renamed setting `include-skipped` to `ignore-skipped` Note: 386 builds for Linux are not available with this release due to an incompatibility with latest version of gorm. From aae5123c3d1ab54de3b784d998ea11bd144039d6 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 12:59:40 +0200 Subject: [PATCH 15/77] Show progress bars as aborted on export / import error --- CHANGES.md | 1 + internal/backends/deezer/deezer.go | 4 ++-- internal/backends/import.go | 2 +- internal/backends/jspf/jspf.go | 4 ++-- internal/backends/lastfm/lastfm.go | 8 ++++---- internal/backends/listenbrainz/listenbrainz.go | 5 +++-- internal/backends/scrobblerlog/scrobblerlog.go | 2 +- internal/backends/spotify/spotify.go | 6 +++--- internal/backends/spotifyhistory/spotifyhistory.go | 4 ++-- internal/backends/subsonic/subsonic.go | 4 ++-- internal/cli/progress.go | 10 ++++++++-- internal/models/models.go | 6 ++++++ 12 files changed, 35 insertions(+), 21 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index ba2685b..a0a60f2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,7 @@ ## 0.6.0 - WIP - Fix program hanging endlessly if import fails (#11) - If import fails still store the last successfully imported timestamp +- Show progress bars as aborted on export / import error - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index 1a5cb30..4ba367d 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -94,7 +94,7 @@ out: for { result, err := b.client.UserHistory(offset, perPage) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.ListensResult{Error: err} return } @@ -160,7 +160,7 @@ out: for { result, err := b.client.UserTracks(offset, perPage) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.LovesResult{Error: err} return } diff --git a/internal/backends/import.go b/internal/backends/import.go index c0d78bc..407082c 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -118,6 +118,6 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( func handleError(result models.ImportResult, err error, progress chan models.Progress) models.ImportResult { result.Error = err - progress <- models.Progress{}.FromImportResult(result).Complete() + progress <- models.Progress{}.FromImportResult(result).Abort() return result } diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 826ea1b..e981741 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -96,7 +96,7 @@ func (b *JSPFBackend) FinishImport() error { func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { err := b.readJSPF() if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.ListensResult{Error: err} return } @@ -128,7 +128,7 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { err := b.readJSPF() if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.LovesResult{Error: err} return } diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index 444e5b0..d45f793 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -108,7 +108,7 @@ out: result, err := b.client.User.GetRecentTracks(args) if err != nil { results <- models.ListensResult{Error: err} - progress <- p.Complete() + progress <- p.Abort() return } @@ -127,7 +127,7 @@ out: timestamp, err := strconv.ParseInt(scrobble.Date.Uts, 10, 64) if err != nil { results <- models.ListensResult{Error: err} - progress <- p.Complete() + progress <- p.Abort() break out } if timestamp > oldestTimestamp.Unix() { @@ -268,7 +268,7 @@ out: "page": page, }) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.LovesResult{Error: err} return } @@ -282,7 +282,7 @@ out: for _, track := range result.Tracks { timestamp, err := strconv.ParseInt(track.Date.Uts, 10, 64) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.LovesResult{Error: err} return } diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index e1ea53d..fffe0f0 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -86,7 +86,7 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result for { result, err := b.client.GetListens(b.username, time.Now(), minTime) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.ListensResult{Error: err} return } @@ -199,8 +199,9 @@ func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results go b.exportLoves(oldestTimestamp, exportChan) for existingLoves := range exportChan { if existingLoves.Error != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.LovesResult{Error: existingLoves.Error} + return } p.Total = int64(existingLoves.Total) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index a355e3e..7890971 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -134,7 +134,7 @@ func (b *ScrobblerLogBackend) FinishImport() error { func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { file, err := os.Open(b.filePath) if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.ListensResult{Error: err} return } diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index 8b6d9da..be48dfe 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -106,7 +106,7 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha for { result, err := b.client.RecentlyPlayedAfter(minTime, MaxItemsPerGet) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.ListensResult{Error: err} return } @@ -118,7 +118,7 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha // Set minTime to the newest returned listen after, err := strconv.ParseInt(result.Cursors.After, 10, 64) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.ListensResult{Error: err} return } else if after <= minTime.Unix() { @@ -169,7 +169,7 @@ out: for { result, err := b.client.UserTracks(offset, perPage) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.LovesResult{Error: err} return } diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index c150d3b..23309f3 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -75,7 +75,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.ListensResult{Error: err} return } @@ -86,7 +86,7 @@ func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results for i, filePath := range files { history, err := readHistoryFile(filePath) if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.ListensResult{Error: err} return } diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index a966c68..370765e 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -66,14 +66,14 @@ func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error { func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { err := b.client.Authenticate(b.password) if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.LovesResult{Error: err} return } starred, err := b.client.GetStarred2(map[string]string{}) if err != nil { - progress <- models.Progress{}.Complete() + progress <- models.Progress{}.Abort() results <- models.LovesResult{Error: err} return } diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 54ee4a8..6b60697 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -59,10 +59,12 @@ func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar { ), mpb.AppendDecorators( decor.OnComplete( - decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}), + decor.OnAbort( + decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}), + i18n.Tr("error"), + ), i18n.Tr("done"), ), - // decor.OnComplete(decor.Percentage(decor.WC{W: 5, C: decor.DSyncWidthR}), "done"), decor.Name(" "), ), ) @@ -73,6 +75,10 @@ func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan model defer wg.Done() lastIterTime := time.Now() for progress := range progressChan { + if progress.Aborted { + bar.Abort(false) + return + } oldIterTime := lastIterTime lastIterTime = time.Now() bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime)) diff --git a/internal/models/models.go b/internal/models/models.go index f2dd71d..b8f9121 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -213,6 +213,7 @@ type Progress struct { Total int64 Elapsed int64 Completed bool + Aborted bool } func (p Progress) FromImportResult(result ImportResult) Progress { @@ -226,3 +227,8 @@ func (p Progress) Complete() Progress { p.Completed = true return p } + +func (p Progress) Abort() Progress { + p.Aborted = true + return p +} From dfe6773744a010fc77ea92fbb9ec6ec42fc52bf0 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 13:07:02 +0200 Subject: [PATCH 16/77] Update translations --- internal/translations/catalog.go | 38 ++++++------- .../translations/locales/de/out.gotext.json | 47 ++++++++-------- .../translations/locales/en/out.gotext.json | 53 +++++++++++-------- 3 files changed, 76 insertions(+), 62 deletions(-) diff --git a/internal/translations/catalog.go b/internal/translations/catalog.go index 3eb2f7e..fbe5eef 100644 --- a/internal/translations/catalog.go +++ b/internal/translations/catalog.go @@ -61,9 +61,9 @@ var messageKeyToIndex = map[string]int{ "Ignore listens in incognito mode": 30, "Ignore skipped listens": 27, "Ignored duplicate listen %v: \"%v\" by %v (%v)": 25, - "Import failed, last reported timestamp was %v (%s)": 45, + "Import failed, last reported timestamp was %v (%s)": 46, "Import log:": 47, - "Imported %v of %v %s into %v.": 46, + "Imported %v of %v %s into %v.": 45, "Latest timestamp: %v (%v)": 50, "Minimum playback duration for skipped tracks (seconds)": 31, "No": 39, @@ -85,6 +85,7 @@ var messageKeyToIndex = map[string]int{ "a service with this name already exists": 4, "backend %s does not implement %s": 13, "done": 37, + "error": 54, "exporting": 35, "importing": 36, "invalid timestamp string \"%v\"": 49, @@ -95,7 +96,7 @@ var messageKeyToIndex = map[string]int{ "unknown backend \"%s\"": 14, } -var deIndex = []uint32{ // 55 elements +var deIndex = []uint32{ // 56 elements // Entry 0 - 1F 0x00000000, 0x00000013, 0x00000027, 0x00000052, 0x0000005e, 0x0000008d, 0x000000bd, 0x00000104, @@ -109,10 +110,10 @@ var deIndex = []uint32{ // 55 elements 0x00000418, 0x00000443, 0x0000046d, 0x000004ad, 0x000004b8, 0x000004c3, 0x000004ca, 0x000004cd, 0x000004d2, 0x000004fb, 0x00000503, 0x0000050b, - 0x00000534, 0x00000552, 0x0000058f, 0x000005ba, + 0x00000534, 0x00000552, 0x0000057d, 0x000005ba, 0x000005c5, 0x000005d2, 0x000005f6, 0x00000619, - 0x0000066a, 0x000006a1, 0x000006c8, -} // Size: 244 bytes + 0x0000066a, 0x000006a1, 0x000006c8, 0x000006c8, +} // Size: 248 bytes const deData string = "" + // Size: 1736 bytes "\x04\x01\x09\x00\x0e\x02Export: %[1]s\x04\x01\x09\x01\x0a\x0e\x02Import:" + @@ -136,15 +137,15 @@ const deData string = "" + // Size: 1736 bytes "rein\x04\x00\x01\x0a;\x02Zugriffstoken erhalten, %[1]v kann jetzt verwen" + "det werden.\x02exportiere\x02importiere\x02fertig\x02Ja\x02Nein\x02keine" + " bestehenden Servicekonfigurationen\x02Service\x02Backend\x02Übertrage %" + - "[1]s von %[2]s nach %[3]s…\x02Ab Zeitstempel: %[1]v (%[2]v)\x02Import fe" + - "hlgeschlagen, letzter Zeitstempel war %[1]v (%[2]s)\x02%[1]v von %[2]v %" + - "[3]s in %[4]v importiert.\x02Importlog:\x02%[1]v: %[2]v\x02ungültiger Ze" + + "[1]s von %[2]s nach %[3]s…\x02Ab Zeitstempel: %[1]v (%[2]v)\x02%[1]v von" + + " %[2]v %[3]s in %[4]v importiert.\x02Import fehlgeschlagen, letzter Zeit" + + "stempel war %[1]v (%[2]s)\x02Importlog:\x02%[1]v: %[2]v\x02ungültiger Ze" + "itstempel „%[1]v“\x02Letzter Zeitstempel: %[1]v (%[2]v)\x02keine Konfigu" + "rationsdatei definiert, Konfiguration kann nicht geschrieben werden\x02S" + "chlüssel darf nur die Zeichen A-Za-z0-9_- beinhalten\x02keine Servicekon" + "figuration „%[1]v“" -var enIndex = []uint32{ // 55 elements +var enIndex = []uint32{ // 56 elements // Entry 0 - 1F 0x00000000, 0x00000013, 0x00000027, 0x00000044, 0x00000051, 0x00000079, 0x000000a1, 0x000000de, @@ -158,12 +159,12 @@ var enIndex = []uint32{ // 55 elements 0x0000039c, 0x000003c3, 0x000003df, 0x00000412, 0x0000041c, 0x00000426, 0x0000042b, 0x0000042f, 0x00000432, 0x00000455, 0x0000045d, 0x00000465, - 0x0000048f, 0x000004ad, 0x000004e6, 0x00000510, + 0x0000048f, 0x000004ad, 0x000004d7, 0x00000510, 0x0000051c, 0x00000529, 0x0000054a, 0x0000056a, - 0x0000059d, 0x000005c2, 0x000005e3, -} // Size: 244 bytes + 0x0000059d, 0x000005c2, 0x000005e3, 0x000005e9, +} // Size: 248 bytes -const enData string = "" + // Size: 1507 bytes +const enData string = "" + // Size: 1513 bytes "\x04\x01\x09\x00\x0e\x02export: %[1]s\x04\x01\x09\x01\x0a\x0e\x02import:" + " %[1]s\x02Failed reading config: %[1]v\x02Service name\x02a service with" + " this name already exists\x02Saved service %[1]v using backend %[2]v\x02" + @@ -184,10 +185,11 @@ const enData string = "" + // Size: 1507 bytes "token received, you can use %[1]v now.\x02exporting\x02importing\x02done" + "\x02Yes\x02No\x02no existing service configurations\x02Service\x02Backen" + "d\x02Transferring %[1]s from %[2]s to %[3]s…\x02From timestamp: %[1]v (%" + - "[2]v)\x02Import failed, last reported timestamp was %[1]v (%[2]s)\x02Imp" + - "orted %[1]v of %[2]v %[3]s into %[4]v.\x02Import log:\x02%[1]v: %[2]v" + + "[2]v)\x02Imported %[1]v of %[2]v %[3]s into %[4]v.\x02Import failed, las" + + "t reported timestamp was %[1]v (%[2]s)\x02Import log:\x02%[1]v: %[2]v" + "\x02invalid timestamp string \x22%[1]v\x22\x02Latest timestamp: %[1]v (%" + "[2]v)\x02no configuration file defined, cannot write config\x02key must " + - "only consist of A-Za-z0-9_-\x02no service configuration \x22%[1]v\x22" + "only consist of A-Za-z0-9_-\x02no service configuration \x22%[1]v\x22" + + "\x02error" - // Total table size 3731 bytes (3KiB); checksum: F7951710 + // Total table size 3745 bytes (3KiB); checksum: 5C167C91 diff --git a/internal/translations/locales/de/out.gotext.json b/internal/translations/locales/de/out.gotext.json index 680505e..06f5f82 100644 --- a/internal/translations/locales/de/out.gotext.json +++ b/internal/translations/locales/de/out.gotext.json @@ -378,6 +378,11 @@ "translatorComment": "Copied from source.", "fuzzy": true }, + { + "id": "error", + "message": "error", + "translation": "" + }, { "id": "done", "message": "done", @@ -462,27 +467,6 @@ } ] }, - { - "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "message": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "translation": "Import fehlgeschlagen, letzter Zeitstempel war {Arg_1} ({Arg_2})", - "placeholders": [ - { - "id": "Arg_1", - "string": "%[1]v", - "type": "", - "underlyingType": "interface{}", - "argNum": 1 - }, - { - "id": "Arg_2", - "string": "%[2]s", - "type": "", - "underlyingType": "string", - "argNum": 2 - } - ] - }, { "id": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", "message": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", @@ -522,6 +506,27 @@ } ] }, + { + "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "message": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "translation": "Import fehlgeschlagen, letzter Zeitstempel war {Arg_1} ({Arg_2})", + "placeholders": [ + { + "id": "Arg_1", + "string": "%[1]v", + "type": "", + "underlyingType": "interface{}", + "argNum": 1 + }, + { + "id": "Arg_2", + "string": "%[2]s", + "type": "", + "underlyingType": "string", + "argNum": 2 + } + ] + }, { "id": "Import log:", "message": "Import log:", diff --git a/internal/translations/locales/en/out.gotext.json b/internal/translations/locales/en/out.gotext.json index eecf359..0ef12a7 100644 --- a/internal/translations/locales/en/out.gotext.json +++ b/internal/translations/locales/en/out.gotext.json @@ -448,6 +448,13 @@ "translatorComment": "Copied from source.", "fuzzy": true }, + { + "id": "error", + "message": "error", + "translation": "error", + "translatorComment": "Copied from source.", + "fuzzy": true + }, { "id": "done", "message": "done", @@ -546,29 +553,6 @@ ], "fuzzy": true }, - { - "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "message": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "translation": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "translatorComment": "Copied from source.", - "placeholders": [ - { - "id": "Arg_1", - "string": "%[1]v", - "type": "", - "underlyingType": "interface{}", - "argNum": 1 - }, - { - "id": "Arg_2", - "string": "%[2]s", - "type": "", - "underlyingType": "string", - "argNum": 2 - } - ], - "fuzzy": true - }, { "id": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", "message": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", @@ -610,6 +594,29 @@ ], "fuzzy": true }, + { + "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "message": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "translation": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "translatorComment": "Copied from source.", + "placeholders": [ + { + "id": "Arg_1", + "string": "%[1]v", + "type": "", + "underlyingType": "interface{}", + "argNum": 1 + }, + { + "id": "Arg_2", + "string": "%[2]s", + "type": "", + "underlyingType": "string", + "argNum": 2 + } + ], + "fuzzy": true + }, { "id": "Import log:", "message": "Import log:", From a8517ea2490cb4ca7b5fa87bd5a9300206788da3 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 13:22:51 +0200 Subject: [PATCH 17/77] funkwhale: fix progress abort on error --- internal/backends/funkwhale/funkwhale.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index 3619869..e32a952 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -119,7 +119,7 @@ out: for { result, err := b.client.GetFavoriteTracks(page, perPage) if err != nil { - progress <- p.Complete() + progress <- p.Abort() results <- models.LovesResult{Error: err} return } From 05f0e8d172a1863c3913f5d0db4ed4f56af2fd43 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 13:24:12 +0200 Subject: [PATCH 18/77] Change string for aborted progress bar --- internal/cli/progress.go | 2 +- internal/translations/catalog.go | 30 +++++++++---------- .../translations/locales/de/out.gotext.json | 4 +-- .../translations/locales/en/out.gotext.json | 6 ++-- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 6b60697..88339ef 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -61,7 +61,7 @@ func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar { decor.OnComplete( decor.OnAbort( decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}), - i18n.Tr("error"), + i18n.Tr("aborted"), ), i18n.Tr("done"), ), diff --git a/internal/translations/catalog.go b/internal/translations/catalog.go index fbe5eef..20f3f35 100644 --- a/internal/translations/catalog.go +++ b/internal/translations/catalog.go @@ -83,17 +83,17 @@ var messageKeyToIndex = map[string]int{ "Visit the URL for authorization: %v": 32, "Yes": 38, "a service with this name already exists": 4, - "backend %s does not implement %s": 13, - "done": 37, - "error": 54, - "exporting": 35, - "importing": 36, - "invalid timestamp string \"%v\"": 49, - "key must only consist of A-Za-z0-9_-": 52, - "no configuration file defined, cannot write config": 51, - "no existing service configurations": 40, - "no service configuration \"%v\"": 53, - "unknown backend \"%s\"": 14, + "aborted": 54, + "backend %s does not implement %s": 13, + "done": 37, + "exporting": 35, + "importing": 36, + "invalid timestamp string \"%v\"": 49, + "key must only consist of A-Za-z0-9_-": 52, + "no configuration file defined, cannot write config": 51, + "no existing service configurations": 40, + "no service configuration \"%v\"": 53, + "unknown backend \"%s\"": 14, } var deIndex = []uint32{ // 56 elements @@ -161,10 +161,10 @@ var enIndex = []uint32{ // 56 elements 0x00000432, 0x00000455, 0x0000045d, 0x00000465, 0x0000048f, 0x000004ad, 0x000004d7, 0x00000510, 0x0000051c, 0x00000529, 0x0000054a, 0x0000056a, - 0x0000059d, 0x000005c2, 0x000005e3, 0x000005e9, + 0x0000059d, 0x000005c2, 0x000005e3, 0x000005eb, } // Size: 248 bytes -const enData string = "" + // Size: 1513 bytes +const enData string = "" + // Size: 1515 bytes "\x04\x01\x09\x00\x0e\x02export: %[1]s\x04\x01\x09\x01\x0a\x0e\x02import:" + " %[1]s\x02Failed reading config: %[1]v\x02Service name\x02a service with" + " this name already exists\x02Saved service %[1]v using backend %[2]v\x02" + @@ -190,6 +190,6 @@ const enData string = "" + // Size: 1513 bytes "\x02invalid timestamp string \x22%[1]v\x22\x02Latest timestamp: %[1]v (%" + "[2]v)\x02no configuration file defined, cannot write config\x02key must " + "only consist of A-Za-z0-9_-\x02no service configuration \x22%[1]v\x22" + - "\x02error" + "\x02aborted" - // Total table size 3745 bytes (3KiB); checksum: 5C167C91 + // Total table size 3747 bytes (3KiB); checksum: 1EAA307C diff --git a/internal/translations/locales/de/out.gotext.json b/internal/translations/locales/de/out.gotext.json index 06f5f82..810655f 100644 --- a/internal/translations/locales/de/out.gotext.json +++ b/internal/translations/locales/de/out.gotext.json @@ -379,8 +379,8 @@ "fuzzy": true }, { - "id": "error", - "message": "error", + "id": "aborted", + "message": "aborted", "translation": "" }, { diff --git a/internal/translations/locales/en/out.gotext.json b/internal/translations/locales/en/out.gotext.json index 0ef12a7..6f6825f 100644 --- a/internal/translations/locales/en/out.gotext.json +++ b/internal/translations/locales/en/out.gotext.json @@ -449,9 +449,9 @@ "fuzzy": true }, { - "id": "error", - "message": "error", - "translation": "error", + "id": "aborted", + "message": "aborted", + "translation": "aborted", "translatorComment": "Copied from source.", "fuzzy": true }, From cb6a534fa184b31bbaf9f0e89c2f41506e020f3b Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 11:25:07 +0000 Subject: [PATCH 19/77] Translated using Weblate (German) Currently translated at 100.0% (55 of 55 strings) Co-authored-by: Philipp Wolfer Translate-URL: https://translate.uploadedlobster.com/projects/scotty/app/de/ Translation: Scotty/app --- internal/translations/locales/de/out.gotext.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/translations/locales/de/out.gotext.json b/internal/translations/locales/de/out.gotext.json index 810655f..d114352 100644 --- a/internal/translations/locales/de/out.gotext.json +++ b/internal/translations/locales/de/out.gotext.json @@ -367,28 +367,28 @@ { "id": "exporting", "message": "exporting", - "translation": "exportiere", "translatorComment": "Copied from source.", - "fuzzy": true + "fuzzy": true, + "translation": "exportiere" }, { "id": "importing", "message": "importing", - "translation": "importiere", "translatorComment": "Copied from source.", - "fuzzy": true + "fuzzy": true, + "translation": "importiere" }, { "id": "aborted", "message": "aborted", - "translation": "" + "translation": "abgebrochen" }, { "id": "done", "message": "done", - "translation": "fertig", "translatorComment": "Copied from source.", - "fuzzy": true + "fuzzy": true, + "translation": "fertig" }, { "id": "Yes", @@ -617,4 +617,4 @@ ] } ] -} \ No newline at end of file +} From 54fffce1d942757a2c63e07f0a5d0c7b99152333 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 13:31:44 +0200 Subject: [PATCH 20/77] Update translation files --- internal/translations/catalog.go | 96 +++++------ .../locales/de/messages.gotext.json | 50 +++--- .../translations/locales/de/out.gotext.json | 17 +- .../locales/en/messages.gotext.json | 162 ++++++------------ .../translations/locales/en/out.gotext.json | 162 ++++++------------ 5 files changed, 189 insertions(+), 298 deletions(-) diff --git a/internal/translations/catalog.go b/internal/translations/catalog.go index 20f3f35..f0aaaae 100644 --- a/internal/translations/catalog.go +++ b/internal/translations/catalog.go @@ -42,12 +42,12 @@ var messageKeyToIndex = map[string]int{ "\tbackend: %v": 11, "\texport: %s": 0, "\timport: %s\n": 1, - "%v: %v": 48, + "%v: %v": 49, "Aborted": 8, "Access token": 19, "Access token received, you can use %v now.\n": 34, "Append to file": 21, - "Backend": 42, + "Backend": 43, "Check for duplicate listens on import (slower)": 24, "Client ID": 15, "Client secret": 16, @@ -57,42 +57,42 @@ var messageKeyToIndex = map[string]int{ "Error: OAuth state mismatch": 33, "Failed reading config: %v": 2, "File path": 20, - "From timestamp: %v (%v)": 44, + "From timestamp: %v (%v)": 45, "Ignore listens in incognito mode": 30, "Ignore skipped listens": 27, "Ignored duplicate listen %v: \"%v\" by %v (%v)": 25, - "Import failed, last reported timestamp was %v (%s)": 46, - "Import log:": 47, - "Imported %v of %v %s into %v.": 45, - "Latest timestamp: %v (%v)": 50, + "Import failed, last reported timestamp was %v (%s)": 47, + "Import log:": 48, + "Imported %v of %v %s into %v.": 46, + "Latest timestamp: %v (%v)": 51, "Minimum playback duration for skipped tracks (seconds)": 31, - "No": 39, + "No": 40, "Playlist title": 22, "Saved service %v using backend %v": 5, "Server URL": 17, - "Service": 41, + "Service": 42, "Service \"%v\" deleted\n": 9, "Service name": 3, "Specify a time zone for the listen timestamps": 28, "The backend %v requires authentication. Authenticate now?": 6, "Token received, you can close this window now.": 12, - "Transferring %s from %s to %s…": 43, + "Transferring %s from %s to %s…": 44, "Unique playlist identifier": 23, "Updated service %v using backend %v\n": 10, "User name": 18, "Visit the URL for authorization: %v": 32, - "Yes": 38, + "Yes": 39, "a service with this name already exists": 4, - "aborted": 54, + "aborted": 37, "backend %s does not implement %s": 13, - "done": 37, + "done": 38, "exporting": 35, "importing": 36, - "invalid timestamp string \"%v\"": 49, - "key must only consist of A-Za-z0-9_-": 52, - "no configuration file defined, cannot write config": 51, - "no existing service configurations": 40, - "no service configuration \"%v\"": 53, + "invalid timestamp string \"%v\"": 50, + "key must only consist of A-Za-z0-9_-": 53, + "no configuration file defined, cannot write config": 52, + "no existing service configurations": 41, + "no service configuration \"%v\"": 54, "unknown backend \"%s\"": 14, } @@ -108,14 +108,14 @@ var deIndex = []uint32{ // 56 elements 0x0000037e, 0x000003a4, 0x000003b4, 0x000003da, // Entry 20 - 3F 0x00000418, 0x00000443, 0x0000046d, 0x000004ad, - 0x000004b8, 0x000004c3, 0x000004ca, 0x000004cd, - 0x000004d2, 0x000004fb, 0x00000503, 0x0000050b, - 0x00000534, 0x00000552, 0x0000057d, 0x000005ba, - 0x000005c5, 0x000005d2, 0x000005f6, 0x00000619, - 0x0000066a, 0x000006a1, 0x000006c8, 0x000006c8, + 0x000004b8, 0x000004c3, 0x000004cf, 0x000004d6, + 0x000004d9, 0x000004de, 0x00000507, 0x0000050f, + 0x00000517, 0x00000540, 0x0000055e, 0x00000589, + 0x000005c6, 0x000005d1, 0x000005de, 0x00000602, + 0x00000625, 0x00000676, 0x000006ad, 0x000006d4, } // Size: 248 bytes -const deData string = "" + // Size: 1736 bytes +const deData string = "" + // Size: 1748 bytes "\x04\x01\x09\x00\x0e\x02Export: %[1]s\x04\x01\x09\x01\x0a\x0e\x02Import:" + " %[1]s\x02Fehler beim Lesen der Konfiguration: %[1]v\x02Servicename\x02e" + "in Service mit diesem Namen existiert bereits\x02Service %[1]v mit dem B" + @@ -135,15 +135,15 @@ const deData string = "" + // Size: 1736 bytes "inimale Wiedergabedauer für übersprungene Titel (Sekunden)\x02Zur Anmeld" + "ung folgende URL aufrufen: %[1]v\x02Fehler: OAuth-State stimmt nicht übe" + "rein\x04\x00\x01\x0a;\x02Zugriffstoken erhalten, %[1]v kann jetzt verwen" + - "det werden.\x02exportiere\x02importiere\x02fertig\x02Ja\x02Nein\x02keine" + - " bestehenden Servicekonfigurationen\x02Service\x02Backend\x02Übertrage %" + - "[1]s von %[2]s nach %[3]s…\x02Ab Zeitstempel: %[1]v (%[2]v)\x02%[1]v von" + - " %[2]v %[3]s in %[4]v importiert.\x02Import fehlgeschlagen, letzter Zeit" + - "stempel war %[1]v (%[2]s)\x02Importlog:\x02%[1]v: %[2]v\x02ungültiger Ze" + - "itstempel „%[1]v“\x02Letzter Zeitstempel: %[1]v (%[2]v)\x02keine Konfigu" + - "rationsdatei definiert, Konfiguration kann nicht geschrieben werden\x02S" + - "chlüssel darf nur die Zeichen A-Za-z0-9_- beinhalten\x02keine Servicekon" + - "figuration „%[1]v“" + "det werden.\x02exportiere\x02importiere\x02abgebrochen\x02fertig\x02Ja" + + "\x02Nein\x02keine bestehenden Servicekonfigurationen\x02Service\x02Backe" + + "nd\x02Übertrage %[1]s von %[2]s nach %[3]s…\x02Ab Zeitstempel: %[1]v (%[" + + "2]v)\x02%[1]v von %[2]v %[3]s in %[4]v importiert.\x02Import fehlgeschla" + + "gen, letzter Zeitstempel war %[1]v (%[2]s)\x02Importlog:\x02%[1]v: %[2]v" + + "\x02ungültiger Zeitstempel „%[1]v“\x02Letzter Zeitstempel: %[1]v (%[2]v)" + + "\x02keine Konfigurationsdatei definiert, Konfiguration kann nicht geschr" + + "ieben werden\x02Schlüssel darf nur die Zeichen A-Za-z0-9_- beinhalten" + + "\x02keine Servicekonfiguration „%[1]v“" var enIndex = []uint32{ // 56 elements // Entry 0 - 1F @@ -157,11 +157,11 @@ var enIndex = []uint32{ // 56 elements 0x00000307, 0x00000335, 0x00000344, 0x00000365, // Entry 20 - 3F 0x0000039c, 0x000003c3, 0x000003df, 0x00000412, - 0x0000041c, 0x00000426, 0x0000042b, 0x0000042f, - 0x00000432, 0x00000455, 0x0000045d, 0x00000465, - 0x0000048f, 0x000004ad, 0x000004d7, 0x00000510, - 0x0000051c, 0x00000529, 0x0000054a, 0x0000056a, - 0x0000059d, 0x000005c2, 0x000005e3, 0x000005eb, + 0x0000041c, 0x00000426, 0x0000042e, 0x00000433, + 0x00000437, 0x0000043a, 0x0000045d, 0x00000465, + 0x0000046d, 0x00000497, 0x000004b5, 0x000004df, + 0x00000518, 0x00000524, 0x00000531, 0x00000552, + 0x00000572, 0x000005a5, 0x000005ca, 0x000005eb, } // Size: 248 bytes const enData string = "" + // Size: 1515 bytes @@ -182,14 +182,14 @@ const enData string = "" + // Size: 1515 bytes "mps\x02Directory path\x02Ignore listens in incognito mode\x02Minimum pla" + "yback duration for skipped tracks (seconds)\x02Visit the URL for authori" + "zation: %[1]v\x02Error: OAuth state mismatch\x04\x00\x01\x0a.\x02Access " + - "token received, you can use %[1]v now.\x02exporting\x02importing\x02done" + - "\x02Yes\x02No\x02no existing service configurations\x02Service\x02Backen" + - "d\x02Transferring %[1]s from %[2]s to %[3]s…\x02From timestamp: %[1]v (%" + - "[2]v)\x02Imported %[1]v of %[2]v %[3]s into %[4]v.\x02Import failed, las" + - "t reported timestamp was %[1]v (%[2]s)\x02Import log:\x02%[1]v: %[2]v" + - "\x02invalid timestamp string \x22%[1]v\x22\x02Latest timestamp: %[1]v (%" + - "[2]v)\x02no configuration file defined, cannot write config\x02key must " + - "only consist of A-Za-z0-9_-\x02no service configuration \x22%[1]v\x22" + - "\x02aborted" + "token received, you can use %[1]v now.\x02exporting\x02importing\x02abor" + + "ted\x02done\x02Yes\x02No\x02no existing service configurations\x02Servic" + + "e\x02Backend\x02Transferring %[1]s from %[2]s to %[3]s…\x02From timestam" + + "p: %[1]v (%[2]v)\x02Imported %[1]v of %[2]v %[3]s into %[4]v.\x02Import " + + "failed, last reported timestamp was %[1]v (%[2]s)\x02Import log:\x02%[1]" + + "v: %[2]v\x02invalid timestamp string \x22%[1]v\x22\x02Latest timestamp: " + + "%[1]v (%[2]v)\x02no configuration file defined, cannot write config\x02k" + + "ey must only consist of A-Za-z0-9_-\x02no service configuration \x22%[1]" + + "v\x22" - // Total table size 3747 bytes (3KiB); checksum: 1EAA307C + // Total table size 3759 bytes (3KiB); checksum: 7B4CF967 diff --git a/internal/translations/locales/de/messages.gotext.json b/internal/translations/locales/de/messages.gotext.json index 8cbe44a..b44b7af 100644 --- a/internal/translations/locales/de/messages.gotext.json +++ b/internal/translations/locales/de/messages.gotext.json @@ -368,21 +368,23 @@ "id": "exporting", "message": "exporting", "translatorComment": "Copied from source.", - "fuzzy": true, "translation": "exportiere" }, { "id": "importing", "message": "importing", "translatorComment": "Copied from source.", - "fuzzy": true, "translation": "importiere" }, + { + "id": "aborted", + "message": "aborted", + "translation": "abgebrochen" + }, { "id": "done", "message": "done", "translatorComment": "Copied from source.", - "fuzzy": true, "translation": "fertig" }, { @@ -462,27 +464,6 @@ } ] }, - { - "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "message": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", - "translation": "Import fehlgeschlagen, letzter Zeitstempel war {Arg_1} ({Arg_2})", - "placeholders": [ - { - "id": "Arg_1", - "string": "%[1]v", - "type": "", - "underlyingType": "interface{}", - "argNum": 1 - }, - { - "id": "Arg_2", - "string": "%[2]s", - "type": "", - "underlyingType": "string", - "argNum": 2 - } - ] - }, { "id": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", "message": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", @@ -522,6 +503,27 @@ } ] }, + { + "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "message": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", + "translation": "Import fehlgeschlagen, letzter Zeitstempel war {Arg_1} ({Arg_2})", + "placeholders": [ + { + "id": "Arg_1", + "string": "%[1]v", + "type": "", + "underlyingType": "interface{}", + "argNum": 1 + }, + { + "id": "Arg_2", + "string": "%[2]s", + "type": "", + "underlyingType": "string", + "argNum": 2 + } + ] + }, { "id": "Import log:", "message": "Import log:", diff --git a/internal/translations/locales/de/out.gotext.json b/internal/translations/locales/de/out.gotext.json index d114352..863d9c8 100644 --- a/internal/translations/locales/de/out.gotext.json +++ b/internal/translations/locales/de/out.gotext.json @@ -367,16 +367,14 @@ { "id": "exporting", "message": "exporting", - "translatorComment": "Copied from source.", - "fuzzy": true, - "translation": "exportiere" + "translation": "exportiere", + "translatorComment": "Copied from source." }, { "id": "importing", "message": "importing", - "translatorComment": "Copied from source.", - "fuzzy": true, - "translation": "importiere" + "translation": "importiere", + "translatorComment": "Copied from source." }, { "id": "aborted", @@ -386,9 +384,8 @@ { "id": "done", "message": "done", - "translatorComment": "Copied from source.", - "fuzzy": true, - "translation": "fertig" + "translation": "fertig", + "translatorComment": "Copied from source." }, { "id": "Yes", @@ -617,4 +614,4 @@ ] } ] -} +} \ No newline at end of file diff --git a/internal/translations/locales/en/messages.gotext.json b/internal/translations/locales/en/messages.gotext.json index ed62636..878db22 100644 --- a/internal/translations/locales/en/messages.gotext.json +++ b/internal/translations/locales/en/messages.gotext.json @@ -15,8 +15,7 @@ "argNum": 1, "expr": "strings.Join(info.ExportCapabilities, \", \")" } - ], - "fuzzy": true + ] }, { "id": "import: {ImportCapabilities__}", @@ -32,8 +31,7 @@ "argNum": 1, "expr": "strings.Join(info.ImportCapabilities, \", \")" } - ], - "fuzzy": true + ] }, { "id": "Failed reading config: {Err}", @@ -49,22 +47,19 @@ "argNum": 1, "expr": "err" } - ], - "fuzzy": true + ] }, { "id": "Service name", "message": "Service name", "translation": "Service name", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "a service with this name already exists", "message": "a service with this name already exists", "translation": "a service with this name already exists", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Saved service {Name} using backend {Backend}", @@ -88,8 +83,7 @@ "argNum": 2, "expr": "service.Backend" } - ], - "fuzzy": true + ] }, { "id": "The backend {Backend} requires authentication. Authenticate now?", @@ -105,8 +99,7 @@ "argNum": 1, "expr": "service.Backend" } - ], - "fuzzy": true + ] }, { "id": "Delete the service configuration \"{Service}\"?", @@ -122,15 +115,13 @@ "argNum": 1, "expr": "service" } - ], - "fuzzy": true + ] }, { "id": "Aborted", "message": "Aborted", "translation": "Aborted", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Service \"{Name}\" deleted", @@ -146,8 +137,7 @@ "argNum": 1, "expr": "service.Name" } - ], - "fuzzy": true + ] }, { "id": "Updated service {Name} using backend {Backend}", @@ -171,8 +161,7 @@ "argNum": 2, "expr": "service.Backend" } - ], - "fuzzy": true + ] }, { "id": "backend: {Backend}", @@ -188,15 +177,13 @@ "argNum": 1, "expr": "s.Backend" } - ], - "fuzzy": true + ] }, { "id": "Token received, you can close this window now.", "message": "Token received, you can close this window now.", "translation": "Token received, you can close this window now.", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "backend {Backend} does not implement {InterfaceName}", @@ -220,8 +207,7 @@ "argNum": 2, "expr": "interfaceName" } - ], - "fuzzy": true + ] }, { "id": "unknown backend \"{BackendName}\"", @@ -237,78 +223,67 @@ "argNum": 1, "expr": "backendName" } - ], - "fuzzy": true + ] }, { "id": "Client ID", "message": "Client ID", "translation": "Client ID", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Client secret", "message": "Client secret", "translation": "Client secret", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Server URL", "message": "Server URL", "translation": "Server URL", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "User name", "message": "User name", "translation": "User name", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Access token", "message": "Access token", "translation": "Access token", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "File path", "message": "File path", "translation": "File path", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Append to file", "message": "Append to file", "translation": "Append to file", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Playlist title", "message": "Playlist title", "translation": "Playlist title", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Unique playlist identifier", "message": "Unique playlist identifier", "translation": "Unique playlist identifier", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Check for duplicate listens on import (slower)", "message": "Check for duplicate listens on import (slower)", "translation": "Check for duplicate listens on import (slower)", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Ignored duplicate listen {ListenedAt}: \"{TrackName}\" by {ArtistName} ({RecordingMBID})", @@ -348,50 +323,43 @@ "argNum": 4, "expr": "l.RecordingMBID" } - ], - "fuzzy": true + ] }, { "id": "Disable auto correction of submitted listens", "message": "Disable auto correction of submitted listens", "translation": "Disable auto correction of submitted listens", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Ignore skipped listens", "message": "Ignore skipped listens", "translation": "Ignore skipped listens", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Specify a time zone for the listen timestamps", "message": "Specify a time zone for the listen timestamps", "translation": "Specify a time zone for the listen timestamps", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Directory path", "message": "Directory path", "translation": "Directory path", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Ignore listens in incognito mode", "message": "Ignore listens in incognito mode", "translation": "Ignore listens in incognito mode", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Minimum playback duration for skipped tracks (seconds)", "message": "Minimum playback duration for skipped tracks (seconds)", "translation": "Minimum playback duration for skipped tracks (seconds)", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Visit the URL for authorization: {URL}", @@ -407,15 +375,13 @@ "argNum": 1, "expr": "authURL.URL" } - ], - "fuzzy": true + ] }, { "id": "Error: OAuth state mismatch", "message": "Error: OAuth state mismatch", "translation": "Error: OAuth state mismatch", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Access token received, you can use {Name} now.", @@ -431,64 +397,55 @@ "argNum": 1, "expr": "service.Name" } - ], - "fuzzy": true + ] }, { "id": "exporting", "message": "exporting", "translation": "exporting", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "importing", "message": "importing", "translation": "importing", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "done", "message": "done", "translation": "done", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Yes", "message": "Yes", "translation": "Yes", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "No", "message": "No", "translation": "No", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "no existing service configurations", "message": "no existing service configurations", "translation": "no existing service configurations", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Service", "message": "Service", "translation": "Service", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Backend", "message": "Backend", "translation": "Backend", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Transferring {Entity} from {SourceName} to {TargetName}…", @@ -520,8 +477,7 @@ "argNum": 3, "expr": "c.targetName" } - ], - "fuzzy": true + ] }, { "id": "From timestamp: {Arg_1} ({Arg_2})", @@ -543,8 +499,7 @@ "underlyingType": "interface{}", "argNum": 2 } - ], - "fuzzy": true + ] }, { "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", @@ -566,8 +521,7 @@ "underlyingType": "string", "argNum": 2 } - ], - "fuzzy": true + ] }, { "id": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", @@ -607,15 +561,13 @@ "argNum": 4, "expr": "c.targetName" } - ], - "fuzzy": true + ] }, { "id": "Import log:", "message": "Import log:", "translation": "Import log:", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "{Type}: {Message}", @@ -639,8 +591,7 @@ "argNum": 2, "expr": "entry.Message" } - ], - "fuzzy": true + ] }, { "id": "invalid timestamp string \"{FlagValue}\"", @@ -656,8 +607,7 @@ "argNum": 1, "expr": "flagValue" } - ], - "fuzzy": true + ] }, { "id": "Latest timestamp: {Arg_1} ({Arg_2})", @@ -679,22 +629,19 @@ "underlyingType": "interface{}", "argNum": 2 } - ], - "fuzzy": true + ] }, { "id": "no configuration file defined, cannot write config", "message": "no configuration file defined, cannot write config", "translation": "no configuration file defined, cannot write config", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "key must only consist of A-Za-z0-9_-", "message": "key must only consist of A-Za-z0-9_-", "translation": "key must only consist of A-Za-z0-9_-", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "no service configuration \"{Name}\"", @@ -710,8 +657,7 @@ "argNum": 1, "expr": "name" } - ], - "fuzzy": true + ] } ] } diff --git a/internal/translations/locales/en/out.gotext.json b/internal/translations/locales/en/out.gotext.json index 6f6825f..c2e0e84 100644 --- a/internal/translations/locales/en/out.gotext.json +++ b/internal/translations/locales/en/out.gotext.json @@ -15,8 +15,7 @@ "argNum": 1, "expr": "strings.Join(info.ExportCapabilities, \", \")" } - ], - "fuzzy": true + ] }, { "id": "import: {ImportCapabilities__}", @@ -32,8 +31,7 @@ "argNum": 1, "expr": "strings.Join(info.ImportCapabilities, \", \")" } - ], - "fuzzy": true + ] }, { "id": "Failed reading config: {Err}", @@ -49,22 +47,19 @@ "argNum": 1, "expr": "err" } - ], - "fuzzy": true + ] }, { "id": "Service name", "message": "Service name", "translation": "Service name", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "a service with this name already exists", "message": "a service with this name already exists", "translation": "a service with this name already exists", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Saved service {Name} using backend {Backend}", @@ -88,8 +83,7 @@ "argNum": 2, "expr": "service.Backend" } - ], - "fuzzy": true + ] }, { "id": "The backend {Backend} requires authentication. Authenticate now?", @@ -105,8 +99,7 @@ "argNum": 1, "expr": "service.Backend" } - ], - "fuzzy": true + ] }, { "id": "Delete the service configuration \"{Service}\"?", @@ -122,15 +115,13 @@ "argNum": 1, "expr": "service" } - ], - "fuzzy": true + ] }, { "id": "Aborted", "message": "Aborted", "translation": "Aborted", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Service \"{Name}\" deleted", @@ -146,8 +137,7 @@ "argNum": 1, "expr": "service.Name" } - ], - "fuzzy": true + ] }, { "id": "Updated service {Name} using backend {Backend}", @@ -171,8 +161,7 @@ "argNum": 2, "expr": "service.Backend" } - ], - "fuzzy": true + ] }, { "id": "backend: {Backend}", @@ -188,15 +177,13 @@ "argNum": 1, "expr": "s.Backend" } - ], - "fuzzy": true + ] }, { "id": "Token received, you can close this window now.", "message": "Token received, you can close this window now.", "translation": "Token received, you can close this window now.", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "backend {Backend} does not implement {InterfaceName}", @@ -220,8 +207,7 @@ "argNum": 2, "expr": "interfaceName" } - ], - "fuzzy": true + ] }, { "id": "unknown backend \"{BackendName}\"", @@ -237,78 +223,67 @@ "argNum": 1, "expr": "backendName" } - ], - "fuzzy": true + ] }, { "id": "Client ID", "message": "Client ID", "translation": "Client ID", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Client secret", "message": "Client secret", "translation": "Client secret", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Server URL", "message": "Server URL", "translation": "Server URL", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "User name", "message": "User name", "translation": "User name", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Access token", "message": "Access token", "translation": "Access token", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "File path", "message": "File path", "translation": "File path", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Append to file", "message": "Append to file", "translation": "Append to file", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Playlist title", "message": "Playlist title", "translation": "Playlist title", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Unique playlist identifier", "message": "Unique playlist identifier", "translation": "Unique playlist identifier", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Check for duplicate listens on import (slower)", "message": "Check for duplicate listens on import (slower)", "translation": "Check for duplicate listens on import (slower)", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Ignored duplicate listen {ListenedAt}: \"{TrackName}\" by {ArtistName} ({RecordingMBID})", @@ -348,50 +323,43 @@ "argNum": 4, "expr": "l.RecordingMBID" } - ], - "fuzzy": true + ] }, { "id": "Disable auto correction of submitted listens", "message": "Disable auto correction of submitted listens", "translation": "Disable auto correction of submitted listens", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Ignore skipped listens", "message": "Ignore skipped listens", "translation": "Ignore skipped listens", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Specify a time zone for the listen timestamps", "message": "Specify a time zone for the listen timestamps", "translation": "Specify a time zone for the listen timestamps", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Directory path", "message": "Directory path", "translation": "Directory path", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Ignore listens in incognito mode", "message": "Ignore listens in incognito mode", "translation": "Ignore listens in incognito mode", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Minimum playback duration for skipped tracks (seconds)", "message": "Minimum playback duration for skipped tracks (seconds)", "translation": "Minimum playback duration for skipped tracks (seconds)", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Visit the URL for authorization: {URL}", @@ -407,15 +375,13 @@ "argNum": 1, "expr": "authURL.URL" } - ], - "fuzzy": true + ] }, { "id": "Error: OAuth state mismatch", "message": "Error: OAuth state mismatch", "translation": "Error: OAuth state mismatch", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Access token received, you can use {Name} now.", @@ -431,22 +397,19 @@ "argNum": 1, "expr": "service.Name" } - ], - "fuzzy": true + ] }, { "id": "exporting", "message": "exporting", "translation": "exporting", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "importing", "message": "importing", "translation": "importing", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "aborted", @@ -459,43 +422,37 @@ "id": "done", "message": "done", "translation": "done", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Yes", "message": "Yes", "translation": "Yes", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "No", "message": "No", "translation": "No", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "no existing service configurations", "message": "no existing service configurations", "translation": "no existing service configurations", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Service", "message": "Service", "translation": "Service", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Backend", "message": "Backend", "translation": "Backend", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "Transferring {Entity} from {SourceName} to {TargetName}…", @@ -527,8 +484,7 @@ "argNum": 3, "expr": "c.targetName" } - ], - "fuzzy": true + ] }, { "id": "From timestamp: {Arg_1} ({Arg_2})", @@ -550,8 +506,7 @@ "underlyingType": "interface{}", "argNum": 2 } - ], - "fuzzy": true + ] }, { "id": "Imported {ImportCount} of {TotalCount} {Entity} into {TargetName}.", @@ -591,8 +546,7 @@ "argNum": 4, "expr": "c.targetName" } - ], - "fuzzy": true + ] }, { "id": "Import failed, last reported timestamp was {Arg_1} ({Arg_2})", @@ -614,15 +568,13 @@ "underlyingType": "string", "argNum": 2 } - ], - "fuzzy": true + ] }, { "id": "Import log:", "message": "Import log:", "translation": "Import log:", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "{Type}: {Message}", @@ -646,8 +598,7 @@ "argNum": 2, "expr": "entry.Message" } - ], - "fuzzy": true + ] }, { "id": "invalid timestamp string \"{FlagValue}\"", @@ -663,8 +614,7 @@ "argNum": 1, "expr": "flagValue" } - ], - "fuzzy": true + ] }, { "id": "Latest timestamp: {Arg_1} ({Arg_2})", @@ -686,22 +636,19 @@ "underlyingType": "interface{}", "argNum": 2 } - ], - "fuzzy": true + ] }, { "id": "no configuration file defined, cannot write config", "message": "no configuration file defined, cannot write config", "translation": "no configuration file defined, cannot write config", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "key must only consist of A-Za-z0-9_-", "message": "key must only consist of A-Za-z0-9_-", "translation": "key must only consist of A-Za-z0-9_-", - "translatorComment": "Copied from source.", - "fuzzy": true + "translatorComment": "Copied from source." }, { "id": "no service configuration \"{Name}\"", @@ -717,8 +664,7 @@ "argNum": 1, "expr": "name" } - ], - "fuzzy": true + ] } ] } \ No newline at end of file From 1f48abc2842e30861c5209271626866c1441da65 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 4 May 2025 15:18:14 +0200 Subject: [PATCH 21/77] Fixed timestamp displayed after import not being the updated one --- internal/cli/transfer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 83bad2a..0391d0d 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -124,7 +124,7 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac progress.Wait() // Update timestamp - err = c.updateTimestamp(result, timestamp) + err = c.updateTimestamp(&result, timestamp) if err != nil { return err } @@ -174,7 +174,7 @@ func (c *TransferCmd[E, I, R]) timestamp() (time.Time, error) { return time.Time{}, errors.New(i18n.Tr("invalid timestamp string \"%v\"", flagValue)) } -func (c *TransferCmd[E, I, R]) updateTimestamp(result models.ImportResult, oldTimestamp time.Time) error { +func (c *TransferCmd[E, I, R]) updateTimestamp(result *models.ImportResult, oldTimestamp time.Time) error { if oldTimestamp.After(result.LastTimestamp) { result.LastTimestamp = oldTimestamp } From b8e6ccffdb2f33aa1485f3d655ac55e5bfd61251 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 5 May 2025 11:38:29 +0200 Subject: [PATCH 22/77] Initial implementation of unified export/import progress Both export and import progress get updated over a unified channel. Most importantly this allows updating the import total from latest export results. --- internal/backends/backends_test.go | 12 ++- internal/backends/deezer/deezer.go | 41 +++++--- internal/backends/dump/dump.go | 10 +- internal/backends/export.go | 8 +- internal/backends/funkwhale/funkwhale.go | 46 ++++++--- internal/backends/import.go | 27 +++--- internal/backends/jspf/jspf.go | 34 +++++-- internal/backends/lastfm/lastfm.go | 56 +++++++---- .../backends/listenbrainz/listenbrainz.go | 53 ++++++---- internal/backends/maloja/maloja.go | 27 ++++-- .../backends/scrobblerlog/scrobblerlog.go | 19 ++-- internal/backends/spotify/spotify.go | 41 +++++--- .../backends/spotifyhistory/spotifyhistory.go | 22 +++-- internal/backends/subsonic/subsonic.go | 19 ++-- internal/cli/progress.go | 97 ++++++++++++++----- internal/cli/transfer.go | 11 +-- internal/models/interfaces.go | 10 +- internal/models/models.go | 30 ++++-- 18 files changed, 369 insertions(+), 194 deletions(-) diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index b6a6968..e115636 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -18,7 +18,6 @@ Scotty. If not, see . package backends_test import ( - "reflect" "testing" "github.com/spf13/viper" @@ -33,6 +32,7 @@ import ( "go.uploadedlobster.com/scotty/internal/backends/maloja" "go.uploadedlobster.com/scotty/internal/backends/scrobblerlog" "go.uploadedlobster.com/scotty/internal/backends/spotify" + "go.uploadedlobster.com/scotty/internal/backends/spotifyhistory" "go.uploadedlobster.com/scotty/internal/backends/subsonic" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" @@ -93,9 +93,9 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &funkwhale.FunkwhaleApiBackend{}) // expectInterface[models.LovesImport](t, &funkwhale.FunkwhaleApiBackend{}) - // expectInterface[models.ListensExport](t, &jspf.JSPFBackend{}) + expectInterface[models.ListensExport](t, &jspf.JSPFBackend{}) expectInterface[models.ListensImport](t, &jspf.JSPFBackend{}) - // expectInterface[models.LovesExport](t, &jspf.JSPFBackend{}) + expectInterface[models.LovesExport](t, &jspf.JSPFBackend{}) expectInterface[models.LovesImport](t, &jspf.JSPFBackend{}) // expectInterface[models.ListensExport](t, &lastfm.LastfmApiBackend{}) @@ -115,6 +115,8 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &spotify.SpotifyApiBackend{}) // expectInterface[models.LovesImport](t, &spotify.SpotifyApiBackend{}) + expectInterface[models.ListensExport](t, &spotifyhistory.SpotifyHistoryBackend{}) + expectInterface[models.ListensExport](t, &scrobblerlog.ScrobblerLogBackend{}) expectInterface[models.ListensImport](t, &scrobblerlog.ScrobblerLogBackend{}) @@ -125,6 +127,6 @@ func TestImplementsInterfaces(t *testing.T) { func expectInterface[T interface{}](t *testing.T, backend models.Backend) { ok, name := backends.ImplementsInterface[T](&backend) if !ok { - t.Errorf("%v expected to implement %v", reflect.TypeOf(backend).Name(), name) + t.Errorf("%v expected to implement %v", backend.Name(), name) } } diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index 4ba367d..2209769 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Scotty is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -77,7 +77,7 @@ func (b *DeezerApiBackend) OAuth2Setup(token oauth2.TokenSource) error { return nil } -func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 @@ -88,13 +88,18 @@ func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan totalDuration := startTime.Sub(oldestTimestamp) - p := models.Progress{Total: int64(totalDuration.Seconds())} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } out: for { result, err := b.client.UserHistory(offset, perPage) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -102,7 +107,6 @@ out: // The offset was higher then the actual number of tracks. Adjust the offset // and continue. if offset >= result.Total { - p.Total = int64(result.Total) offset = max(result.Total-perPage, 0) continue } @@ -128,7 +132,8 @@ out: } remainingTime := startTime.Sub(minTime) - p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + p.Export.TotalItems += len(listens) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) progress <- p results <- models.ListensResult{Items: listens, OldestTimestamp: minTime} @@ -144,23 +149,29 @@ out: } results <- models.ListensResult{OldestTimestamp: minTime} - progress <- p.Complete() + p.Export.Complete() + progress <- p } -func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 perPage := MaxItemsPerGet - p := models.Progress{Total: int64(perPage)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(perPage), + }, + } var totalCount int out: for { result, err := b.client.UserTracks(offset, perPage) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } @@ -168,8 +179,8 @@ out: // The offset was higher then the actual number of tracks. Adjust the offset // and continue. if offset >= result.Total { - p.Total = int64(result.Total) totalCount = result.Total + p.Export.Total = int64(totalCount) offset = max(result.Total-perPage, 0) continue } @@ -186,13 +197,14 @@ out: loves = append(loves, love) } else { totalCount -= 1 - break } } sort.Sort(loves) results <- models.LovesResult{Items: loves, Total: totalCount} - p.Elapsed += int64(count) + p.Export.TotalItems = totalCount + p.Export.Total = int64(totalCount) + p.Export.Elapsed += int64(count) progress <- p if offset <= 0 { @@ -206,7 +218,8 @@ out: } } - progress <- p.Complete() + p.Export.Complete() + progress <- p } func (t Listen) AsListen() models.Listen { diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index 70be12d..add8711 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -36,27 +36,27 @@ func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error { func (b *DumpBackend) StartImport() error { return nil } func (b *DumpBackend) FinishImport() error { return nil } -func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { importResult.UpdateTimestamp(listen.ListenedAt) importResult.ImportCount += 1 msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) importResult.Log(models.Info, msg) - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil } -func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { importResult.UpdateTimestamp(love.Created) importResult.ImportCount += 1 msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) importResult.Log(models.Info, msg) - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil diff --git a/internal/backends/export.go b/internal/backends/export.go index 0346af2..c7a1f58 100644 --- a/internal/backends/export.go +++ b/internal/backends/export.go @@ -23,7 +23,7 @@ import ( type ExportProcessor[T models.ListensResult | models.LovesResult] interface { ExportBackend() models.Backend - Process(oldestTimestamp time.Time, results chan T, progress chan models.Progress) + Process(oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress) } type ListensExportProcessor struct { @@ -34,9 +34,8 @@ func (p ListensExportProcessor) ExportBackend() models.Backend { return p.Backend } -func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { defer close(results) - defer close(progress) p.Backend.ExportListens(oldestTimestamp, results, progress) } @@ -48,8 +47,7 @@ func (p LovesExportProcessor) ExportBackend() models.Backend { return p.Backend } -func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { defer close(results) - defer close(progress) p.Backend.ExportLoves(oldestTimestamp, results, progress) } diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index e32a952..cd2f28e 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -60,19 +60,26 @@ func (b *FunkwhaleApiBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := 1 perPage := MaxItemsPerGet // We need to gather the full list of listens in order to sort them listens := make(models.ListensList, 0, 2*perPage) - p := models.Progress{Total: int64(perPage)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(perPage), + }, + } out: for { result, err := b.client.GetHistoryListenings(b.username, page, perPage) if err != nil { + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} + return } count := len(result.Results) @@ -83,7 +90,7 @@ out: for _, fwListen := range result.Results { listen := fwListen.AsListen() if listen.ListenedAt.After(oldestTimestamp) { - p.Elapsed += 1 + p.Export.Elapsed += 1 listens = append(listens, listen) } else { break out @@ -92,34 +99,42 @@ out: if result.Next == "" { // No further results - p.Total = p.Elapsed - p.Total -= int64(perPage - count) + p.Export.Total = p.Export.Elapsed + p.Export.Total -= int64(perPage - count) break out } - p.Total += int64(perPage) + p.Export.TotalItems = len(listens) + p.Export.Total += int64(perPage) progress <- p page += 1 } sort.Sort(listens) - progress <- p.Complete() + p.Export.TotalItems = len(listens) + p.Export.Complete() + progress <- p results <- models.ListensResult{Items: listens} } -func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { page := 1 perPage := MaxItemsPerGet // We need to gather the full list of listens in order to sort them loves := make(models.LovesList, 0, 2*perPage) - p := models.Progress{Total: int64(perPage)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(perPage), + }, + } out: for { result, err := b.client.GetFavoriteTracks(page, perPage) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } @@ -132,7 +147,7 @@ out: for _, favorite := range result.Results { love := favorite.AsLove() if love.Created.After(oldestTimestamp) { - p.Elapsed += 1 + p.Export.Elapsed += 1 loves = append(loves, love) } else { break out @@ -144,13 +159,16 @@ out: break out } - p.Total += int64(perPage) + p.Export.TotalItems = len(loves) + p.Export.Total += int64(perPage) progress <- p page += 1 } sort.Sort(loves) - progress <- p.Complete() + p.Export.TotalItems = len(loves) + p.Export.Complete() + progress <- p results <- models.LovesResult{Items: loves} } diff --git a/internal/backends/import.go b/internal/backends/import.go index 407082c..d3b86ac 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -23,8 +23,8 @@ import ( type ImportProcessor[T models.ListensResult | models.LovesResult] interface { ImportBackend() models.ImportBackend - Process(results chan T, out chan models.ImportResult, progress chan models.Progress) - Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) + Process(results chan T, out chan models.ImportResult, progress chan models.TransferProgress) + Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) } type ListensImportProcessor struct { @@ -35,11 +35,11 @@ func (p ListensImportProcessor) ImportBackend() models.ImportBackend { return p.Backend } -func (p ListensImportProcessor) Process(results chan models.ListensResult, out chan models.ImportResult, progress chan models.Progress) { +func (p ListensImportProcessor) Process(results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) { process(p, results, out, progress) } -func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { if export.Error != nil { return result, export.Error } @@ -64,11 +64,11 @@ func (p LovesImportProcessor) ImportBackend() models.ImportBackend { return p.Backend } -func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan models.ImportResult, progress chan models.Progress) { +func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) { process(p, results, out, progress) } -func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { if export.Error != nil { return result, export.Error } @@ -85,11 +85,10 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im return importResult, nil } -func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](processor P, results chan R, out chan models.ImportResult, progress chan models.Progress) { +func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](processor P, results chan R, out chan models.ImportResult, progress chan models.TransferProgress) { defer close(out) - defer close(progress) result := models.ImportResult{} - p := models.Progress{} + p := models.TransferProgress{} if err := processor.ImportBackend().StartImport(); err != nil { out <- handleError(result, err, progress) @@ -104,7 +103,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( out <- handleError(result, err, progress) return } - progress <- p.FromImportResult(result) + progress <- p.FromImportResult(result, false) } if err := processor.ImportBackend().FinishImport(); err != nil { @@ -112,12 +111,14 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( return } - progress <- p.FromImportResult(result).Complete() + progress <- p.FromImportResult(result, true) out <- result } -func handleError(result models.ImportResult, err error, progress chan models.Progress) models.ImportResult { +func handleError(result models.ImportResult, err error, progress chan models.TransferProgress) models.ImportResult { result.Error = err - progress <- models.Progress{}.FromImportResult(result).Abort() + p := models.TransferProgress{}.FromImportResult(result, false) + p.Import.Abort() + progress <- p return result } diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index e981741..0e200f2 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -93,10 +93,15 @@ func (b *JSPFBackend) FinishImport() error { return b.writeJSPF() } -func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { err := b.readJSPF() + p := models.TransferProgress{ + Export: &models.Progress{}, + } + if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -109,11 +114,13 @@ func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan mode } } sort.Sort(listens) - progress <- models.Progress{Total: int64(len(listens))}.Complete() + p.Export.Total = int64(len(listens)) + p.Export.Complete() + progress <- p results <- models.ListensResult{Items: listens} } -func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { track := listenAsTrack(listen) b.playlist.Tracks = append(b.playlist.Tracks, track) @@ -121,14 +128,19 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo importResult.UpdateTimestamp(listen.ListenedAt) } - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } -func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { err := b.readJSPF() + p := models.TransferProgress{ + Export: &models.Progress{}, + } + if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } @@ -141,11 +153,13 @@ func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models } } sort.Sort(loves) - progress <- models.Progress{Total: int64(len(loves))}.Complete() + p.Export.Total = int64(len(loves)) + p.Export.Complete() + progress <- p results <- models.LovesResult{Items: loves} } -func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { track := loveAsTrack(love) b.playlist.Tracks = append(b.playlist.Tracks, track) @@ -153,7 +167,7 @@ func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models importResult.UpdateTimestamp(love.Created) } - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index d45f793..d262ada 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Scotty is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -88,13 +88,17 @@ func (b *LastfmApiBackend) OAuth2Setup(token oauth2.TokenSource) error { return nil } -func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := MaxPage minTime := oldestTimestamp perPage := MaxItemsPerGet // We need to gather the full list of listens in order to sort them - p := models.Progress{Total: int64(page)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(page), + }, + } out: for page > 0 { @@ -108,7 +112,8 @@ out: result, err := b.client.User.GetRecentTracks(args) if err != nil { results <- models.ListensResult{Error: err} - progress <- p.Abort() + p.Export.Abort() + progress <- p return } @@ -127,11 +132,12 @@ out: timestamp, err := strconv.ParseInt(scrobble.Date.Uts, 10, 64) if err != nil { results <- models.ListensResult{Error: err} - progress <- p.Abort() + p.Export.Abort() + progress <- p break out } if timestamp > oldestTimestamp.Unix() { - p.Elapsed += 1 + p.Export.Elapsed += 1 listen := models.Listen{ ListenedAt: time.Unix(timestamp, 0), UserName: b.username, @@ -165,16 +171,18 @@ out: Total: result.Total, OldestTimestamp: minTime, } - p.Total = int64(result.TotalPages) - p.Elapsed = int64(result.TotalPages - page) + p.Export.Total = int64(result.TotalPages) + p.Export.Elapsed = int64(result.TotalPages - page) + p.Export.TotalItems += len(listens) progress <- p } results <- models.ListensResult{OldestTimestamp: minTime} - progress <- p.Complete() + p.Export.Complete() + progress <- p } -func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) for i := 0; i < total; i += MaxListensPerSubmission { listens := export.Items[i:min(i+MaxListensPerSubmission, total)] @@ -244,20 +252,24 @@ func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResu importResult.UpdateTimestamp(listens[count-1].ListenedAt) importResult.ImportCount += accepted - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil } -func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. page := 1 perPage := MaxItemsPerGet loves := make(models.LovesList, 0, 2*MaxItemsPerGet) - p := models.Progress{Total: int64(perPage)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(perPage), + }, + } var totalCount int out: @@ -268,12 +280,12 @@ out: "page": page, }) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } - p.Total = int64(result.Total) count := len(result.Tracks) if count == 0 { break out @@ -282,7 +294,8 @@ out: for _, track := range result.Tracks { timestamp, err := strconv.ParseInt(track.Date.Uts, 10, 64) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } @@ -308,18 +321,21 @@ out: } } - p.Elapsed += int64(count) + p.Export.Total += int64(perPage) + p.Export.TotalItems = totalCount + p.Export.Elapsed += int64(count) progress <- p page += 1 } sort.Sort(loves) + p.Export.Complete() + progress <- p results <- models.LovesResult{Items: loves, Total: totalCount} - progress <- p.Complete() } -func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { err := b.client.Track.Love(lastfm.P{ "track": love.TrackName, @@ -335,7 +351,7 @@ func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult m importResult.Log(models.Error, msg) } - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index fffe0f0..61597d1 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -72,21 +72,25 @@ func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error func (b *ListenBrainzApiBackend) StartImport() error { return nil } func (b *ListenBrainzApiBackend) FinishImport() error { return nil } -func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { startTime := time.Now() minTime := oldestTimestamp if minTime.Unix() < 1 { minTime = time.Unix(1, 0) } - totalDuration := startTime.Sub(minTime) - - p := models.Progress{Total: int64(totalDuration.Seconds())} + totalDuration := startTime.Sub(oldestTimestamp) + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } for { result, err := b.client.GetListens(b.username, time.Now(), minTime) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -96,7 +100,7 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result if minTime.Unix() < result.Payload.OldestListenTimestamp { minTime = time.Unix(result.Payload.OldestListenTimestamp, 0) totalDuration = startTime.Sub(minTime) - p.Total = int64(totalDuration.Seconds()) + p.Export.Total = int64(totalDuration.Seconds()) continue } else { break @@ -119,18 +123,20 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result } sort.Sort(listens) - p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + p.Export.TotalItems += len(listens) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) progress <- p results <- models.ListensResult{Items: listens, OldestTimestamp: minTime} } results <- models.ListensResult{OldestTimestamp: minTime} - progress <- p.Complete() + p.Export.Complete() + progress <- p } -func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) - p := models.Progress{}.FromImportResult(importResult) + p := models.TransferProgress{}.FromImportResult(importResult, false) for i := 0; i < total; i += MaxListensPerRequest { listens := export.Items[i:min(i+MaxListensPerRequest, total)] count := len(listens) @@ -146,7 +152,7 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo for _, l := range listens { if b.checkDuplicates { isDupe, err := b.checkDuplicateListen(l) - p.Elapsed += 1 + p.Import.Elapsed += 1 progress <- p if err != nil { return importResult, err @@ -186,31 +192,36 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo importResult.UpdateTimestamp(listens[count-1].ListenedAt) } importResult.ImportCount += count - progress <- p.FromImportResult(importResult) + progress <- p.FromImportResult(importResult, false) } return importResult, nil } -func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { exportChan := make(chan models.LovesResult) - p := models.Progress{} + p := models.TransferProgress{ + Export: &models.Progress{}, + } go b.exportLoves(oldestTimestamp, exportChan) for existingLoves := range exportChan { if existingLoves.Error != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: existingLoves.Error} return } - p.Total = int64(existingLoves.Total) - p.Elapsed += int64(existingLoves.Items.Len()) + p.Export.TotalItems = existingLoves.Total + p.Export.Total = int64(existingLoves.Total) + p.Export.Elapsed += int64(len(existingLoves.Items)) progress <- p results <- existingLoves } - progress <- p.Complete() + p.Export.Complete() + progress <- p } func (b *ListenBrainzApiBackend) exportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { @@ -260,7 +271,7 @@ out: } } -func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { if len(b.existingMBIDs) == 0 { existingLovesChan := make(chan models.LovesResult) go b.exportLoves(time.Unix(0, 0), existingLovesChan) @@ -330,7 +341,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importRe importResult.Log(models.Error, msg) } - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index a22393b..8968942 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -63,19 +63,24 @@ func (b *MalojaApiBackend) InitConfig(config *config.ServiceConfig) error { func (b *MalojaApiBackend) StartImport() error { return nil } func (b *MalojaApiBackend) FinishImport() error { return nil } -func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := 0 perPage := MaxItemsPerGet // We need to gather the full list of listens in order to sort them listens := make(models.ListensList, 0, 2*perPage) - p := models.Progress{Total: int64(perPage)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(perPage), + }, + } out: for { result, err := b.client.GetScrobbles(page, perPage) if err != nil { - progress <- p.Complete() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -87,25 +92,27 @@ out: for _, scrobble := range result.List { if scrobble.ListenedAt > oldestTimestamp.Unix() { - p.Elapsed += 1 + p.Export.Elapsed += 1 listens = append(listens, scrobble.AsListen()) } else { break out } } - p.Total += int64(perPage) + p.Export.TotalItems = len(listens) + p.Export.Total += int64(perPage) progress <- p page += 1 } sort.Sort(listens) - progress <- p.Complete() + p.Export.Complete() + progress <- p results <- models.ListensResult{Items: listens} } -func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { - p := models.Progress{}.FromImportResult(importResult) +func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + p := models.TransferProgress{}.FromImportResult(importResult, false) for _, listen := range export.Items { scrobble := NewScrobble{ Title: listen.TrackName, @@ -126,7 +133,7 @@ func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResu importResult.UpdateTimestamp(listen.ListenedAt) importResult.ImportCount += 1 - progress <- p.FromImportResult(importResult) + progress <- p.FromImportResult(importResult, false) } return importResult, nil diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 7890971..db4e349 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -131,10 +131,14 @@ func (b *ScrobblerLogBackend) FinishImport() error { return b.file.Close() } -func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { file, err := os.Open(b.filePath) + p := models.TransferProgress{ + Export: &models.Progress{}, + } if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -143,7 +147,8 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c err = b.log.Parse(file, b.ignoreSkipped) if err != nil { - progress <- models.Progress{}.Complete() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -157,11 +162,13 @@ func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results c } } sort.Sort(listens) - progress <- models.Progress{Total: int64(len(listens))}.Complete() + p.Export.Total = int64(len(listens)) + p.Export.Complete() + progress <- p results <- models.ListensResult{Items: listens} } -func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.Progress) (models.ImportResult, error) { +func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { records := make([]scrobblerlog.Record, len(export.Items)) for i, listen := range export.Items { records[i] = listenToRecord(listen) @@ -173,7 +180,7 @@ func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importR importResult.UpdateTimestamp(lastTimestamp) importResult.ImportCount += len(export.Items) - progress <- models.Progress{}.FromImportResult(importResult) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index be48dfe..5d45087 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -95,18 +95,22 @@ func (b *SpotifyApiBackend) OAuth2Setup(token oauth2.TokenSource) error { return nil } -func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { startTime := time.Now() minTime := oldestTimestamp totalDuration := startTime.Sub(oldestTimestamp) - - p := models.Progress{Total: int64(totalDuration.Seconds())} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } for { result, err := b.client.RecentlyPlayedAfter(minTime, MaxItemsPerGet) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -118,7 +122,8 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha // Set minTime to the newest returned listen after, err := strconv.ParseInt(result.Cursors.After, 10, 64) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } else if after <= minTime.Unix() { @@ -146,22 +151,28 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha } sort.Sort(listens) - p.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + p.Export.TotalItems += len(listens) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) progress <- p results <- models.ListensResult{Items: listens, OldestTimestamp: minTime} } results <- models.ListensResult{OldestTimestamp: minTime} - progress <- p.Complete() + p.Export.Complete() + progress <- p } -func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 perPage := MaxItemsPerGet - p := models.Progress{Total: int64(perPage)} + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(perPage), + }, + } totalCount := 0 exportCount := 0 @@ -169,7 +180,8 @@ out: for { result, err := b.client.UserTracks(offset, perPage) if err != nil { - progress <- p.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } @@ -177,7 +189,7 @@ out: // The offset was higher then the actual number of tracks. Adjust the offset // and continue. if offset >= result.Total { - p.Total = int64(result.Total) + p.Export.Total = int64(result.Total) totalCount = result.Total offset = max(result.Total-perPage, 0) continue @@ -201,7 +213,7 @@ out: exportCount += len(loves) sort.Sort(loves) results <- models.LovesResult{Items: loves, Total: totalCount} - p.Elapsed += int64(count) + p.Export.Elapsed += int64(count) progress <- p if offset <= 0 { @@ -216,7 +228,8 @@ out: } results <- models.LovesResult{Total: exportCount} - progress <- p.Complete() + p.Export.Complete() + progress <- p } func (l Listen) AsListen() models.Listen { diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 23309f3..d5c87bb 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -72,21 +72,27 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.Progress) { +func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) + p := models.TransferProgress{ + Export: &models.Progress{}, + } + if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } slices.Sort(files) fileCount := int64(len(files)) - p := models.Progress{Total: fileCount} + p.Export.Total = fileCount for i, filePath := range files { history, err := readHistoryFile(filePath) if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.ListensResult{Error: err} return } @@ -97,11 +103,13 @@ func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results }) sort.Sort(listens) results <- models.ListensResult{Items: listens} - p.Elapsed = int64(i) + p.Export.Elapsed = int64(i) + p.Export.TotalItems += len(listens) progress <- p } - progress <- p.Complete() + p.Export.Complete() + progress <- p } func readHistoryFile(filePath string) (StreamingHistory, error) { diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index 370765e..2098688 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -63,25 +63,30 @@ func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.Progress) { +func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { err := b.client.Authenticate(b.password) + p := models.TransferProgress{ + Export: &models.Progress{}, + } if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } starred, err := b.client.GetStarred2(map[string]string{}) if err != nil { - progress <- models.Progress{}.Abort() + p.Export.Abort() + progress <- p results <- models.LovesResult{Error: err} return } loves := b.filterSongs(starred.Song, oldestTimestamp) - progress <- models.Progress{ - Total: int64(loves.Len()), - }.Complete() + p.Export.Total = int64(len(loves)) + p.Export.Complete() + progress <- p results <- models.LovesResult{Items: loves} } diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 88339ef..b640815 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -28,7 +28,18 @@ import ( "go.uploadedlobster.com/scotty/internal/models" ) -func progressBar(exportProgress chan models.Progress, importProgress chan models.Progress) *mpb.Progress { +type progressBarUpdater struct { + wg *sync.WaitGroup + progress *mpb.Progress + exportBar *mpb.Bar + importBar *mpb.Bar + updateChan chan models.TransferProgress + lastExportUpdate time.Time + totalItems int + importedItems int +} + +func setupProgressBars(updateChan chan models.TransferProgress) progressBarUpdater { wg := &sync.WaitGroup{} p := mpb.New( mpb.WithWaitGroup(wg), @@ -37,15 +48,71 @@ func progressBar(exportProgress chan models.Progress, importProgress chan models mpb.WithAutoRefresh(), ) - exportBar := setupProgressBar(p, i18n.Tr("exporting")) - importBar := setupProgressBar(p, i18n.Tr("importing")) - go updateProgressBar(exportBar, wg, exportProgress) - go updateProgressBar(importBar, wg, importProgress) + u := progressBarUpdater{ + wg: wg, + progress: p, + exportBar: initProgressBar(p, i18n.Tr("exporting")), + importBar: initProgressBar(p, i18n.Tr("importing")), + updateChan: updateChan, + } - return p + go u.update() + return u } -func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar { +func (u *progressBarUpdater) wait() { + // FIXME: This should probably be closed elsewhere + close(u.updateChan) + u.progress.Wait() +} + +func (u *progressBarUpdater) update() { + u.wg.Add(1) + defer u.wg.Done() + u.lastExportUpdate = time.Now() + for progress := range u.updateChan { + if progress.Export != nil { + u.updateExportProgress(progress.Export) + } + + if progress.Import != nil { + if int64(u.totalItems) > progress.Import.Total { + progress.Import.Total = int64(u.totalItems) + } + u.updateImportProgress(progress.Import) + } + } +} + +func (u *progressBarUpdater) updateExportProgress(progress *models.Progress) { + bar := u.exportBar + u.totalItems = progress.TotalItems + + if progress.Aborted { + bar.Abort(false) + return + } + + oldIterTime := u.lastExportUpdate + u.lastExportUpdate = time.Now() + elapsedTime := u.lastExportUpdate.Sub(oldIterTime) + bar.EwmaSetCurrent(progress.Elapsed, elapsedTime) + bar.SetTotal(progress.Total, progress.Completed) +} + +func (u *progressBarUpdater) updateImportProgress(progress *models.Progress) { + bar := u.importBar + + if progress.Aborted { + bar.Abort(false) + return + } + + bar.SetCurrent(progress.Elapsed) + bar.SetTotal(progress.Total, progress.Completed) +} + +func initProgressBar(p *mpb.Progress, name string) *mpb.Bar { green := color.New(color.FgGreen).SprintFunc() return p.New(0, mpb.BarStyle(), @@ -69,19 +136,3 @@ func setupProgressBar(p *mpb.Progress, name string) *mpb.Bar { ), ) } - -func updateProgressBar(bar *mpb.Bar, wg *sync.WaitGroup, progressChan chan models.Progress) { - wg.Add(1) - defer wg.Done() - lastIterTime := time.Now() - for progress := range progressChan { - if progress.Aborted { - bar.Abort(false) - return - } - oldIterTime := lastIterTime - lastIterTime = time.Now() - bar.EwmaSetCurrent(progress.Elapsed, lastIterTime.Sub(oldIterTime)) - bar.SetTotal(progress.Total, progress.Completed) - } -} diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 0391d0d..b16c590 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -109,19 +109,18 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac printTimestamp("From timestamp: %v (%v)", timestamp) // Prepare progress bars - exportProgress := make(chan models.Progress) - importProgress := make(chan models.Progress) - progress := progressBar(exportProgress, importProgress) + progressChan := make(chan models.TransferProgress) + progress := setupProgressBars(progressChan) // Export from source exportChan := make(chan R, 1000) - go exp.Process(timestamp, exportChan, exportProgress) + go exp.Process(timestamp, exportChan, progressChan) // Import into target resultChan := make(chan models.ImportResult) - go imp.Process(exportChan, resultChan, importProgress) + go imp.Process(exportChan, resultChan, progressChan) result := <-resultChan - progress.Wait() + progress.wait() // Update timestamp err = c.updateTimestamp(&result, timestamp) diff --git a/internal/models/interfaces.go b/internal/models/interfaces.go index 1c593d0..bb97dac 100644 --- a/internal/models/interfaces.go +++ b/internal/models/interfaces.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer This file is part of Scotty. @@ -55,7 +55,7 @@ type ListensExport interface { // Returns a list of all listens newer then oldestTimestamp. // The returned list of listens is supposed to be ordered by the // Listen.ListenedAt timestamp, with the oldest entry first. - ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan Progress) + ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan TransferProgress) } // Must be implemented by services supporting the import of listens. @@ -63,7 +63,7 @@ type ListensImport interface { ImportBackend // Imports the given list of listens. - ImportListens(export ListensResult, importResult ImportResult, progress chan Progress) (ImportResult, error) + ImportListens(export ListensResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error) } // Must be implemented by services supporting the export of loves. @@ -73,7 +73,7 @@ type LovesExport interface { // Returns a list of all loves newer then oldestTimestamp. // The returned list of listens is supposed to be ordered by the // Love.Created timestamp, with the oldest entry first. - ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan Progress) + ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan TransferProgress) } // Must be implemented by services supporting the import of loves. @@ -81,5 +81,5 @@ type LovesImport interface { ImportBackend // Imports the given list of loves. - ImportLoves(export LovesResult, importResult ImportResult, progress chan Progress) (ImportResult, error) + ImportLoves(export LovesResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error) } diff --git a/internal/models/models.go b/internal/models/models.go index b8f9121..081266d 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -209,11 +209,25 @@ func (i *ImportResult) Log(t LogEntryType, msg string) { }) } +type TransferProgress struct { + Export *Progress + Import *Progress +} + +func (p TransferProgress) FromImportResult(result ImportResult, completed bool) TransferProgress { + importProgress := Progress{ + Completed: completed, + }.FromImportResult(result) + p.Import = &importProgress + return p +} + type Progress struct { - Total int64 - Elapsed int64 - Completed bool - Aborted bool + TotalItems int + Total int64 + Elapsed int64 + Completed bool + Aborted bool } func (p Progress) FromImportResult(result ImportResult) Progress { @@ -222,13 +236,11 @@ func (p Progress) FromImportResult(result ImportResult) Progress { return p } -func (p Progress) Complete() Progress { +func (p *Progress) Complete() { p.Elapsed = p.Total p.Completed = true - return p } -func (p Progress) Abort() Progress { +func (p *Progress) Abort() { p.Aborted = true - return p } From 17cee9cb8bf852a4c11d41dec75c384475e3ddb8 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 5 May 2025 17:39:47 +0200 Subject: [PATCH 23/77] For import progress show actually processed and total count --- internal/cli/progress.go | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/internal/cli/progress.go b/internal/cli/progress.go index b640815..6696226 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -51,8 +51,8 @@ func setupProgressBars(updateChan chan models.TransferProgress) progressBarUpdat u := progressBarUpdater{ wg: wg, progress: p, - exportBar: initProgressBar(p, i18n.Tr("exporting")), - importBar: initProgressBar(p, i18n.Tr("importing")), + exportBar: initExportProgressBar(p, i18n.Tr("exporting")), + importBar: initImportProgressBar(p, i18n.Tr("importing")), updateChan: updateChan, } @@ -112,8 +112,18 @@ func (u *progressBarUpdater) updateImportProgress(progress *models.Progress) { bar.SetTotal(progress.Total, progress.Completed) } -func initProgressBar(p *mpb.Progress, name string) *mpb.Bar { +func initExportProgressBar(p *mpb.Progress, name string) *mpb.Bar { + return initProgressBar(p, name, + decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth})) +} + +func initImportProgressBar(p *mpb.Progress, name string) *mpb.Bar { + return initProgressBar(p, name, decor.Counters(0, "%d / %d")) +} + +func initProgressBar(p *mpb.Progress, name string, progressDecorator decor.Decorator) *mpb.Bar { green := color.New(color.FgGreen).SprintFunc() + red := color.New(color.FgHiRed, color.Bold).SprintFunc() return p.New(0, mpb.BarStyle(), mpb.PrependDecorators( @@ -127,8 +137,8 @@ func initProgressBar(p *mpb.Progress, name string) *mpb.Bar { mpb.AppendDecorators( decor.OnComplete( decor.OnAbort( - decor.EwmaETA(decor.ET_STYLE_GO, 0, decor.WC{C: decor.DSyncWidth}), - i18n.Tr("aborted"), + progressDecorator, + red(i18n.Tr("aborted")), ), i18n.Tr("done"), ), From a87c42059f48e750aa11adb28af586f341a9be7e Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 5 May 2025 17:49:44 +0200 Subject: [PATCH 24/77] Use a WaitGroup to wait for both export and import goroutine to finish --- internal/backends/export.go | 11 ++++++++--- internal/backends/import.go | 16 ++++++++++------ internal/cli/progress.go | 3 +-- internal/cli/transfer.go | 10 +++++++--- 4 files changed, 26 insertions(+), 14 deletions(-) diff --git a/internal/backends/export.go b/internal/backends/export.go index c7a1f58..54daafb 100644 --- a/internal/backends/export.go +++ b/internal/backends/export.go @@ -16,6 +16,7 @@ Scotty. If not, see . package backends import ( + "sync" "time" "go.uploadedlobster.com/scotty/internal/models" @@ -23,7 +24,7 @@ import ( type ExportProcessor[T models.ListensResult | models.LovesResult] interface { ExportBackend() models.Backend - Process(oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress) + Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress) } type ListensExportProcessor struct { @@ -34,7 +35,9 @@ func (p ListensExportProcessor) ExportBackend() models.Backend { return p.Backend } -func (p ListensExportProcessor) Process(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { +func (p ListensExportProcessor) Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + wg.Add(1) + defer wg.Done() defer close(results) p.Backend.ExportListens(oldestTimestamp, results, progress) } @@ -47,7 +50,9 @@ func (p LovesExportProcessor) ExportBackend() models.Backend { return p.Backend } -func (p LovesExportProcessor) Process(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { +func (p LovesExportProcessor) Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + wg.Add(1) + defer wg.Done() defer close(results) p.Backend.ExportLoves(oldestTimestamp, results, progress) } diff --git a/internal/backends/import.go b/internal/backends/import.go index d3b86ac..0a2e341 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -18,12 +18,14 @@ Scotty. If not, see . package backends import ( + "sync" + "go.uploadedlobster.com/scotty/internal/models" ) type ImportProcessor[T models.ListensResult | models.LovesResult] interface { ImportBackend() models.ImportBackend - Process(results chan T, out chan models.ImportResult, progress chan models.TransferProgress) + Process(wg *sync.WaitGroup, results chan T, out chan models.ImportResult, progress chan models.TransferProgress) Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) } @@ -35,8 +37,8 @@ func (p ListensImportProcessor) ImportBackend() models.ImportBackend { return p.Backend } -func (p ListensImportProcessor) Process(results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) { - process(p, results, out, progress) +func (p ListensImportProcessor) Process(wg *sync.WaitGroup, results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) { + process(wg, p, results, out, progress) } func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { @@ -64,8 +66,8 @@ func (p LovesImportProcessor) ImportBackend() models.ImportBackend { return p.Backend } -func (p LovesImportProcessor) Process(results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) { - process(p, results, out, progress) +func (p LovesImportProcessor) Process(wg *sync.WaitGroup, results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) { + process(wg, p, results, out, progress) } func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { @@ -85,7 +87,9 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im return importResult, nil } -func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](processor P, results chan R, out chan models.ImportResult, progress chan models.TransferProgress) { +func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](wg *sync.WaitGroup, processor P, results chan R, out chan models.ImportResult, progress chan models.TransferProgress) { + wg.Add(1) + defer wg.Done() defer close(out) result := models.ImportResult{} p := models.TransferProgress{} diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 6696226..e93ec18 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -60,8 +60,7 @@ func setupProgressBars(updateChan chan models.TransferProgress) progressBarUpdat return u } -func (u *progressBarUpdater) wait() { - // FIXME: This should probably be closed elsewhere +func (u *progressBarUpdater) close() { close(u.updateChan) u.progress.Wait() } diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index b16c590..62dd079 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "strconv" + "sync" "time" "github.com/spf13/cobra" @@ -112,15 +113,18 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac progressChan := make(chan models.TransferProgress) progress := setupProgressBars(progressChan) + wg := &sync.WaitGroup{} + // Export from source exportChan := make(chan R, 1000) - go exp.Process(timestamp, exportChan, progressChan) + go exp.Process(wg, timestamp, exportChan, progressChan) // Import into target resultChan := make(chan models.ImportResult) - go imp.Process(exportChan, resultChan, progressChan) + go imp.Process(wg, exportChan, resultChan, progressChan) result := <-resultChan - progress.wait() + wg.Wait() + progress.close() // Update timestamp err = c.updateTimestamp(&result, timestamp) From a42b5d784df19c77dba9352e9b932d24487174b4 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 8 May 2025 07:40:12 +0200 Subject: [PATCH 25/77] Added short doc string to ratelimit package --- pkg/ratelimit/httpheader.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/ratelimit/httpheader.go b/pkg/ratelimit/httpheader.go index 3f2552c..dba5e30 100644 --- a/pkg/ratelimit/httpheader.go +++ b/pkg/ratelimit/httpheader.go @@ -13,6 +13,7 @@ You should have received a copy of the GNU General Public License along with Scotty. If not, see . */ +// Helper functions to set up rate limiting with resty. package ratelimit import ( From 97600d8190f80d0125a83849ad88e533433e8ef3 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 9 May 2025 07:38:28 +0200 Subject: [PATCH 26/77] Update dependencies --- go.mod | 20 ++++++++++---------- go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index 8ed329b..47c7e88 100644 --- a/go.mod +++ b/go.mod @@ -23,11 +23,11 @@ require ( github.com/vbauerster/mpb/v8 v8.10.0 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.14.0 - golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 - golang.org/x/oauth2 v0.29.0 - golang.org/x/text v0.24.0 + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 + golang.org/x/oauth2 v0.30.0 + golang.org/x/text v0.25.0 gorm.io/datatypes v1.2.5 - gorm.io/gorm v1.26.0 + gorm.io/gorm v1.26.1 ) require ( @@ -58,15 +58,15 @@ require ( github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/image v0.26.0 // indirect + golang.org/x/image v0.27.0 // indirect golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.39.0 // indirect - golang.org/x/sync v0.13.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/tools v0.32.0 // indirect + golang.org/x/net v0.40.0 // indirect + golang.org/x/sync v0.14.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/tools v0.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.7 // indirect - modernc.org/libc v1.65.0 // indirect + modernc.org/libc v1.65.2 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.10.0 // indirect modernc.org/sqlite v1.37.0 // indirect diff --git a/go.sum b/go.sum index 4d3cb1b..426e8c0 100644 --- a/go.sum +++ b/go.sum @@ -136,13 +136,13 @@ go.uploadedlobster.com/musicbrainzws2 v0.14.0 h1:YaEtxNwLSNT1gzFipQ4XlaThNfXjBpz go.uploadedlobster.com/musicbrainzws2 v0.14.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= -golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= -golang.org/x/image v0.26.0 h1:4XjIFEZWQmCZi6Wv8BoxsDhRU3RVnLX04dToTDAEPlY= -golang.org/x/image v0.26.0/go.mod h1:lcxbMFAovzpnJxzXS3nyL83K27tmqtKzIJpctK8YO5c= +golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w= +golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= @@ -151,15 +151,15 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= -golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= -golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= -golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= -golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= +golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -169,8 +169,8 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -179,16 +179,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= -golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -206,18 +206,18 @@ gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2e gorm.io/driver/sqlserver v1.5.4 h1:xA+Y1KDNspv79q43bPyjDMUgHoYHLhXYmdFcYPobg8g= gorm.io/driver/sqlserver v1.5.4/go.mod h1:+frZ/qYmuna11zHPlh5oc2O6ZA/lS88Keb0XSH1Zh/g= gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.26.0 h1:9lqQVPG5aNNS6AyHdRiwScAVnXHg/L/Srzx55G5fOgs= -gorm.io/gorm v1.26.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= -modernc.org/cc/v4 v4.26.0 h1:QMYvbVduUGH0rrO+5mqF/PSPPRZNpRtg2CLELy7vUpA= -modernc.org/cc/v4 v4.26.0/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.26.0 h1:gVzXaDzGeBYJ2uXTOpR8FR7OlksDOe9jxnjhIKCsiTc= -modernc.org/ccgo/v4 v4.26.0/go.mod h1:Sem8f7TFUtVXkG2fiaChQtyyfkqhJBg/zjEJBkmuAVY= +gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw= +gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= +modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.27.1 h1:emhLB4uoOmkZUnTDFcMI3AbkmU/Evjuerit9Taqe6Ss= +modernc.org/ccgo/v4 v4.27.1/go.mod h1:543Q0qQhJWekKVS5P6yL5fO6liNhla9Lbm2/B3rEKDE= modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.65.0 h1:e183gLDnAp9VJh6gWKdTy0CThL9Pt7MfcR/0bgb7Y1Y= -modernc.org/libc v1.65.0/go.mod h1:7m9VzGq7APssBTydds2zBcxGREwvIGpuUBaKTXdm2Qs= +modernc.org/libc v1.65.2 h1:drWL1QO9fKXr3kXDN8y+4lKyBr8bA3mtUBQpftq3IJw= +modernc.org/libc v1.65.2/go.mod h1:VI3V2S5mNka4deJErQ0jsMXe7jgxojE2fOB/mWoHlbc= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4= From 536fae6a46ffdb6b76a6034fdbab9ffe14e4b81f Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 11:51:23 +0200 Subject: [PATCH 27/77] ScrobblerLog.ReadHeader now accepts io.Reader --- internal/backends/scrobblerlog/scrobblerlog.go | 4 +--- pkg/scrobblerlog/parser.go | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index db4e349..6454b7b 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -17,7 +17,6 @@ Scotty. If not, see . package scrobblerlog import ( - "bufio" "fmt" "os" "sort" @@ -105,8 +104,7 @@ func (b *ScrobblerLogBackend) StartImport() error { b.append = false } else { // Verify existing file is a scrobbler log - reader := bufio.NewReader(file) - if err = b.log.ReadHeader(reader); err != nil { + if err = b.log.ReadHeader(file); err != nil { file.Close() return err } diff --git a/pkg/scrobblerlog/parser.go b/pkg/scrobblerlog/parser.go index 6b9d1ba..8bad56d 100644 --- a/pkg/scrobblerlog/parser.go +++ b/pkg/scrobblerlog/parser.go @@ -94,7 +94,7 @@ func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { l.Records = make([]Record, 0) reader := bufio.NewReader(data) - err := l.ReadHeader(reader) + err := l.readHeader(reader) if err != nil { return err } @@ -173,7 +173,11 @@ func (l *ScrobblerLog) Append(data io.Writer, records []Record) (lastTimestamp t // Parses just the header of a scrobbler log file from the given reader. // // This function sets [ScrobblerLog.TZ] and [ScrobblerLog.Client]. -func (l *ScrobblerLog) ReadHeader(reader *bufio.Reader) error { +func (l *ScrobblerLog) ReadHeader(reader io.Reader) error { + return l.readHeader(bufio.NewReader(reader)) +} + +func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { // Skip header for i := 0; i < 3; i++ { line, _, err := reader.ReadLine() From 3b545a0fd6548bdcc89c5e096d013ba35eacd4fa Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 08:48:37 +0200 Subject: [PATCH 28/77] Prepare using a context for export / import This will allow cancelling the export if the import fails before the export finished. For now the context isn't passed on to the actual export functions, hence there is not yet any cancellation happening. --- internal/backends/export.go | 7 ++++--- internal/backends/import.go | 35 ++++++++++++++++++++++++----------- internal/cli/transfer.go | 11 +++++++++-- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/internal/backends/export.go b/internal/backends/export.go index 54daafb..0deebc6 100644 --- a/internal/backends/export.go +++ b/internal/backends/export.go @@ -16,6 +16,7 @@ Scotty. If not, see . package backends import ( + "context" "sync" "time" @@ -24,7 +25,7 @@ import ( type ExportProcessor[T models.ListensResult | models.LovesResult] interface { ExportBackend() models.Backend - Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress) + Process(ctx context.Context, wg *sync.WaitGroup, oldestTimestamp time.Time, results chan T, progress chan models.TransferProgress) } type ListensExportProcessor struct { @@ -35,7 +36,7 @@ func (p ListensExportProcessor) ExportBackend() models.Backend { return p.Backend } -func (p ListensExportProcessor) Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { +func (p ListensExportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { wg.Add(1) defer wg.Done() defer close(results) @@ -50,7 +51,7 @@ func (p LovesExportProcessor) ExportBackend() models.Backend { return p.Backend } -func (p LovesExportProcessor) Process(wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { +func (p LovesExportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { wg.Add(1) defer wg.Done() defer close(results) diff --git a/internal/backends/import.go b/internal/backends/import.go index 0a2e341..e7006bd 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -18,6 +18,7 @@ Scotty. If not, see . package backends import ( + "context" "sync" "go.uploadedlobster.com/scotty/internal/models" @@ -25,7 +26,7 @@ import ( type ImportProcessor[T models.ListensResult | models.LovesResult] interface { ImportBackend() models.ImportBackend - Process(wg *sync.WaitGroup, results chan T, out chan models.ImportResult, progress chan models.TransferProgress) + Process(ctx context.Context, wg *sync.WaitGroup, results chan T, out chan models.ImportResult, progress chan models.TransferProgress) Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) } @@ -37,8 +38,8 @@ func (p ListensImportProcessor) ImportBackend() models.ImportBackend { return p.Backend } -func (p ListensImportProcessor) Process(wg *sync.WaitGroup, results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) { - process(wg, p, results, out, progress) +func (p ListensImportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, results chan models.ListensResult, out chan models.ImportResult, progress chan models.TransferProgress) { + process(ctx, wg, p, results, out, progress) } func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { @@ -66,8 +67,8 @@ func (p LovesImportProcessor) ImportBackend() models.ImportBackend { return p.Backend } -func (p LovesImportProcessor) Process(wg *sync.WaitGroup, results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) { - process(wg, p, results, out, progress) +func (p LovesImportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, results chan models.LovesResult, out chan models.ImportResult, progress chan models.TransferProgress) { + process(ctx, wg, p, results, out, progress) } func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { @@ -87,7 +88,12 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im return importResult, nil } -func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]](wg *sync.WaitGroup, processor P, results chan R, out chan models.ImportResult, progress chan models.TransferProgress) { +func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( + ctx context.Context, wg *sync.WaitGroup, + processor P, results chan R, + out chan models.ImportResult, + progress chan models.TransferProgress, +) { wg.Add(1) defer wg.Done() defer close(out) @@ -100,14 +106,21 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( } for exportResult := range results { - importResult, err := processor.Import(exportResult, result, out, progress) - result.Update(importResult) - if err != nil { + select { + case <-ctx.Done(): processor.ImportBackend().FinishImport() - out <- handleError(result, err, progress) + out <- handleError(result, ctx.Err(), progress) return + default: + importResult, err := processor.Import(exportResult, result, out, progress) + result.Update(importResult) + if err != nil { + processor.ImportBackend().FinishImport() + out <- handleError(result, err, progress) + return + } + progress <- p.FromImportResult(result, false) } - progress <- p.FromImportResult(result, false) } if err := processor.ImportBackend().FinishImport(); err != nil { diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 62dd079..79be3f0 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -16,6 +16,7 @@ Scotty. If not, see . package cli import ( + "context" "errors" "fmt" "strconv" @@ -113,16 +114,22 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac progressChan := make(chan models.TransferProgress) progress := setupProgressBars(progressChan) + ctx, cancel := context.WithCancel(context.Background()) wg := &sync.WaitGroup{} // Export from source exportChan := make(chan R, 1000) - go exp.Process(wg, timestamp, exportChan, progressChan) + go exp.Process(ctx, wg, timestamp, exportChan, progressChan) // Import into target resultChan := make(chan models.ImportResult) - go imp.Process(wg, exportChan, resultChan, progressChan) + go imp.Process(ctx, wg, exportChan, resultChan, progressChan) result := <-resultChan + + // Once import is done, the context can be cancelled + cancel() + + // Wait for all goroutines to finish wg.Wait() progress.close() From adfe3f5771f933bcb6012d24aa53a436370be677 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 08:52:52 +0200 Subject: [PATCH 29/77] Use the transfer context also for the progress bars --- internal/cli/progress.go | 6 ++++-- internal/cli/transfer.go | 14 ++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/cli/progress.go b/internal/cli/progress.go index e93ec18..db862a1 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -18,6 +18,7 @@ Scotty. If not, see . package cli import ( + "context" "sync" "time" @@ -39,9 +40,10 @@ type progressBarUpdater struct { importedItems int } -func setupProgressBars(updateChan chan models.TransferProgress) progressBarUpdater { +func setupProgressBars(ctx context.Context, updateChan chan models.TransferProgress) progressBarUpdater { wg := &sync.WaitGroup{} - p := mpb.New( + p := mpb.NewWithContext( + ctx, mpb.WithWaitGroup(wg), mpb.WithOutput(color.Output), // mpb.WithWidth(64), diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 79be3f0..3aabb4b 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -110,11 +110,13 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac } printTimestamp("From timestamp: %v (%v)", timestamp) + // Use a context with cancel to abort the transfer + ctx, cancel := context.WithCancel(context.Background()) + // Prepare progress bars progressChan := make(chan models.TransferProgress) - progress := setupProgressBars(progressChan) + progress := setupProgressBars(ctx, progressChan) - ctx, cancel := context.WithCancel(context.Background()) wg := &sync.WaitGroup{} // Export from source @@ -126,8 +128,12 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac go imp.Process(ctx, wg, exportChan, resultChan, progressChan) result := <-resultChan - // Once import is done, the context can be cancelled - cancel() + // If the import has errored, the context can be cancelled immediately + if result.Error != nil { + cancel() + } else { + defer cancel() + } // Wait for all goroutines to finish wg.Wait() From d1642b7f1f675a0eb72e2e60aaa2049cd8b55b87 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 09:22:05 +0200 Subject: [PATCH 30/77] Make web service clients context aware --- internal/backends/deezer/client.go | 14 ++++---- internal/backends/deezer/client_test.go | 9 +++-- internal/backends/deezer/deezer.go | 9 +++-- internal/backends/funkwhale/client.go | 33 ++++++++++--------- internal/backends/funkwhale/client_test.go | 9 +++-- internal/backends/funkwhale/funkwhale.go | 7 ++-- internal/backends/listenbrainz/client.go | 18 ++++++---- internal/backends/listenbrainz/client_test.go | 20 +++++++---- .../backends/listenbrainz/listenbrainz.go | 27 ++++++++------- internal/backends/maloja/client.go | 9 +++-- internal/backends/maloja/client_test.go | 9 +++-- internal/backends/maloja/maloja.go | 8 +++-- internal/backends/spotify/client.go | 16 +++++---- internal/backends/spotify/client_test.go | 9 +++-- internal/backends/spotify/spotify.go | 7 ++-- 15 files changed, 128 insertions(+), 76 deletions(-) diff --git a/internal/backends/deezer/client.go b/internal/backends/deezer/client.go index 05264ae..3ab2b6c 100644 --- a/internal/backends/deezer/client.go +++ b/internal/backends/deezer/client.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,7 @@ THE SOFTWARE. package deezer import ( + "context" "errors" "strconv" @@ -52,14 +53,14 @@ func NewClient(token oauth2.TokenSource) Client { } } -func (c Client) UserHistory(offset int, limit int) (result HistoryResult, err error) { +func (c Client) UserHistory(ctx context.Context, offset int, limit int) (result HistoryResult, err error) { const path = "/user/me/history" - return listRequest[HistoryResult](c, path, offset, limit) + return listRequest[HistoryResult](ctx, c, path, offset, limit) } -func (c Client) UserTracks(offset int, limit int) (TracksResult, error) { +func (c Client) UserTracks(ctx context.Context, offset int, limit int) (TracksResult, error) { const path = "/user/me/tracks" - return listRequest[TracksResult](c, path, offset, limit) + return listRequest[TracksResult](ctx, c, path, offset, limit) } func (c Client) setToken(req *resty.Request) error { @@ -72,8 +73,9 @@ func (c Client) setToken(req *resty.Request) error { return nil } -func listRequest[T Result](c Client, path string, offset int, limit int) (result T, err error) { +func listRequest[T Result](ctx context.Context, c Client, path string, offset int, limit int) (result T, err error) { request := c.HTTPClient.R(). + SetContext(ctx). SetQueryParams(map[string]string{ "index": strconv.Itoa(offset), "limit": strconv.Itoa(limit), diff --git a/internal/backends/deezer/client_test.go b/internal/backends/deezer/client_test.go index c90b01a..8b61804 100644 --- a/internal/backends/deezer/client_test.go +++ b/internal/backends/deezer/client_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23,6 +23,7 @@ THE SOFTWARE. package deezer_test import ( + "context" "net/http" "testing" @@ -48,7 +49,8 @@ func TestGetUserHistory(t *testing.T) { "https://api.deezer.com/user/me/history", "testdata/user-history.json") - result, err := client.UserHistory(0, 2) + ctx := context.Background() + result, err := client.UserHistory(ctx, 0, 2) require.NoError(t, err) assert := assert.New(t) @@ -69,7 +71,8 @@ func TestGetUserTracks(t *testing.T) { "https://api.deezer.com/user/me/tracks", "testdata/user-tracks.json") - result, err := client.UserTracks(0, 2) + ctx := context.Background() + result, err := client.UserTracks(ctx, 0, 2) require.NoError(t, err) assert := assert.New(t) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index 2209769..f3e3d37 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -16,6 +16,7 @@ Scotty. If not, see . package deezer import ( + "context" "fmt" "math" "net/url" @@ -78,6 +79,8 @@ func (b *DeezerApiBackend) OAuth2Setup(token oauth2.TokenSource) error { } func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + ctx := context.TODO() + // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 @@ -96,7 +99,7 @@ func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan out: for { - result, err := b.client.UserHistory(offset, perPage) + result, err := b.client.UserHistory(ctx, offset, perPage) if err != nil { p.Export.Abort() progress <- p @@ -154,6 +157,8 @@ out: } func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + ctx := context.TODO() + // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 @@ -168,7 +173,7 @@ func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan m out: for { - result, err := b.client.UserTracks(offset, perPage) + result, err := b.client.UserTracks(ctx, offset, perPage) if err != nil { p.Export.Abort() progress <- p diff --git a/internal/backends/funkwhale/client.go b/internal/backends/funkwhale/client.go index c231c94..3471612 100644 --- a/internal/backends/funkwhale/client.go +++ b/internal/backends/funkwhale/client.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package funkwhale import ( + "context" "errors" "strconv" @@ -54,15 +55,10 @@ func NewClient(serverURL string, token string) Client { } } -func (c Client) GetHistoryListenings(user string, page int, perPage int) (result ListeningsResult, err error) { +func (c Client) GetHistoryListenings(ctx context.Context, user string, page int, perPage int) (result ListeningsResult, err error) { const path = "/api/v1/history/listenings" - response, err := c.HTTPClient.R(). - SetQueryParams(map[string]string{ - "username": user, - "page": strconv.Itoa(page), - "page_size": strconv.Itoa(perPage), - "ordering": "-creation_date", - }). + response, err := c.buildListRequest(ctx, page, perPage). + SetQueryParam("username", user). SetResult(&result). Get(path) @@ -73,14 +69,9 @@ func (c Client) GetHistoryListenings(user string, page int, perPage int) (result return } -func (c Client) GetFavoriteTracks(page int, perPage int) (result FavoriteTracksResult, err error) { +func (c Client) GetFavoriteTracks(ctx context.Context, page int, perPage int) (result FavoriteTracksResult, err error) { const path = "/api/v1/favorites/tracks" - response, err := c.HTTPClient.R(). - SetQueryParams(map[string]string{ - "page": strconv.Itoa(page), - "page_size": strconv.Itoa(perPage), - "ordering": "-creation_date", - }). + response, err := c.buildListRequest(ctx, page, perPage). SetResult(&result). Get(path) @@ -90,3 +81,13 @@ func (c Client) GetFavoriteTracks(page int, perPage int) (result FavoriteTracksR } return } + +func (c Client) buildListRequest(ctx context.Context, page int, perPage int) *resty.Request { + return c.HTTPClient.R(). + SetContext(ctx). + SetQueryParams(map[string]string{ + "page": strconv.Itoa(page), + "page_size": strconv.Itoa(perPage), + "ordering": "-creation_date", + }) +} diff --git a/internal/backends/funkwhale/client_test.go b/internal/backends/funkwhale/client_test.go index e850a4d..d6b04e0 100644 --- a/internal/backends/funkwhale/client_test.go +++ b/internal/backends/funkwhale/client_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package funkwhale_test import ( + "context" "net/http" "testing" @@ -49,7 +50,8 @@ func TestGetHistoryListenings(t *testing.T) { "https://funkwhale.example.com/api/v1/history/listenings", "testdata/listenings.json") - result, err := client.GetHistoryListenings("outsidecontext", 0, 2) + ctx := context.Background() + result, err := client.GetHistoryListenings(ctx, "outsidecontext", 0, 2) require.NoError(t, err) assert := assert.New(t) @@ -73,7 +75,8 @@ func TestGetFavoriteTracks(t *testing.T) { "https://funkwhale.example.com/api/v1/favorites/tracks", "testdata/favorite-tracks.json") - result, err := client.GetFavoriteTracks(0, 2) + ctx := context.Background() + result, err := client.GetFavoriteTracks(ctx, 0, 2) require.NoError(t, err) assert := assert.New(t) diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index cd2f28e..434716f 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -17,6 +17,7 @@ Scotty. If not, see . package funkwhale import ( + "context" "sort" "time" @@ -61,6 +62,7 @@ func (b *FunkwhaleApiBackend) InitConfig(config *config.ServiceConfig) error { } func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + ctx := context.TODO() page := 1 perPage := MaxItemsPerGet @@ -74,7 +76,7 @@ func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results c out: for { - result, err := b.client.GetHistoryListenings(b.username, page, perPage) + result, err := b.client.GetHistoryListenings(ctx, b.username, page, perPage) if err != nil { p.Export.Abort() progress <- p @@ -118,6 +120,7 @@ out: } func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + ctx := context.TODO() page := 1 perPage := MaxItemsPerGet @@ -131,7 +134,7 @@ func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results cha out: for { - result, err := b.client.GetFavoriteTracks(page, perPage) + result, err := b.client.GetFavoriteTracks(ctx, page, perPage) if err != nil { p.Export.Abort() progress <- p diff --git a/internal/backends/listenbrainz/client.go b/internal/backends/listenbrainz/client.go index fff476c..d1a1fa6 100644 --- a/internal/backends/listenbrainz/client.go +++ b/internal/backends/listenbrainz/client.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package listenbrainz import ( + "context" "errors" "strconv" "time" @@ -60,10 +61,11 @@ func NewClient(token string) Client { } } -func (c Client) GetListens(user string, maxTime time.Time, minTime time.Time) (result GetListensResult, err error) { +func (c Client) GetListens(ctx context.Context, user string, maxTime time.Time, minTime time.Time) (result GetListensResult, err error) { const path = "/user/{username}/listens" errorResult := ErrorResult{} response, err := c.HTTPClient.R(). + SetContext(ctx). SetPathParam("username", user). SetQueryParams(map[string]string{ "max_ts": strconv.FormatInt(maxTime.Unix(), 10), @@ -81,10 +83,11 @@ func (c Client) GetListens(user string, maxTime time.Time, minTime time.Time) (r return } -func (c Client) SubmitListens(listens ListenSubmission) (result StatusResult, err error) { +func (c Client) SubmitListens(ctx context.Context, listens ListenSubmission) (result StatusResult, err error) { const path = "/submit-listens" errorResult := ErrorResult{} response, err := c.HTTPClient.R(). + SetContext(ctx). SetBody(listens). SetResult(&result). SetError(&errorResult). @@ -97,10 +100,11 @@ func (c Client) SubmitListens(listens ListenSubmission) (result StatusResult, er return } -func (c Client) GetFeedback(user string, status int, offset int) (result GetFeedbackResult, err error) { +func (c Client) GetFeedback(ctx context.Context, user string, status int, offset int) (result GetFeedbackResult, err error) { const path = "/feedback/user/{username}/get-feedback" errorResult := ErrorResult{} response, err := c.HTTPClient.R(). + SetContext(ctx). SetPathParam("username", user). SetQueryParams(map[string]string{ "status": strconv.Itoa(status), @@ -119,10 +123,11 @@ func (c Client) GetFeedback(user string, status int, offset int) (result GetFeed return } -func (c Client) SendFeedback(feedback Feedback) (result StatusResult, err error) { +func (c Client) SendFeedback(ctx context.Context, feedback Feedback) (result StatusResult, err error) { const path = "/feedback/recording-feedback" errorResult := ErrorResult{} response, err := c.HTTPClient.R(). + SetContext(ctx). SetBody(feedback). SetResult(&result). SetError(&errorResult). @@ -135,10 +140,11 @@ func (c Client) SendFeedback(feedback Feedback) (result StatusResult, err error) return } -func (c Client) Lookup(recordingName string, artistName string) (result LookupResult, err error) { +func (c Client) Lookup(ctx context.Context, recordingName string, artistName string) (result LookupResult, err error) { const path = "/metadata/lookup" errorResult := ErrorResult{} response, err := c.HTTPClient.R(). + SetContext(ctx). SetQueryParams(map[string]string{ "recording_name": recordingName, "artist_name": artistName, diff --git a/internal/backends/listenbrainz/client_test.go b/internal/backends/listenbrainz/client_test.go index 2e841ae..45bb0de 100644 --- a/internal/backends/listenbrainz/client_test.go +++ b/internal/backends/listenbrainz/client_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package listenbrainz_test import ( + "context" "net/http" "testing" "time" @@ -49,7 +50,9 @@ func TestGetListens(t *testing.T) { "https://api.listenbrainz.org/1/user/outsidecontext/listens", "testdata/listens.json") - result, err := client.GetListens("outsidecontext", time.Now(), time.Now().Add(-2*time.Hour)) + ctx := context.Background() + result, err := client.GetListens(ctx, "outsidecontext", + time.Now(), time.Now().Add(-2*time.Hour)) require.NoError(t, err) assert := assert.New(t) @@ -92,8 +95,8 @@ func TestSubmitListens(t *testing.T) { }, }, } - result, err := client.SubmitListens(listens) - require.NoError(t, err) + ctx := context.Background() + result, err := client.SubmitListens(ctx, listens) assert.Equal(t, "ok", result.Status) } @@ -107,7 +110,8 @@ func TestGetFeedback(t *testing.T) { "https://api.listenbrainz.org/1/feedback/user/outsidecontext/get-feedback", "testdata/feedback.json") - result, err := client.GetFeedback("outsidecontext", 1, 3) + ctx := context.Background() + result, err := client.GetFeedback(ctx, "outsidecontext", 1, 0) require.NoError(t, err) assert := assert.New(t) @@ -135,7 +139,8 @@ func TestSendFeedback(t *testing.T) { RecordingMBID: "c0a1fc94-5f04-4a5f-bc09-e5de0c49cd12", Score: 1, } - result, err := client.SendFeedback(feedback) + ctx := context.Background() + result, err := client.SendFeedback(ctx, feedback) require.NoError(t, err) assert.Equal(t, "ok", result.Status) @@ -149,7 +154,8 @@ func TestLookup(t *testing.T) { "https://api.listenbrainz.org/1/metadata/lookup", "testdata/lookup.json") - result, err := client.Lookup("Paradise Lost", "Say Just Words") + ctx := context.Background() + result, err := client.Lookup(ctx, "Paradise Lost", "Say Just Words") require.NoError(t, err) assert := assert.New(t) diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 61597d1..d622aff 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -17,6 +17,7 @@ Scotty. If not, see . package listenbrainz import ( + "context" "fmt" "sort" "time" @@ -73,6 +74,7 @@ func (b *ListenBrainzApiBackend) StartImport() error { return nil } func (b *ListenBrainzApiBackend) FinishImport() error { return nil } func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + ctx := context.TODO() startTime := time.Now() minTime := oldestTimestamp if minTime.Unix() < 1 { @@ -87,7 +89,7 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result } for { - result, err := b.client.GetListens(b.username, time.Now(), minTime) + result, err := b.client.GetListens(ctx, b.username, time.Now(), minTime) if err != nil { p.Export.Abort() progress <- p @@ -135,6 +137,7 @@ func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, result } func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + ctx := context.TODO() total := len(export.Items) p := models.TransferProgress{}.FromImportResult(importResult, false) for i := 0; i < total; i += MaxListensPerRequest { @@ -151,7 +154,7 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo for _, l := range listens { if b.checkDuplicates { - isDupe, err := b.checkDuplicateListen(l) + isDupe, err := b.checkDuplicateListen(ctx, l) p.Import.Elapsed += 1 progress <- p if err != nil { @@ -182,7 +185,7 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo } if len(submission.Payload) > 0 { - _, err := b.client.SubmitListens(submission) + _, err := b.client.SubmitListens(ctx, submission) if err != nil { return importResult, err } @@ -199,12 +202,13 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo } func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + ctx := context.TODO() exportChan := make(chan models.LovesResult) p := models.TransferProgress{ Export: &models.Progress{}, } - go b.exportLoves(oldestTimestamp, exportChan) + go b.exportLoves(ctx, oldestTimestamp, exportChan) for existingLoves := range exportChan { if existingLoves.Error != nil { p.Export.Abort() @@ -224,14 +228,14 @@ func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results progress <- p } -func (b *ListenBrainzApiBackend) exportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { +func (b *ListenBrainzApiBackend) exportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult) { offset := 0 defer close(results) loves := make(models.LovesList, 0, 2*MaxItemsPerGet) out: for { - result, err := b.client.GetFeedback(b.username, 1, offset) + result, err := b.client.GetFeedback(ctx, b.username, 1, offset) if err != nil { results <- models.LovesResult{Error: err} return @@ -272,9 +276,10 @@ out: } func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + ctx := context.TODO() if len(b.existingMBIDs) == 0 { existingLovesChan := make(chan models.LovesResult) - go b.exportLoves(time.Unix(0, 0), existingLovesChan) + go b.exportLoves(ctx, time.Unix(0, 0), existingLovesChan) // TODO: Store MBIDs directly b.existingMBIDs = make(map[mbtypes.MBID]bool, MaxItemsPerGet) @@ -303,7 +308,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importRe } if recordingMBID == "" { - lookup, err := b.client.Lookup(love.TrackName, love.ArtistName()) + lookup, err := b.client.Lookup(ctx, love.TrackName, love.ArtistName()) if err == nil { recordingMBID = lookup.RecordingMBID } @@ -315,7 +320,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importRe if b.existingMBIDs[recordingMBID] { ok = true } else { - resp, err := b.client.SendFeedback(Feedback{ + resp, err := b.client.SendFeedback(ctx, Feedback{ RecordingMBID: recordingMBID, Score: 1, }) @@ -351,7 +356,7 @@ var defaultDuration = time.Duration(3 * time.Minute) const trackSimilarityThreshold = 0.9 -func (b *ListenBrainzApiBackend) checkDuplicateListen(listen models.Listen) (bool, error) { +func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, listen models.Listen) (bool, error) { // Find listens duration := listen.Duration if duration == 0 { @@ -359,7 +364,7 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(listen models.Listen) (boo } minTime := listen.ListenedAt.Add(-duration) maxTime := listen.ListenedAt.Add(duration) - candidates, err := b.client.GetListens(b.username, maxTime, minTime) + candidates, err := b.client.GetListens(ctx, b.username, maxTime, minTime) if err != nil { return false, err } diff --git a/internal/backends/maloja/client.go b/internal/backends/maloja/client.go index 249819a..b80cb56 100644 --- a/internal/backends/maloja/client.go +++ b/internal/backends/maloja/client.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package maloja import ( + "context" "errors" "strconv" @@ -48,9 +49,10 @@ func NewClient(serverURL string, token string) Client { } } -func (c Client) GetScrobbles(page int, perPage int) (result GetScrobblesResult, err error) { +func (c Client) GetScrobbles(ctx context.Context, page int, perPage int) (result GetScrobblesResult, err error) { const path = "/apis/mlj_1/scrobbles" response, err := c.HTTPClient.R(). + SetContext(ctx). SetQueryParams(map[string]string{ "page": strconv.Itoa(page), "perpage": strconv.Itoa(perPage), @@ -65,10 +67,11 @@ func (c Client) GetScrobbles(page int, perPage int) (result GetScrobblesResult, return } -func (c Client) NewScrobble(scrobble NewScrobble) (result NewScrobbleResult, err error) { +func (c Client) NewScrobble(ctx context.Context, scrobble NewScrobble) (result NewScrobbleResult, err error) { const path = "/apis/mlj_1/newscrobble" scrobble.Key = c.token response, err := c.HTTPClient.R(). + SetContext(ctx). SetBody(scrobble). SetResult(&result). Post(path) diff --git a/internal/backends/maloja/client_test.go b/internal/backends/maloja/client_test.go index 54316a8..415f911 100644 --- a/internal/backends/maloja/client_test.go +++ b/internal/backends/maloja/client_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package maloja_test import ( + "context" "net/http" "testing" @@ -48,7 +49,8 @@ func TestGetScrobbles(t *testing.T) { "https://maloja.example.com/apis/mlj_1/scrobbles", "testdata/scrobbles.json") - result, err := client.GetScrobbles(0, 2) + ctx := context.Background() + result, err := client.GetScrobbles(ctx, 0, 2) require.NoError(t, err) assert := assert.New(t) @@ -69,12 +71,13 @@ func TestNewScrobble(t *testing.T) { url := server + "/apis/mlj_1/newscrobble" httpmock.RegisterResponder("POST", url, responder) + ctx := context.Background() scrobble := maloja.NewScrobble{ Title: "Oweynagat", Artist: "Dool", Time: 1699574369, } - result, err := client.NewScrobble(scrobble) + result, err := client.NewScrobble(ctx, scrobble) require.NoError(t, err) assert.Equal(t, "success", result.Status) diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index 8968942..4a4965e 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -17,6 +17,7 @@ Scotty. If not, see . package maloja import ( + "context" "errors" "sort" "strings" @@ -64,6 +65,7 @@ func (b *MalojaApiBackend) StartImport() error { return nil } func (b *MalojaApiBackend) FinishImport() error { return nil } func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + ctx := context.TODO() page := 0 perPage := MaxItemsPerGet @@ -77,7 +79,7 @@ func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan out: for { - result, err := b.client.GetScrobbles(page, perPage) + result, err := b.client.GetScrobbles(ctx, page, perPage) if err != nil { p.Export.Abort() progress <- p @@ -112,6 +114,8 @@ out: } func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + ctx := context.TODO() + p := models.TransferProgress{}.FromImportResult(importResult, false) for _, listen := range export.Items { scrobble := NewScrobble{ @@ -124,7 +128,7 @@ func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResu Nofix: b.nofix, } - resp, err := b.client.NewScrobble(scrobble) + resp, err := b.client.NewScrobble(ctx, scrobble) if err != nil { return importResult, err } else if resp.Status != "success" { diff --git a/internal/backends/spotify/client.go b/internal/backends/spotify/client.go index ff2b0a3..94d50ac 100644 --- a/internal/backends/spotify/client.go +++ b/internal/backends/spotify/client.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -59,17 +59,18 @@ func NewClient(token oauth2.TokenSource) Client { } } -func (c Client) RecentlyPlayedAfter(after time.Time, limit int) (RecentlyPlayedResult, error) { - return c.recentlyPlayed(&after, nil, limit) +func (c Client) RecentlyPlayedAfter(ctx context.Context, after time.Time, limit int) (RecentlyPlayedResult, error) { + return c.recentlyPlayed(ctx, &after, nil, limit) } -func (c Client) RecentlyPlayedBefore(before time.Time, limit int) (RecentlyPlayedResult, error) { - return c.recentlyPlayed(nil, &before, limit) +func (c Client) RecentlyPlayedBefore(ctx context.Context, before time.Time, limit int) (RecentlyPlayedResult, error) { + return c.recentlyPlayed(ctx, nil, &before, limit) } -func (c Client) recentlyPlayed(after *time.Time, before *time.Time, limit int) (result RecentlyPlayedResult, err error) { +func (c Client) recentlyPlayed(ctx context.Context, after *time.Time, before *time.Time, limit int) (result RecentlyPlayedResult, err error) { const path = "/me/player/recently-played" request := c.HTTPClient.R(). + SetContext(ctx). SetQueryParam("limit", strconv.Itoa(limit)). SetResult(&result) if after != nil { @@ -85,9 +86,10 @@ func (c Client) recentlyPlayed(after *time.Time, before *time.Time, limit int) ( return } -func (c Client) UserTracks(offset int, limit int) (result TracksResult, err error) { +func (c Client) UserTracks(ctx context.Context, offset int, limit int) (result TracksResult, err error) { const path = "/me/tracks" response, err := c.HTTPClient.R(). + SetContext(ctx). SetQueryParams(map[string]string{ "offset": strconv.Itoa(offset), "limit": strconv.Itoa(limit), diff --git a/internal/backends/spotify/client_test.go b/internal/backends/spotify/client_test.go index 78ff063..8135e1d 100644 --- a/internal/backends/spotify/client_test.go +++ b/internal/backends/spotify/client_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -22,6 +22,7 @@ THE SOFTWARE. package spotify_test import ( + "context" "net/http" "testing" "time" @@ -47,7 +48,8 @@ func TestRecentlyPlayedAfter(t *testing.T) { "https://api.spotify.com/v1/me/player/recently-played", "testdata/recently-played.json") - result, err := client.RecentlyPlayedAfter(time.Now(), 3) + ctx := context.Background() + result, err := client.RecentlyPlayedAfter(ctx, time.Now(), 3) require.NoError(t, err) assert := assert.New(t) @@ -67,7 +69,8 @@ func TestGetUserTracks(t *testing.T) { "https://api.spotify.com/v1/me/tracks", "testdata/user-tracks.json") - result, err := client.UserTracks(0, 2) + ctx := context.Background() + result, err := client.UserTracks(ctx, 0, 2) require.NoError(t, err) assert := assert.New(t) diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index 5d45087..73434b3 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -18,6 +18,7 @@ Scotty. If not, see . package spotify import ( + "context" "math" "net/url" "sort" @@ -96,6 +97,7 @@ func (b *SpotifyApiBackend) OAuth2Setup(token oauth2.TokenSource) error { } func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + ctx := context.TODO() startTime := time.Now() minTime := oldestTimestamp @@ -107,7 +109,7 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha } for { - result, err := b.client.RecentlyPlayedAfter(minTime, MaxItemsPerGet) + result, err := b.client.RecentlyPlayedAfter(ctx, minTime, MaxItemsPerGet) if err != nil { p.Export.Abort() progress <- p @@ -163,6 +165,7 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha } func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + ctx := context.TODO() // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 @@ -178,7 +181,7 @@ func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan out: for { - result, err := b.client.UserTracks(offset, perPage) + result, err := b.client.UserTracks(ctx, offset, perPage) if err != nil { p.Export.Abort() progress <- p From b5bca1d4abfb957f6f942f8dd9484147bccd7918 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 09:24:06 +0200 Subject: [PATCH 31/77] Use context aware musicbrainzws2 --- go.mod | 2 +- go.sum | 4 ++-- internal/backends/listenbrainz/listenbrainz.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 47c7e88..db7ecc3 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d github.com/vbauerster/mpb/v8 v8.10.0 go.uploadedlobster.com/mbtypes v0.4.0 - go.uploadedlobster.com/musicbrainzws2 v0.14.0 + go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 golang.org/x/oauth2 v0.30.0 golang.org/x/text v0.25.0 diff --git a/go.sum b/go.sum index 426e8c0..ef278f9 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s= go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= -go.uploadedlobster.com/musicbrainzws2 v0.14.0 h1:YaEtxNwLSNT1gzFipQ4XlaThNfXjBpzzb4I6WhIeUwg= -go.uploadedlobster.com/musicbrainzws2 v0.14.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= +go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400 h1:wMJloSsyWjfXznQNjvsrqAeL61BGoil7t4H9hPt18fc= +go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index d622aff..9f269a2 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -251,7 +251,7 @@ out: // longer available and might have been merged. Try fetching details // from MusicBrainz. if feedback.TrackMetadata == nil { - track, err := b.lookupRecording(feedback.RecordingMBID) + track, err := b.lookupRecording(ctx, feedback.RecordingMBID) if err == nil { feedback.TrackMetadata = track } @@ -379,11 +379,11 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste return false, nil } -func (b *ListenBrainzApiBackend) lookupRecording(mbid mbtypes.MBID) (*Track, error) { +func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtypes.MBID) (*Track, error) { filter := musicbrainzws2.IncludesFilter{ Includes: []string{"artist-credits"}, } - recording, err := b.mbClient.LookupRecording(mbid, filter) + recording, err := b.mbClient.LookupRecording(ctx, mbid, filter) if err != nil { return nil, err } From 26d9f5e840e357d94731210edf0337750c989eef Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 11:09:39 +0200 Subject: [PATCH 32/77] Pass context to export backends --- internal/backends/deezer/deezer.go | 8 ++----- internal/backends/export.go | 4 ++-- internal/backends/funkwhale/funkwhale.go | 6 ++--- internal/backends/jspf/jspf.go | 5 ++-- internal/backends/lastfm/lastfm.go | 23 +++++++++++++++++-- .../backends/listenbrainz/listenbrainz.go | 6 ++--- internal/backends/maloja/maloja.go | 3 +-- .../backends/scrobblerlog/scrobblerlog.go | 3 ++- internal/backends/spotify/spotify.go | 6 ++--- .../backends/spotifyhistory/spotifyhistory.go | 16 ++++++++++--- internal/backends/subsonic/subsonic.go | 3 ++- internal/models/interfaces.go | 5 ++-- 12 files changed, 55 insertions(+), 33 deletions(-) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index f3e3d37..c38f4e7 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -78,9 +78,7 @@ func (b *DeezerApiBackend) OAuth2Setup(token oauth2.TokenSource) error { return nil } -func (b *DeezerApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - ctx := context.TODO() - +func (b *DeezerApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 @@ -156,9 +154,7 @@ out: progress <- p } -func (b *DeezerApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { - ctx := context.TODO() - +func (b *DeezerApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 diff --git a/internal/backends/export.go b/internal/backends/export.go index 0deebc6..29ae595 100644 --- a/internal/backends/export.go +++ b/internal/backends/export.go @@ -40,7 +40,7 @@ func (p ListensExportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, wg.Add(1) defer wg.Done() defer close(results) - p.Backend.ExportListens(oldestTimestamp, results, progress) + p.Backend.ExportListens(ctx, oldestTimestamp, results, progress) } type LovesExportProcessor struct { @@ -55,5 +55,5 @@ func (p LovesExportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, o wg.Add(1) defer wg.Done() defer close(results) - p.Backend.ExportLoves(oldestTimestamp, results, progress) + p.Backend.ExportLoves(ctx, oldestTimestamp, results, progress) } diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index 434716f..d9632a6 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -61,8 +61,7 @@ func (b *FunkwhaleApiBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *FunkwhaleApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := 1 perPage := MaxItemsPerGet @@ -119,8 +118,7 @@ out: results <- models.ListensResult{Items: listens} } -func (b *FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *FunkwhaleApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { page := 1 perPage := MaxItemsPerGet diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 0e200f2..77fed4b 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -18,6 +18,7 @@ Scotty. If not, see . package jspf import ( + "context" "errors" "os" "sort" @@ -93,7 +94,7 @@ func (b *JSPFBackend) FinishImport() error { return b.writeJSPF() } -func (b *JSPFBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { +func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { err := b.readJSPF() p := models.TransferProgress{ Export: &models.Progress{}, @@ -132,7 +133,7 @@ func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult mo return importResult, nil } -func (b *JSPFBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { +func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { err := b.readJSPF() p := models.TransferProgress{ Export: &models.Progress{}, diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index d262ada..3de75d1 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -16,6 +16,7 @@ Scotty. If not, see . package lastfm import ( + "context" "fmt" "net/url" "sort" @@ -88,7 +89,7 @@ func (b *LastfmApiBackend) OAuth2Setup(token oauth2.TokenSource) error { return nil } -func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { +func (b *LastfmApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := MaxPage minTime := oldestTimestamp perPage := MaxItemsPerGet @@ -102,6 +103,15 @@ func (b *LastfmApiBackend) ExportListens(oldestTimestamp time.Time, results chan out: for page > 0 { + select { + case <-ctx.Done(): + results <- models.ListensResult{Error: ctx.Err()} + p.Export.Abort() + progress <- p + return + default: + } + args := lastfm.P{ "user": b.username, "limit": MaxListensPerGet, @@ -258,7 +268,7 @@ func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResu return importResult, nil } -func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { +func (b *LastfmApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. page := 1 @@ -274,6 +284,15 @@ func (b *LastfmApiBackend) ExportLoves(oldestTimestamp time.Time, results chan m out: for { + select { + case <-ctx.Done(): + results <- models.LovesResult{Error: ctx.Err()} + p.Export.Abort() + progress <- p + return + default: + } + result, err := b.client.User.GetLovedTracks(lastfm.P{ "user": b.username, "limit": MaxItemsPerGet, diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 9f269a2..ca1c0f0 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -73,8 +73,7 @@ func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error func (b *ListenBrainzApiBackend) StartImport() error { return nil } func (b *ListenBrainzApiBackend) FinishImport() error { return nil } -func (b *ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { startTime := time.Now() minTime := oldestTimestamp if minTime.Unix() < 1 { @@ -201,8 +200,7 @@ func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, impo return importResult, nil } -func (b *ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *ListenBrainzApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { exportChan := make(chan models.LovesResult) p := models.TransferProgress{ Export: &models.Progress{}, diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index 4a4965e..8642924 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -64,8 +64,7 @@ func (b *MalojaApiBackend) InitConfig(config *config.ServiceConfig) error { func (b *MalojaApiBackend) StartImport() error { return nil } func (b *MalojaApiBackend) FinishImport() error { return nil } -func (b *MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *MalojaApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := 0 perPage := MaxItemsPerGet diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6454b7b..c7eb636 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -17,6 +17,7 @@ Scotty. If not, see . package scrobblerlog import ( + "context" "fmt" "os" "sort" @@ -129,7 +130,7 @@ func (b *ScrobblerLogBackend) FinishImport() error { return b.file.Close() } -func (b *ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { +func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { file, err := os.Open(b.filePath) p := models.TransferProgress{ Export: &models.Progress{}, diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index 73434b3..b00ebba 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -96,8 +96,7 @@ func (b *SpotifyApiBackend) OAuth2Setup(token oauth2.TokenSource) error { return nil } -func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *SpotifyApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { startTime := time.Now() minTime := oldestTimestamp @@ -164,8 +163,7 @@ func (b *SpotifyApiBackend) ExportListens(oldestTimestamp time.Time, results cha progress <- p } -func (b *SpotifyApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { - ctx := context.TODO() +func (b *SpotifyApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { // Choose a high offset, we attempt to search the loves backwards starting // at the oldest one. offset := math.MaxInt32 diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index d5c87bb..9a1ab2b 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -18,6 +18,7 @@ Scotty. If not, see . package spotifyhistory import ( + "context" "os" "path" "path/filepath" @@ -72,7 +73,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { +func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) p := models.TransferProgress{ Export: &models.Progress{}, @@ -89,11 +90,20 @@ func (b *SpotifyHistoryBackend) ExportListens(oldestTimestamp time.Time, results fileCount := int64(len(files)) p.Export.Total = fileCount for i, filePath := range files { - history, err := readHistoryFile(filePath) - if err != nil { + select { + case <-ctx.Done(): + results <- models.ListensResult{Error: ctx.Err()} p.Export.Abort() progress <- p + return + default: + } + + history, err := readHistoryFile(filePath) + if err != nil { results <- models.ListensResult{Error: err} + p.Export.Abort() + progress <- p return } listens := history.AsListenList(ListenListOptions{ diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index 2098688..aa1b1e3 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -17,6 +17,7 @@ Scotty. If not, see . package subsonic import ( + "context" "net/http" "sort" "time" @@ -63,7 +64,7 @@ func (b *SubsonicApiBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { +func (b *SubsonicApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { err := b.client.Authenticate(b.password) p := models.TransferProgress{ Export: &models.Progress{}, diff --git a/internal/models/interfaces.go b/internal/models/interfaces.go index bb97dac..2b45d18 100644 --- a/internal/models/interfaces.go +++ b/internal/models/interfaces.go @@ -17,6 +17,7 @@ Scotty. If not, see . package models import ( + "context" "time" // "go.uploadedlobster.com/scotty/internal/auth" @@ -55,7 +56,7 @@ type ListensExport interface { // Returns a list of all listens newer then oldestTimestamp. // The returned list of listens is supposed to be ordered by the // Listen.ListenedAt timestamp, with the oldest entry first. - ExportListens(oldestTimestamp time.Time, results chan ListensResult, progress chan TransferProgress) + ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan ListensResult, progress chan TransferProgress) } // Must be implemented by services supporting the import of listens. @@ -73,7 +74,7 @@ type LovesExport interface { // Returns a list of all loves newer then oldestTimestamp. // The returned list of listens is supposed to be ordered by the // Love.Created timestamp, with the oldest entry first. - ExportLoves(oldestTimestamp time.Time, results chan LovesResult, progress chan TransferProgress) + ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan LovesResult, progress chan TransferProgress) } // Must be implemented by services supporting the import of loves. From 4a66e3d43285ecf4179eb146f3346122b18e99e3 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 11:20:09 +0200 Subject: [PATCH 33/77] Pass context to import backends --- internal/backends/dump/dump.go | 39 ++++++++++++------- internal/backends/import.go | 12 +++--- internal/backends/jspf/jspf.go | 30 +++++++++----- internal/backends/lastfm/lastfm.go | 16 +++++++- .../backends/listenbrainz/listenbrainz.go | 6 +-- internal/backends/maloja/maloja.go | 4 +- .../backends/scrobblerlog/scrobblerlog.go | 2 +- internal/models/interfaces.go | 4 +- 8 files changed, 71 insertions(+), 42 deletions(-) diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index add8711..14583f6 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -17,6 +17,7 @@ Scotty. If not, see . package dump import ( + "context" "fmt" "go.uploadedlobster.com/scotty/internal/config" @@ -36,27 +37,37 @@ func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error { func (b *DumpBackend) StartImport() error { return nil } func (b *DumpBackend) FinishImport() error { return nil } -func (b *DumpBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { - importResult.UpdateTimestamp(listen.ListenedAt) - importResult.ImportCount += 1 - msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", - listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) - importResult.Log(models.Info, msg) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) + select { + case <-ctx.Done(): + return importResult, ctx.Err() + default: + importResult.UpdateTimestamp(listen.ListenedAt) + importResult.ImportCount += 1 + msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", + listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) + importResult.Log(models.Info, msg) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) + } } return importResult, nil } -func (b *DumpBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *DumpBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { - importResult.UpdateTimestamp(love.Created) - importResult.ImportCount += 1 - msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", - love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) - importResult.Log(models.Info, msg) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) + select { + case <-ctx.Done(): + return importResult, ctx.Err() + default: + importResult.UpdateTimestamp(love.Created) + importResult.ImportCount += 1 + msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", + love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) + importResult.Log(models.Info, msg) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) + } } return importResult, nil diff --git a/internal/backends/import.go b/internal/backends/import.go index e7006bd..3d77b44 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -27,7 +27,7 @@ import ( type ImportProcessor[T models.ListensResult | models.LovesResult] interface { ImportBackend() models.ImportBackend Process(ctx context.Context, wg *sync.WaitGroup, results chan T, out chan models.ImportResult, progress chan models.TransferProgress) - Import(export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) + Import(ctx context.Context, export T, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) } type ListensImportProcessor struct { @@ -42,7 +42,7 @@ func (p ListensImportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, process(ctx, wg, p, results, out, progress) } -func (p ListensImportProcessor) Import(export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (p ListensImportProcessor) Import(ctx context.Context, export models.ListensResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { if export.Error != nil { return result, export.Error } @@ -52,7 +52,7 @@ func (p ListensImportProcessor) Import(export models.ListensResult, result model } else { result.TotalCount += len(export.Items) } - importResult, err := p.Backend.ImportListens(export, result, progress) + importResult, err := p.Backend.ImportListens(ctx, export, result, progress) if err != nil { return importResult, err } @@ -71,7 +71,7 @@ func (p LovesImportProcessor) Process(ctx context.Context, wg *sync.WaitGroup, r process(ctx, wg, p, results, out, progress) } -func (p LovesImportProcessor) Import(export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (p LovesImportProcessor) Import(ctx context.Context, export models.LovesResult, result models.ImportResult, out chan models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { if export.Error != nil { return result, export.Error } @@ -81,7 +81,7 @@ func (p LovesImportProcessor) Import(export models.LovesResult, result models.Im } else { result.TotalCount += len(export.Items) } - importResult, err := p.Backend.ImportLoves(export, result, progress) + importResult, err := p.Backend.ImportLoves(ctx, export, result, progress) if err != nil { return importResult, err } @@ -112,7 +112,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( out <- handleError(result, ctx.Err(), progress) return default: - importResult, err := processor.Import(exportResult, result, out, progress) + importResult, err := processor.Import(ctx, exportResult, result, out, progress) result.Update(importResult) if err != nil { processor.ImportBackend().FinishImport() diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 77fed4b..a8a1929 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -121,12 +121,17 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti results <- models.ListensResult{Items: listens} } -func (b *JSPFBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { - track := listenAsTrack(listen) - b.playlist.Tracks = append(b.playlist.Tracks, track) - importResult.ImportCount += 1 - importResult.UpdateTimestamp(listen.ListenedAt) + select { + case <-ctx.Done(): + return importResult, ctx.Err() + default: + track := listenAsTrack(listen) + b.playlist.Tracks = append(b.playlist.Tracks, track) + importResult.ImportCount += 1 + importResult.UpdateTimestamp(listen.ListenedAt) + } } progress <- models.TransferProgress{}.FromImportResult(importResult, false) @@ -160,12 +165,17 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time results <- models.LovesResult{Items: loves} } -func (b *JSPFBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { - track := loveAsTrack(love) - b.playlist.Tracks = append(b.playlist.Tracks, track) - importResult.ImportCount += 1 - importResult.UpdateTimestamp(love.Created) + select { + case <-ctx.Done(): + return importResult, ctx.Err() + default: + track := loveAsTrack(love) + b.playlist.Tracks = append(b.playlist.Tracks, track) + importResult.ImportCount += 1 + importResult.UpdateTimestamp(love.Created) + } } progress <- models.TransferProgress{}.FromImportResult(importResult, false) diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index 3de75d1..afc1fa3 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -192,9 +192,15 @@ out: progress <- p } -func (b *LastfmApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *LastfmApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) for i := 0; i < total; i += MaxListensPerSubmission { + select { + case <-ctx.Done(): + return importResult, ctx.Err() + default: + } + listens := export.Items[i:min(i+MaxListensPerSubmission, total)] count := len(listens) if count == 0 { @@ -354,8 +360,14 @@ out: results <- models.LovesResult{Items: loves, Total: totalCount} } -func (b *LastfmApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *LastfmApiBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { + select { + case <-ctx.Done(): + return importResult, ctx.Err() + default: + } + err := b.client.Track.Love(lastfm.P{ "track": love.TrackName, "artist": love.ArtistName(), diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index ca1c0f0..bf46c22 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -135,8 +135,7 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest progress <- p } -func (b *ListenBrainzApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - ctx := context.TODO() +func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) p := models.TransferProgress{}.FromImportResult(importResult, false) for i := 0; i < total; i += MaxListensPerRequest { @@ -273,8 +272,7 @@ out: } } -func (b *ListenBrainzApiBackend) ImportLoves(export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - ctx := context.TODO() +func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { if len(b.existingMBIDs) == 0 { existingLovesChan := make(chan models.LovesResult) go b.exportLoves(ctx, time.Unix(0, 0), existingLovesChan) diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index 8642924..f082d9b 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -112,9 +112,7 @@ out: results <- models.ListensResult{Items: listens} } -func (b *MalojaApiBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - ctx := context.TODO() - +func (b *MalojaApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { p := models.TransferProgress{}.FromImportResult(importResult, false) for _, listen := range export.Items { scrobble := NewScrobble{ diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index c7eb636..6d331ce 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -167,7 +167,7 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp results <- models.ListensResult{Items: listens} } -func (b *ScrobblerLogBackend) ImportListens(export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { +func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { records := make([]scrobblerlog.Record, len(export.Items)) for i, listen := range export.Items { records[i] = listenToRecord(listen) diff --git a/internal/models/interfaces.go b/internal/models/interfaces.go index 2b45d18..2f4beaf 100644 --- a/internal/models/interfaces.go +++ b/internal/models/interfaces.go @@ -64,7 +64,7 @@ type ListensImport interface { ImportBackend // Imports the given list of listens. - ImportListens(export ListensResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error) + ImportListens(ctx context.Context, export ListensResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error) } // Must be implemented by services supporting the export of loves. @@ -82,5 +82,5 @@ type LovesImport interface { ImportBackend // Imports the given list of loves. - ImportLoves(export LovesResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error) + ImportLoves(ctx context.Context, export LovesResult, importResult ImportResult, progress chan TransferProgress) (ImportResult, error) } From 20853f7601c80b1254c59d903b0e46667e98cecc Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 14:13:31 +0200 Subject: [PATCH 34/77] Simplify context cancellation checks --- internal/backends/dump/dump.go | 38 +++++++++---------- internal/backends/import.go | 23 ++++++----- internal/backends/jspf/jspf.go | 30 +++++++-------- internal/backends/lastfm/lastfm.go | 24 ++++-------- .../backends/spotifyhistory/spotifyhistory.go | 6 +-- 5 files changed, 53 insertions(+), 68 deletions(-) diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index 14583f6..1fcd864 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -39,17 +39,16 @@ func (b *DumpBackend) FinishImport() error { return nil } func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { - select { - case <-ctx.Done(): - return importResult, ctx.Err() - default: - importResult.UpdateTimestamp(listen.ListenedAt) - importResult.ImportCount += 1 - msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", - listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) - importResult.Log(models.Info, msg) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) + if err := ctx.Err(); err != nil { + return importResult, err } + + importResult.UpdateTimestamp(listen.ListenedAt) + importResult.ImportCount += 1 + msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", + listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) + importResult.Log(models.Info, msg) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil @@ -57,17 +56,16 @@ func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensRe func (b *DumpBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { - select { - case <-ctx.Done(): - return importResult, ctx.Err() - default: - importResult.UpdateTimestamp(love.Created) - importResult.ImportCount += 1 - msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", - love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) - importResult.Log(models.Info, msg) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) + if err := ctx.Err(); err != nil { + return importResult, err } + + importResult.UpdateTimestamp(love.Created) + importResult.ImportCount += 1 + msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", + love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) + importResult.Log(models.Info, msg) + progress <- models.TransferProgress{}.FromImportResult(importResult, false) } return importResult, nil diff --git a/internal/backends/import.go b/internal/backends/import.go index 3d77b44..e7a6add 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -106,21 +106,20 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( } for exportResult := range results { - select { - case <-ctx.Done(): + if err := ctx.Err(); err != nil { processor.ImportBackend().FinishImport() - out <- handleError(result, ctx.Err(), progress) + out <- handleError(result, err, progress) return - default: - importResult, err := processor.Import(ctx, exportResult, result, out, progress) - result.Update(importResult) - if err != nil { - processor.ImportBackend().FinishImport() - out <- handleError(result, err, progress) - return - } - progress <- p.FromImportResult(result, false) } + + importResult, err := processor.Import(ctx, exportResult, result, out, progress) + result.Update(importResult) + if err != nil { + processor.ImportBackend().FinishImport() + out <- handleError(result, err, progress) + return + } + progress <- p.FromImportResult(result, false) } if err := processor.ImportBackend().FinishImport(); err != nil { diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index a8a1929..354640e 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -123,15 +123,14 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { - select { - case <-ctx.Done(): - return importResult, ctx.Err() - default: - track := listenAsTrack(listen) - b.playlist.Tracks = append(b.playlist.Tracks, track) - importResult.ImportCount += 1 - importResult.UpdateTimestamp(listen.ListenedAt) + if err := ctx.Err(); err != nil { + return importResult, err } + + track := listenAsTrack(listen) + b.playlist.Tracks = append(b.playlist.Tracks, track) + importResult.ImportCount += 1 + importResult.UpdateTimestamp(listen.ListenedAt) } progress <- models.TransferProgress{}.FromImportResult(importResult, false) @@ -167,15 +166,14 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { - select { - case <-ctx.Done(): - return importResult, ctx.Err() - default: - track := loveAsTrack(love) - b.playlist.Tracks = append(b.playlist.Tracks, track) - importResult.ImportCount += 1 - importResult.UpdateTimestamp(love.Created) + if err := ctx.Err(); err != nil { + return importResult, err } + + track := loveAsTrack(love) + b.playlist.Tracks = append(b.playlist.Tracks, track) + importResult.ImportCount += 1 + importResult.UpdateTimestamp(love.Created) } progress <- models.TransferProgress{}.FromImportResult(importResult, false) diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index afc1fa3..b34452e 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -103,13 +103,11 @@ func (b *LastfmApiBackend) ExportListens(ctx context.Context, oldestTimestamp ti out: for page > 0 { - select { - case <-ctx.Done(): - results <- models.ListensResult{Error: ctx.Err()} + if err := ctx.Err(); err != nil { + results <- models.ListensResult{Error: err} p.Export.Abort() progress <- p return - default: } args := lastfm.P{ @@ -195,10 +193,8 @@ out: func (b *LastfmApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) for i := 0; i < total; i += MaxListensPerSubmission { - select { - case <-ctx.Done(): - return importResult, ctx.Err() - default: + if err := ctx.Err(); err != nil { + return importResult, err } listens := export.Items[i:min(i+MaxListensPerSubmission, total)] @@ -290,13 +286,11 @@ func (b *LastfmApiBackend) ExportLoves(ctx context.Context, oldestTimestamp time out: for { - select { - case <-ctx.Done(): - results <- models.LovesResult{Error: ctx.Err()} + if err := ctx.Err(); err != nil { + results <- models.LovesResult{Error: err} p.Export.Abort() progress <- p return - default: } result, err := b.client.User.GetLovedTracks(lastfm.P{ @@ -362,10 +356,8 @@ out: func (b *LastfmApiBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, love := range export.Items { - select { - case <-ctx.Done(): - return importResult, ctx.Err() - default: + if err := ctx.Err(); err != nil { + return importResult, err } err := b.client.Track.Love(lastfm.P{ diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 9a1ab2b..76d0c9e 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -90,13 +90,11 @@ func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimesta fileCount := int64(len(files)) p.Export.Total = fileCount for i, filePath := range files { - select { - case <-ctx.Done(): - results <- models.ListensResult{Error: ctx.Err()} + if err := ctx.Err(); err != nil { + results <- models.ListensResult{Error: err} p.Export.Abort() progress <- p return - default: } history, err := readHistoryFile(filePath) From dacfb72f7d2ea66644e74cf466cc29e7f1db14cd Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 17:09:26 +0200 Subject: [PATCH 35/77] Upgrade dependencies --- go.mod | 8 ++++---- go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index db7ecc3..f5dea40 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d - github.com/vbauerster/mpb/v8 v8.10.0 + github.com/vbauerster/mpb/v8 v8.10.1 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 @@ -66,10 +66,10 @@ require ( golang.org/x/tools v0.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gorm.io/driver/mysql v1.5.7 // indirect - modernc.org/libc v1.65.2 // indirect + modernc.org/libc v1.65.7 // indirect modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.10.0 // indirect - modernc.org/sqlite v1.37.0 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.37.1 // indirect ) tool golang.org/x/text/cmd/gotext diff --git a/go.sum b/go.sum index ef278f9..02063fc 100644 --- a/go.sum +++ b/go.sum @@ -125,8 +125,8 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4= -github.com/vbauerster/mpb/v8 v8.10.0 h1:5ZYEWM4ovaZGAibjzW4PlQNb5k+JpzMqVwgNyk+K0M8= -github.com/vbauerster/mpb/v8 v8.10.0/go.mod h1:DYPFebxSahB+f7tuEUGauLQ7w8ij3wMr4clsVuJCV4I= +github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= +github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= @@ -210,24 +210,24 @@ gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw= gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.27.1 h1:emhLB4uoOmkZUnTDFcMI3AbkmU/Evjuerit9Taqe6Ss= -modernc.org/ccgo/v4 v4.27.1/go.mod h1:543Q0qQhJWekKVS5P6yL5fO6liNhla9Lbm2/B3rEKDE= +modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= +modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.65.2 h1:drWL1QO9fKXr3kXDN8y+4lKyBr8bA3mtUBQpftq3IJw= -modernc.org/libc v1.65.2/go.mod h1:VI3V2S5mNka4deJErQ0jsMXe7jgxojE2fOB/mWoHlbc= +modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00= +modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= -modernc.org/memory v1.10.0 h1:fzumd51yQ1DxcOxSO+S6X7+QTuVU+n8/Aj7swYjFfC4= -modernc.org/memory v1.10.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.37.0 h1:s1TMe7T3Q3ovQiK2Ouz4Jwh7dw4ZDqbebSDTlSJdfjI= -modernc.org/sqlite v1.37.0/go.mod h1:5YiWv+YviqGMuGw4V+PNplcyaJ5v+vQd7TQOgkACoJM= +modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs= +modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From e9768c09346e27e6baef8fe48ddef897c6aa0dc3 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 18:45:39 +0200 Subject: [PATCH 36/77] Upgrade musicbrainzws2 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f5dea40..a00b416 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d github.com/vbauerster/mpb/v8 v8.10.1 go.uploadedlobster.com/mbtypes v0.4.0 - go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400 + go.uploadedlobster.com/musicbrainzws2 v0.15.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 golang.org/x/oauth2 v0.30.0 golang.org/x/text v0.25.0 diff --git a/go.sum b/go.sum index 02063fc..3cd01a6 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s= go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= -go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400 h1:wMJloSsyWjfXznQNjvsrqAeL61BGoil7t4H9hPt18fc= -go.uploadedlobster.com/musicbrainzws2 v0.14.1-0.20250522060150-50bf4bea5400/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= +go.uploadedlobster.com/musicbrainzws2 v0.15.0 h1:njJeyf1dDwfz2toEHaZSuockVsn1fg+967/tVfLHhwQ= +go.uploadedlobster.com/musicbrainzws2 v0.15.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= From 12eb7acd9871110b5f08fd2d2a7e77703e34ccd6 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 18:56:49 +0200 Subject: [PATCH 37/77] Update changelog --- CHANGES.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index a0a60f2..64f854f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,9 +1,13 @@ # Scotty Changelog ## 0.6.0 - WIP -- Fix program hanging endlessly if import fails (#11) -- If import fails still store the last successfully imported timestamp -- Show progress bars as aborted on export / import error +- Fully reworked progress report + - Cancel both export and import on error + - Show progress bars as aborted on export / import error + - The import progress is now aware of the total amount of exported items + - The import progress shows total items processed instead of time estimate + - Fix program hanging endlessly if import fails (#11) + - If import fails still store the last successfully imported timestamp - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID @@ -11,6 +15,7 @@ in the existing JSPF file - scrobblerlog: fix timezone not being set from config (#6) - scrobblerlog: fix listen export not considering latest timestamp +- Funkwhale: fix progress abort on error ## 0.5.2 - 2025-05-01 From 83eac8c801ea34f328c084e841855a3b858a73f6 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Thu, 22 May 2025 08:29:48 +0200 Subject: [PATCH 38/77] Import progress shows actual number of processed items --- internal/cli/progress.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/cli/progress.go b/internal/cli/progress.go index db862a1..d17594c 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -87,7 +87,10 @@ func (u *progressBarUpdater) update() { func (u *progressBarUpdater) updateExportProgress(progress *models.Progress) { bar := u.exportBar - u.totalItems = progress.TotalItems + if progress.TotalItems != u.totalItems { + u.totalItems = progress.TotalItems + u.importBar.SetTotal(int64(u.totalItems), false) + } if progress.Aborted { bar.Abort(false) From c7af90b585bfe6ca59dbfc0ec4ea42eb49263909 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 07:47:52 +0200 Subject: [PATCH 39/77] More granular progress report for JSPF and scrobblerlog --- CHANGES.md | 1 + internal/backends/jspf/jspf.go | 24 ++++++------- .../backends/scrobblerlog/scrobblerlog.go | 13 ++++--- internal/models/models.go | 36 +++++++++++++++++++ 4 files changed, 55 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 64f854f..8dc9838 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ - The import progress shows total items processed instead of time estimate - Fix program hanging endlessly if import fails (#11) - If import fails still store the last successfully imported timestamp + - More granular progress updates for JSPF and scrobblerlog - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 354640e..e2bcde1 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -108,21 +108,22 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti } listens := make(models.ListensList, 0, len(b.playlist.Tracks)) - for _, track := range b.playlist.Tracks { + p.Export.Total = int64(len(b.playlist.Tracks)) + for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { listen, err := trackAsListen(track) if err == nil && listen != nil && listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, *listen) + p.Export.TotalItems += 1 } } + sort.Sort(listens) - p.Export.Total = int64(len(listens)) - p.Export.Complete() - progress <- p results <- models.ListensResult{Items: listens} } func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - for _, listen := range export.Items { + p := models.TransferProgress{}.FromImportResult(importResult, false) + for _, listen := range models.IterImportProgress(export.Items, &p, progress) { if err := ctx.Err(); err != nil { return importResult, err } @@ -133,7 +134,6 @@ func (b *JSPFBackend) ImportListens(ctx context.Context, export models.ListensRe importResult.UpdateTimestamp(listen.ListenedAt) } - progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } @@ -151,21 +151,22 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time } loves := make(models.LovesList, 0, len(b.playlist.Tracks)) - for _, track := range b.playlist.Tracks { + p.Export.Total = int64(len(b.playlist.Tracks)) + for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { love, err := trackAsLove(track) if err == nil && love != nil && love.Created.After(oldestTimestamp) { loves = append(loves, *love) + p.Export.TotalItems += 1 } } + sort.Sort(loves) - p.Export.Total = int64(len(loves)) - p.Export.Complete() - progress <- p results <- models.LovesResult{Items: loves} } func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { - for _, love := range export.Items { + p := models.TransferProgress{}.FromImportResult(importResult, false) + for _, love := range models.IterImportProgress(export.Items, &p, progress) { if err := ctx.Err(); err != nil { return importResult, err } @@ -176,7 +177,6 @@ func (b *JSPFBackend) ImportLoves(ctx context.Context, export models.LovesResult importResult.UpdateTimestamp(love.Created) } - progress <- models.TransferProgress{}.FromImportResult(importResult, false) return importResult, nil } diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d331ce..6d42f3c 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -154,22 +154,23 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens := make(models.ListensList, 0, len(b.log.Records)) client := strings.Split(b.log.Client, " ")[0] - for _, record := range b.log.Records { + p.Export.Total = int64(len(b.log.Records)) + for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { listen := recordToListen(record, client) if listen.ListenedAt.After(oldestTimestamp) { listens = append(listens, recordToListen(record, client)) + p.Export.TotalItems += 1 } } + sort.Sort(listens) - p.Export.Total = int64(len(listens)) - p.Export.Complete() - progress <- p results <- models.ListensResult{Items: listens} } func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { + p := models.TransferProgress{}.FromImportResult(importResult, false) records := make([]scrobblerlog.Record, len(export.Items)) - for i, listen := range export.Items { + for i, listen := range models.IterImportProgress(export.Items, &p, progress) { records[i] = listenToRecord(listen) } lastTimestamp, err := b.log.Append(b.file, records) @@ -179,8 +180,6 @@ func (b *ScrobblerLogBackend) ImportListens(ctx context.Context, export models.L importResult.UpdateTimestamp(lastTimestamp) importResult.ImportCount += len(export.Items) - progress <- models.TransferProgress{}.FromImportResult(importResult, false) - return importResult, nil } diff --git a/internal/models/models.go b/internal/models/models.go index 081266d..09b4d6b 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -22,6 +22,7 @@ THE SOFTWARE. package models import ( + "iter" "strings" "time" @@ -244,3 +245,38 @@ func (p *Progress) Complete() { func (p *Progress) Abort() { p.Aborted = true } + +func IterExportProgress[T any]( + items []T, t *TransferProgress, c chan TransferProgress, +) iter.Seq2[int, T] { + return iterProgress(items, t, t.Export, c, true) +} + +func IterImportProgress[T any]( + items []T, t *TransferProgress, c chan TransferProgress, +) iter.Seq2[int, T] { + return iterProgress(items, t, t.Import, c, false) +} + +func iterProgress[T any]( + items []T, t *TransferProgress, + p *Progress, c chan TransferProgress, + autocomplete bool, +) iter.Seq2[int, T] { + // Report progress in 1% steps + steps := len(items) / 100 + return func(yield func(int, T) bool) { + for i, item := range items { + yield(i, item) + p.Elapsed++ + if i%steps == 0 { + c <- *t + } + } + + if autocomplete { + p.Complete() + c <- *t + } + } +} From a8ce2be5d714b5ac145cd34a7a30a62fc24afc15 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 07:48:42 +0200 Subject: [PATCH 40/77] jspf/scrobblerlog: return results in batches This allows the importer to start working while export is still in progress --- internal/backends/jspf/jspf.go | 15 +++++++++++++-- internal/backends/scrobblerlog/scrobblerlog.go | 9 ++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index e2bcde1..8da585f 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -36,6 +36,7 @@ const ( artistMBIDPrefix = "https://musicbrainz.org/artist/" recordingMBIDPrefix = "https://musicbrainz.org/recording/" releaseMBIDPrefix = "https://musicbrainz.org/release/" + batchSize = 1000 ) type JSPFBackend struct { @@ -107,7 +108,7 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti return } - listens := make(models.ListensList, 0, len(b.playlist.Tracks)) + listens := make(models.ListensList, 0, batchSize) p.Export.Total = int64(len(b.playlist.Tracks)) for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { listen, err := trackAsListen(track) @@ -115,6 +116,11 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti listens = append(listens, *listen) p.Export.TotalItems += 1 } + + if len(listens) >= batchSize { + results <- models.ListensResult{Items: listens} + listens = listens[:0] + } } sort.Sort(listens) @@ -150,7 +156,7 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time return } - loves := make(models.LovesList, 0, len(b.playlist.Tracks)) + loves := make(models.LovesList, 0, batchSize) p.Export.Total = int64(len(b.playlist.Tracks)) for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { love, err := trackAsLove(track) @@ -158,6 +164,11 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time loves = append(loves, *love) p.Export.TotalItems += 1 } + + if len(loves) >= batchSize { + results <- models.LovesResult{Items: loves} + loves = loves[:0] + } } sort.Sort(loves) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d42f3c..84080ae 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -30,6 +30,8 @@ import ( "go.uploadedlobster.com/scotty/pkg/scrobblerlog" ) +const batchSize = 1000 + type ScrobblerLogBackend struct { filePath string ignoreSkipped bool @@ -152,7 +154,7 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp return } - listens := make(models.ListensList, 0, len(b.log.Records)) + listens := make(models.ListensList, 0, batchSize) client := strings.Split(b.log.Client, " ")[0] p.Export.Total = int64(len(b.log.Records)) for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { @@ -161,6 +163,11 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens = append(listens, recordToListen(record, client)) p.Export.TotalItems += 1 } + + if len(listens) >= batchSize { + results <- models.ListensResult{Items: listens} + listens = listens[:0] + } } sort.Sort(listens) From b7ce09041e448a6dcc3544e8607e443c41c539db Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 08:12:37 +0200 Subject: [PATCH 41/77] Fix potential zero division error in iterProgress --- internal/models/models.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/models/models.go b/internal/models/models.go index 09b4d6b..94e3897 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -264,7 +264,7 @@ func iterProgress[T any]( autocomplete bool, ) iter.Seq2[int, T] { // Report progress in 1% steps - steps := len(items) / 100 + steps := max(len(items)/100, 1) return func(yield func(int, T) bool) { for i, item := range items { yield(i, item) From 5927f41a830dec79877d87998b972a2988c06658 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 08:52:23 +0200 Subject: [PATCH 42/77] Revert "jspf/scrobblerlog: return results in batches" This reverts commit a8ce2be5d714b5ac145cd34a7a30a62fc24afc15. --- internal/backends/jspf/jspf.go | 15 ++------------- internal/backends/scrobblerlog/scrobblerlog.go | 9 +-------- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 8da585f..e2bcde1 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -36,7 +36,6 @@ const ( artistMBIDPrefix = "https://musicbrainz.org/artist/" recordingMBIDPrefix = "https://musicbrainz.org/recording/" releaseMBIDPrefix = "https://musicbrainz.org/release/" - batchSize = 1000 ) type JSPFBackend struct { @@ -108,7 +107,7 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti return } - listens := make(models.ListensList, 0, batchSize) + listens := make(models.ListensList, 0, len(b.playlist.Tracks)) p.Export.Total = int64(len(b.playlist.Tracks)) for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { listen, err := trackAsListen(track) @@ -116,11 +115,6 @@ func (b *JSPFBackend) ExportListens(ctx context.Context, oldestTimestamp time.Ti listens = append(listens, *listen) p.Export.TotalItems += 1 } - - if len(listens) >= batchSize { - results <- models.ListensResult{Items: listens} - listens = listens[:0] - } } sort.Sort(listens) @@ -156,7 +150,7 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time return } - loves := make(models.LovesList, 0, batchSize) + loves := make(models.LovesList, 0, len(b.playlist.Tracks)) p.Export.Total = int64(len(b.playlist.Tracks)) for _, track := range models.IterExportProgress(b.playlist.Tracks, &p, progress) { love, err := trackAsLove(track) @@ -164,11 +158,6 @@ func (b *JSPFBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time loves = append(loves, *love) p.Export.TotalItems += 1 } - - if len(loves) >= batchSize { - results <- models.LovesResult{Items: loves} - loves = loves[:0] - } } sort.Sort(loves) diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 84080ae..6d42f3c 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -30,8 +30,6 @@ import ( "go.uploadedlobster.com/scotty/pkg/scrobblerlog" ) -const batchSize = 1000 - type ScrobblerLogBackend struct { filePath string ignoreSkipped bool @@ -154,7 +152,7 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp return } - listens := make(models.ListensList, 0, batchSize) + listens := make(models.ListensList, 0, len(b.log.Records)) client := strings.Split(b.log.Client, " ")[0] p.Export.Total = int64(len(b.log.Records)) for _, record := range models.IterExportProgress(b.log.Records, &p, progress) { @@ -163,11 +161,6 @@ func (b *ScrobblerLogBackend) ExportListens(ctx context.Context, oldestTimestamp listens = append(listens, recordToListen(record, client)) p.Export.TotalItems += 1 } - - if len(listens) >= batchSize { - results <- models.ListensResult{Items: listens} - listens = listens[:0] - } } sort.Sort(listens) From 15755458e90ec6d33a55415d82494395c8b6889a Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 09:59:34 +0200 Subject: [PATCH 43/77] Fixed iterProgress not stopping if yield returns false --- internal/models/models.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/models/models.go b/internal/models/models.go index 94e3897..78d9965 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -267,7 +267,9 @@ func iterProgress[T any]( steps := max(len(items)/100, 1) return func(yield func(int, T) bool) { for i, item := range items { - yield(i, item) + if !yield(i, item) { + return + } p.Elapsed++ if i%steps == 0 { c <- *t From 3b9d07e6b589247626d20581956fdcaa5eceb83e Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 10:00:22 +0200 Subject: [PATCH 44/77] Implemented ScrobblerLog.ParseIter --- pkg/scrobblerlog/parser.go | 150 ++++++++++++++++++++------------ pkg/scrobblerlog/parser_test.go | 36 +++++++- 2 files changed, 129 insertions(+), 57 deletions(-) diff --git a/pkg/scrobblerlog/parser.go b/pkg/scrobblerlog/parser.go index 8bad56d..48fadcf 100644 --- a/pkg/scrobblerlog/parser.go +++ b/pkg/scrobblerlog/parser.go @@ -39,6 +39,7 @@ import ( "encoding/csv" "fmt" "io" + "iter" "strconv" "strings" "time" @@ -91,53 +92,36 @@ type ScrobblerLog struct { // The reader must provide a valid scrobbler log file with a valid header. // This function implicitly calls [ScrobblerLog.ReadHeader]. func (l *ScrobblerLog) Parse(data io.Reader, ignoreSkipped bool) error { - l.Records = make([]Record, 0) - - reader := bufio.NewReader(data) - err := l.readHeader(reader) + tsvReader, err := l.initReader(data) if err != nil { return err } - tsvReader := csv.NewReader(reader) - tsvReader.Comma = '\t' - // Row length is often flexible - tsvReader.FieldsPerRecord = -1 - - for { - // A row is: - // artistName releaseName trackName trackNumber duration rating timestamp recordingMBID - row, err := tsvReader.Read() - if err == io.EOF { - break - } else if err != nil { - return err - } - - // fmt.Printf("row: %v\n", row) - - // We consider only the last field (recording MBID) optional - // This was added in the 1.1 file format. - if len(row) < 7 { - line, _ := tsvReader.FieldPos(0) - return fmt.Errorf("invalid record in scrobblerlog line %v", line) - } - - record, err := l.rowToRecord(row) + for _, err := range l.iterRecords(tsvReader, ignoreSkipped) { if err != nil { return err } - - if ignoreSkipped && record.Rating == RatingSkipped { - continue - } - - l.Records = append(l.Records, record) } return nil } +// Parses a scrobbler log file from the given reader and returns an iterator over all records. +// +// The reader must provide a valid scrobbler log file with a valid header. +// This function implicitly calls [ScrobblerLog.ReadHeader]. +func (l *ScrobblerLog) ParseIter(data io.Reader, ignoreSkipped bool) iter.Seq2[Record, error] { + + tsvReader, err := l.initReader(data) + if err != nil { + return func(yield func(Record, error) bool) { + yield(Record{}, err) + } + } + + return l.iterRecords(tsvReader, ignoreSkipped) +} + // Append writes the given records to the writer. // // The writer should be for an existing scrobbler log file or @@ -177,6 +161,37 @@ func (l *ScrobblerLog) ReadHeader(reader io.Reader) error { return l.readHeader(bufio.NewReader(reader)) } +// Writes the header of a scrobbler log file to the given writer. +func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { + headers := []string{ + "#AUDIOSCROBBLER/1.1\n", + "#TZ/" + string(l.TZ) + "\n", + "#CLIENT/" + l.Client + "\n", + } + for _, line := range headers { + _, err := writer.Write([]byte(line)) + if err != nil { + return err + } + } + return nil +} + +func (l *ScrobblerLog) initReader(data io.Reader) (*csv.Reader, error) { + reader := bufio.NewReader(data) + err := l.readHeader(reader) + if err != nil { + return nil, err + } + + tsvReader := csv.NewReader(reader) + tsvReader.Comma = '\t' + // Row length is often flexible + tsvReader.FieldsPerRecord = -1 + + return tsvReader, nil +} + func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { // Skip header for i := 0; i < 3; i++ { @@ -215,37 +230,64 @@ func (l *ScrobblerLog) readHeader(reader *bufio.Reader) error { return nil } -// Writes the header of a scrobbler log file to the given writer. -func (l *ScrobblerLog) WriteHeader(writer io.Writer) error { - headers := []string{ - "#AUDIOSCROBBLER/1.1\n", - "#TZ/" + string(l.TZ) + "\n", - "#CLIENT/" + l.Client + "\n", - } - for _, line := range headers { - _, err := writer.Write([]byte(line)) - if err != nil { - return err +func (l *ScrobblerLog) iterRecords(reader *csv.Reader, ignoreSkipped bool) iter.Seq2[Record, error] { + return func(yield func(Record, error) bool) { + l.Records = make([]Record, 0) + for { + record, err := l.parseRow(reader) + if err == io.EOF { + break + } else if err != nil { + yield(Record{}, err) + break + } + + if ignoreSkipped && record.Rating == RatingSkipped { + continue + } + + l.Records = append(l.Records, *record) + if !yield(*record, nil) { + break + } } } - return nil } -func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { - var record Record +func (l *ScrobblerLog) parseRow(reader *csv.Reader) (*Record, error) { + // A row is: + // artistName releaseName trackName trackNumber duration rating timestamp recordingMBID + row, err := reader.Read() + if err != nil { + return nil, err + } + + // fmt.Printf("row: %v\n", row) + + // We consider only the last field (recording MBID) optional + // This was added in the 1.1 file format. + if len(row) < 7 { + line, _ := reader.FieldPos(0) + return nil, fmt.Errorf("invalid record in scrobblerlog line %v", line) + } + + return l.rowToRecord(row) +} + +func (l ScrobblerLog) rowToRecord(row []string) (*Record, error) { trackNumber, err := strconv.Atoi(row[3]) if err != nil { - return record, err + return nil, err } duration, err := strconv.Atoi(row[4]) if err != nil { - return record, err + return nil, err } timestamp, err := strconv.ParseInt(row[6], 10, 64) if err != nil { - return record, err + return nil, err } var timezone *time.Location = nil @@ -253,7 +295,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { timezone = l.FallbackTimezone } - record = Record{ + record := Record{ ArtistName: row[0], AlbumName: row[1], TrackName: row[2], @@ -267,7 +309,7 @@ func (l ScrobblerLog) rowToRecord(row []string) (Record, error) { record.MusicBrainzRecordingID = mbtypes.MBID(row[7]) } - return record, nil + return &record, nil } // Convert a Unix timestamp to a [time.Time] object, but treat the timestamp diff --git a/pkg/scrobblerlog/parser_test.go b/pkg/scrobblerlog/parser_test.go index 8dc30e5..26990f9 100644 --- a/pkg/scrobblerlog/parser_test.go +++ b/pkg/scrobblerlog/parser_test.go @@ -44,7 +44,14 @@ Kraftwerk Trans-Europe Express The Hall of Mirrors 2 474 S 1260358000 385ba9e9-6 Teeth Agency You Don't Have To Live In Pain Wolfs Jam 2 107 L 1260359404 1262beaf-19f8-4534-b9ed-7eef9ca8e83f ` -func TestParser(t *testing.T) { +var testScrobblerLogInvalid = `#AUDIOSCROBBLER/1.1 +#TZ/UNKNOWN +#CLIENT/Rockbox sansaclipplus $Revision$ +Özcan Deniz Ses ve Ayrilik Sevdanin rengi (sipacik) byMrTurkey 5 306 L 1260342084 +Özcan Deniz Hediye 2@V@7 Bir Dudaktan 1 210 L +` + +func TestParse(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{} @@ -68,7 +75,7 @@ func TestParser(t *testing.T) { record4.MusicBrainzRecordingID) } -func TestParserIgnoreSkipped(t *testing.T) { +func TestParseIgnoreSkipped(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{} @@ -81,7 +88,7 @@ func TestParserIgnoreSkipped(t *testing.T) { record4.MusicBrainzRecordingID) } -func TestParserFallbackTimezone(t *testing.T) { +func TestParseFallbackTimezone(t *testing.T) { assert := assert.New(t) data := bytes.NewBufferString(testScrobblerLog) result := scrobblerlog.ScrobblerLog{ @@ -96,6 +103,29 @@ func TestParserFallbackTimezone(t *testing.T) { ) } +func TestParseInvalid(t *testing.T) { + assert := assert.New(t) + data := bytes.NewBufferString(testScrobblerLogInvalid) + result := scrobblerlog.ScrobblerLog{} + err := result.Parse(data, true) + assert.ErrorContains(err, "invalid record in scrobblerlog line 2") +} + +func TestParseIter(t *testing.T) { + assert := assert.New(t) + data := bytes.NewBufferString(testScrobblerLog) + result := scrobblerlog.ScrobblerLog{} + records := make([]scrobblerlog.Record, 0) + for record, err := range result.ParseIter(data, false) { + require.NoError(t, err) + records = append(records, record) + } + + assert.Len(records, 5) + record1 := result.Records[0] + assert.Equal("Ses ve Ayrilik", record1.AlbumName) +} + func TestAppend(t *testing.T) { assert := assert.New(t) data := make([]byte, 0, 10) From 142d38e9db51122ca37a209a7b509a1095a45d63 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 10:10:08 +0200 Subject: [PATCH 45/77] Release 0.6.0 --- CHANGES.md | 6 +++--- internal/version/version.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 8dc9838..486d0ff 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,6 @@ # Scotty Changelog -## 0.6.0 - WIP +## 0.6.0 - 2025-05-23 - Fully reworked progress report - Cancel both export and import on error - Show progress bars as aborted on export / import error @@ -12,8 +12,8 @@ - JSPF: implemented export as loves and listens - JSPF: write track duration - JSPF: read username and recording MSID -- JSPF: add MusicBrainz playlist extension in append mode, if it does not exist - in the existing JSPF file +- JSPF: add MusicBrainz playlist extension in append mode, if it does not + exist in the existing JSPF file - scrobblerlog: fix timezone not being set from config (#6) - scrobblerlog: fix listen export not considering latest timestamp - Funkwhale: fix progress abort on error diff --git a/internal/version/version.go b/internal/version/version.go index b38a40f..f3bc081 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -17,7 +17,7 @@ package version const ( AppName = "scotty" - AppVersion = "0.5.2" + AppVersion = "0.6.0" AppURL = "https://git.sr.ht/~phw/scotty/" ) From 34b6bb9aa33c918b138546146f16a841319d3c21 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 00:37:05 +0200 Subject: [PATCH 46/77] Use filepath.Join instead of file.Join --- internal/backends/spotifyhistory/spotifyhistory.go | 3 +-- internal/config/config.go | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 76d0c9e..ce470ff 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -20,7 +20,6 @@ package spotifyhistory import ( "context" "os" - "path" "path/filepath" "slices" "sort" @@ -74,7 +73,7 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - files, err := filepath.Glob(path.Join(b.dirPath, historyFileGlob)) + files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob)) p := models.TransferProgress{ Export: &models.Progress{}, } diff --git a/internal/config/config.go b/internal/config/config.go index a529b92..94da799 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -19,7 +19,6 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "regexp" "strings" @@ -40,7 +39,7 @@ const ( func DefaultConfigDir() string { configDir, err := os.UserConfigDir() cobra.CheckErr(err) - return path.Join(configDir, version.AppName) + return filepath.Join(configDir, version.AppName) } // initConfig reads in config file and ENV variables if set. From 5c56e480f1a4a0b8d6e82beb747e9852fde738a3 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 16:33:28 +0200 Subject: [PATCH 47/77] Moved general LB related code to separate package --- .../backends/listenbrainz/listenbrainz.go | 54 ++++++++++--------- .../listenbrainz/listenbrainz_test.go | 11 ++-- .../backends => pkg}/listenbrainz/client.go | 5 +- .../listenbrainz/client_test.go | 14 ++--- .../backends => pkg}/listenbrainz/models.go | 15 +++--- .../listenbrainz/models_test.go | 4 +- .../listenbrainz/testdata/feedback.json | 0 .../listenbrainz/testdata/listen.json | 0 .../listenbrainz/testdata/listens.json | 0 .../listenbrainz/testdata/lookup.json | 0 10 files changed, 54 insertions(+), 49 deletions(-) rename {internal/backends => pkg}/listenbrainz/client.go (96%) rename {internal/backends => pkg}/listenbrainz/client_test.go (93%) rename {internal/backends => pkg}/listenbrainz/models.go (91%) rename {internal/backends => pkg}/listenbrainz/models_test.go (97%) rename {internal/backends => pkg}/listenbrainz/testdata/feedback.json (100%) rename {internal/backends => pkg}/listenbrainz/testdata/listen.json (100%) rename {internal/backends => pkg}/listenbrainz/testdata/listens.json (100%) rename {internal/backends => pkg}/listenbrainz/testdata/lookup.json (100%) diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index bf46c22..5e80a10 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -29,10 +29,11 @@ import ( "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/internal/similarity" "go.uploadedlobster.com/scotty/internal/version" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) type ListenBrainzApiBackend struct { - client Client + client listenbrainz.Client mbClient musicbrainzws2.Client username string checkDuplicates bool @@ -58,13 +59,13 @@ func (b *ListenBrainzApiBackend) Options() []models.BackendOption { } func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error { - b.client = NewClient(config.GetString("token")) + b.client = listenbrainz.NewClient(config.GetString("token"), version.UserAgent()) b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: version.AppName, Version: version.AppVersion, URL: version.AppURL, }) - b.client.MaxResults = MaxItemsPerGet + b.client.MaxResults = listenbrainz.MaxItemsPerGet b.username = config.GetString("username") b.checkDuplicates = config.GetBool("check-duplicate-listens", false) return nil @@ -116,7 +117,7 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest for _, listen := range result.Payload.Listens { if listen.ListenedAt > oldestTimestamp.Unix() { - listens = append(listens, listen.AsListen()) + listens = append(listens, AsListen(listen)) } else { // result contains listens older then oldestTimestamp break @@ -138,16 +139,16 @@ func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimest func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { total := len(export.Items) p := models.TransferProgress{}.FromImportResult(importResult, false) - for i := 0; i < total; i += MaxListensPerRequest { - listens := export.Items[i:min(i+MaxListensPerRequest, total)] + for i := 0; i < total; i += listenbrainz.MaxListensPerRequest { + listens := export.Items[i:min(i+listenbrainz.MaxListensPerRequest, total)] count := len(listens) if count == 0 { break } - submission := ListenSubmission{ - ListenType: Import, - Payload: make([]Listen, 0, count), + submission := listenbrainz.ListenSubmission{ + ListenType: listenbrainz.Import, + Payload: make([]listenbrainz.Listen, 0, count), } for _, l := range listens { @@ -167,9 +168,9 @@ func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export model } l.FillAdditionalInfo() - listen := Listen{ + listen := listenbrainz.Listen{ ListenedAt: l.ListenedAt.Unix(), - TrackMetadata: Track{ + TrackMetadata: listenbrainz.Track{ TrackName: l.TrackName, ReleaseName: l.ReleaseName, ArtistName: l.ArtistName(), @@ -228,7 +229,7 @@ func (b *ListenBrainzApiBackend) ExportLoves(ctx context.Context, oldestTimestam func (b *ListenBrainzApiBackend) exportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult) { offset := 0 defer close(results) - loves := make(models.LovesList, 0, 2*MaxItemsPerGet) + loves := make(models.LovesList, 0, 2*listenbrainz.MaxItemsPerGet) out: for { @@ -254,7 +255,7 @@ out: } } - love := feedback.AsLove() + love := AsLove(feedback) if love.Created.After(oldestTimestamp) { loves = append(loves, love) } else { @@ -262,7 +263,7 @@ out: } } - offset += MaxItemsPerGet + offset += listenbrainz.MaxItemsPerGet } sort.Sort(loves) @@ -278,7 +279,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models. go b.exportLoves(ctx, time.Unix(0, 0), existingLovesChan) // TODO: Store MBIDs directly - b.existingMBIDs = make(map[mbtypes.MBID]bool, MaxItemsPerGet) + b.existingMBIDs = make(map[mbtypes.MBID]bool, listenbrainz.MaxItemsPerGet) for existingLoves := range existingLovesChan { if existingLoves.Error != nil { @@ -316,7 +317,7 @@ func (b *ListenBrainzApiBackend) ImportLoves(ctx context.Context, export models. if b.existingMBIDs[recordingMBID] { ok = true } else { - resp, err := b.client.SendFeedback(ctx, Feedback{ + resp, err := b.client.SendFeedback(ctx, listenbrainz.Feedback{ RecordingMBID: recordingMBID, Score: 1, }) @@ -366,7 +367,7 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste } for _, c := range candidates.Payload.Listens { - sim := similarity.CompareTracks(listen.Track, c.TrackMetadata.AsTrack()) + sim := similarity.CompareTracks(listen.Track, AsTrack(c.TrackMetadata)) if sim >= trackSimilarityThreshold { return true, nil } @@ -375,7 +376,8 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste return false, nil } -func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtypes.MBID) (*Track, error) { +func (b *ListenBrainzApiBackend) lookupRecording( + ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) { filter := musicbrainzws2.IncludesFilter{ Includes: []string{"artist-credits"}, } @@ -388,10 +390,10 @@ func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtyp for _, artist := range recording.ArtistCredit { artistMBIDs = append(artistMBIDs, artist.Artist.ID) } - track := Track{ + track := listenbrainz.Track{ TrackName: recording.Title, ArtistName: recording.ArtistCredit.String(), - MBIDMapping: &MBIDMapping{ + MBIDMapping: &listenbrainz.MBIDMapping{ // In case of redirects this MBID differs from the looked up MBID RecordingMBID: recording.ID, ArtistMBIDs: artistMBIDs, @@ -400,26 +402,26 @@ func (b *ListenBrainzApiBackend) lookupRecording(ctx context.Context, mbid mbtyp return &track, nil } -func (lbListen Listen) AsListen() models.Listen { +func AsListen(lbListen listenbrainz.Listen) models.Listen { listen := models.Listen{ ListenedAt: time.Unix(lbListen.ListenedAt, 0), UserName: lbListen.UserName, - Track: lbListen.TrackMetadata.AsTrack(), + Track: AsTrack(lbListen.TrackMetadata), } return listen } -func (f Feedback) AsLove() models.Love { +func AsLove(f listenbrainz.Feedback) models.Love { recordingMBID := f.RecordingMBID track := f.TrackMetadata if track == nil { - track = &Track{} + track = &listenbrainz.Track{} } love := models.Love{ UserName: f.UserName, RecordingMBID: recordingMBID, Created: time.Unix(f.Created, 0), - Track: track.AsTrack(), + Track: AsTrack(*track), } if love.Track.RecordingMBID == "" { @@ -429,7 +431,7 @@ func (f Feedback) AsLove() models.Love { return love } -func (t Track) AsTrack() models.Track { +func AsTrack(t listenbrainz.Track) models.Track { track := models.Track{ TrackName: t.TrackName, ReleaseName: t.ReleaseName, diff --git a/internal/backends/listenbrainz/listenbrainz_test.go b/internal/backends/listenbrainz/listenbrainz_test.go index bf2e4d3..dd3e1d3 100644 --- a/internal/backends/listenbrainz/listenbrainz_test.go +++ b/internal/backends/listenbrainz/listenbrainz_test.go @@ -24,15 +24,16 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) func TestInitConfig(t *testing.T) { c := viper.New() c.Set("token", "thetoken") service := config.NewServiceConfig("test", c) - backend := listenbrainz.ListenBrainzApiBackend{} + backend := lbapi.ListenBrainzApiBackend{} err := backend.InitConfig(&service) assert.NoError(t, err) } @@ -57,7 +58,7 @@ func TestListenBrainzListenAsListen(t *testing.T) { }, }, } - listen := lbListen.AsListen() + listen := lbapi.AsListen(lbListen) assert.Equal(t, time.Unix(1699289873, 0), listen.ListenedAt) assert.Equal(t, lbListen.UserName, listen.UserName) assert.Equal(t, time.Duration(413787*time.Millisecond), listen.Duration) @@ -93,7 +94,7 @@ func TestListenBrainzFeedbackAsLove(t *testing.T) { }, }, } - love := feedback.AsLove() + love := lbapi.AsLove(feedback) assert := assert.New(t) assert.Equal(time.Unix(1699859066, 0).Unix(), love.Created.Unix()) assert.Equal(feedback.UserName, love.UserName) @@ -114,7 +115,7 @@ func TestListenBrainzPartialFeedbackAsLove(t *testing.T) { RecordingMBID: recordingMBID, Score: 1, } - love := feedback.AsLove() + love := lbapi.AsLove(feedback) assert := assert.New(t) assert.Equal(time.Unix(1699859066, 0).Unix(), love.Created.Unix()) assert.Equal(recordingMBID, love.RecordingMBID) diff --git a/internal/backends/listenbrainz/client.go b/pkg/listenbrainz/client.go similarity index 96% rename from internal/backends/listenbrainz/client.go rename to pkg/listenbrainz/client.go index d1a1fa6..957a946 100644 --- a/internal/backends/listenbrainz/client.go +++ b/pkg/listenbrainz/client.go @@ -28,7 +28,6 @@ import ( "time" "github.com/go-resty/resty/v2" - "go.uploadedlobster.com/scotty/internal/version" "go.uploadedlobster.com/scotty/pkg/ratelimit" ) @@ -44,13 +43,13 @@ type Client struct { MaxResults int } -func NewClient(token string) Client { +func NewClient(token string, userAgent string) Client { client := resty.New() client.SetBaseURL(listenBrainzBaseURL) client.SetAuthScheme("Token") client.SetAuthToken(token) client.SetHeader("Accept", "application/json") - client.SetHeader("User-Agent", version.UserAgent()) + client.SetHeader("User-Agent", userAgent) // Handle rate limiting (see https://listenbrainz.readthedocs.io/en/latest/users/api/index.html#rate-limiting) ratelimit.EnableHTTPHeaderRateLimit(client, "X-RateLimit-Reset-In") diff --git a/internal/backends/listenbrainz/client_test.go b/pkg/listenbrainz/client_test.go similarity index 93% rename from internal/backends/listenbrainz/client_test.go rename to pkg/listenbrainz/client_test.go index 45bb0de..3742ca9 100644 --- a/internal/backends/listenbrainz/client_test.go +++ b/pkg/listenbrainz/client_test.go @@ -31,12 +31,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) func TestNewClient(t *testing.T) { token := "foobar123" - client := listenbrainz.NewClient(token) + client := listenbrainz.NewClient(token, "test/1.0") assert.Equal(t, token, client.HTTPClient.Token) assert.Equal(t, listenbrainz.DefaultItemsPerGet, client.MaxResults) } @@ -44,7 +44,7 @@ func TestNewClient(t *testing.T) { func TestGetListens(t *testing.T) { defer httpmock.DeactivateAndReset() - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") client.MaxResults = 2 setupHTTPMock(t, client.HTTPClient.GetClient(), "https://api.listenbrainz.org/1/user/outsidecontext/listens", @@ -64,7 +64,7 @@ func TestGetListens(t *testing.T) { } func TestSubmitListens(t *testing.T) { - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") httpmock.ActivateNonDefault(client.HTTPClient.GetClient()) responder, err := httpmock.NewJsonResponder(200, listenbrainz.StatusResult{ @@ -104,7 +104,7 @@ func TestSubmitListens(t *testing.T) { func TestGetFeedback(t *testing.T) { defer httpmock.DeactivateAndReset() - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") client.MaxResults = 2 setupHTTPMock(t, client.HTTPClient.GetClient(), "https://api.listenbrainz.org/1/feedback/user/outsidecontext/get-feedback", @@ -123,7 +123,7 @@ func TestGetFeedback(t *testing.T) { } func TestSendFeedback(t *testing.T) { - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") httpmock.ActivateNonDefault(client.HTTPClient.GetClient()) responder, err := httpmock.NewJsonResponder(200, listenbrainz.StatusResult{ @@ -149,7 +149,7 @@ func TestSendFeedback(t *testing.T) { func TestLookup(t *testing.T) { defer httpmock.DeactivateAndReset() - client := listenbrainz.NewClient("thetoken") + client := listenbrainz.NewClient("thetoken", "test/1.0") setupHTTPMock(t, client.HTTPClient.GetClient(), "https://api.listenbrainz.org/1/metadata/lookup", "testdata/lookup.json") diff --git a/internal/backends/listenbrainz/models.go b/pkg/listenbrainz/models.go similarity index 91% rename from internal/backends/listenbrainz/models.go rename to pkg/listenbrainz/models.go index ada75d3..2dac432 100644 --- a/internal/backends/listenbrainz/models.go +++ b/pkg/listenbrainz/models.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -66,16 +66,19 @@ type Track struct { TrackName string `json:"track_name,omitempty"` ArtistName string `json:"artist_name,omitempty"` ReleaseName string `json:"release_name,omitempty"` + RecordingMSID string `json:"recording_msid,omitempty"` AdditionalInfo map[string]any `json:"additional_info,omitempty"` MBIDMapping *MBIDMapping `json:"mbid_mapping,omitempty"` } type MBIDMapping struct { - RecordingName string `json:"recording_name,omitempty"` - RecordingMBID mbtypes.MBID `json:"recording_mbid,omitempty"` - ReleaseMBID mbtypes.MBID `json:"release_mbid,omitempty"` - ArtistMBIDs []mbtypes.MBID `json:"artist_mbids,omitempty"` - Artists []Artist `json:"artists,omitempty"` + ArtistMBIDs []mbtypes.MBID `json:"artist_mbids,omitempty"` + Artists []Artist `json:"artists,omitempty"` + RecordingMBID mbtypes.MBID `json:"recording_mbid,omitempty"` + RecordingName string `json:"recording_name,omitempty"` + ReleaseMBID mbtypes.MBID `json:"release_mbid,omitempty"` + CAAID int `json:"caa_id,omitempty"` + CAAReleaseMBID mbtypes.MBID `json:"caa_release_mbid,omitempty"` } type Artist struct { diff --git a/internal/backends/listenbrainz/models_test.go b/pkg/listenbrainz/models_test.go similarity index 97% rename from internal/backends/listenbrainz/models_test.go rename to pkg/listenbrainz/models_test.go index 02cbe98..8fb4994 100644 --- a/internal/backends/listenbrainz/models_test.go +++ b/pkg/listenbrainz/models_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) func TestTrackDurationMillisecondsInt(t *testing.T) { diff --git a/internal/backends/listenbrainz/testdata/feedback.json b/pkg/listenbrainz/testdata/feedback.json similarity index 100% rename from internal/backends/listenbrainz/testdata/feedback.json rename to pkg/listenbrainz/testdata/feedback.json diff --git a/internal/backends/listenbrainz/testdata/listen.json b/pkg/listenbrainz/testdata/listen.json similarity index 100% rename from internal/backends/listenbrainz/testdata/listen.json rename to pkg/listenbrainz/testdata/listen.json diff --git a/internal/backends/listenbrainz/testdata/listens.json b/pkg/listenbrainz/testdata/listens.json similarity index 100% rename from internal/backends/listenbrainz/testdata/listens.json rename to pkg/listenbrainz/testdata/listens.json diff --git a/internal/backends/listenbrainz/testdata/lookup.json b/pkg/listenbrainz/testdata/lookup.json similarity index 100% rename from internal/backends/listenbrainz/testdata/lookup.json rename to pkg/listenbrainz/testdata/lookup.json From 92e7216fac128ef0607a46adef6d8df456e12814 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 23 May 2025 19:03:06 +0200 Subject: [PATCH 48/77] Implemented listenbrainz-archive backend with listen export support --- README.md | 25 +- config.example.toml | 7 + go.mod | 1 + go.sum | 2 + internal/backends/backends.go | 24 +- internal/backends/backends_test.go | 6 + internal/backends/lbarchive/lbarchive.go | 121 ++++++++ internal/backends/lbarchive/lbarchive_test.go | 40 +++ pkg/listenbrainz/archive.go | 267 ++++++++++++++++++ pkg/listenbrainz/models.go | 10 +- 10 files changed, 475 insertions(+), 28 deletions(-) create mode 100644 internal/backends/lbarchive/lbarchive.go create mode 100644 internal/backends/lbarchive/lbarchive_test.go create mode 100644 pkg/listenbrainz/archive.go diff --git a/README.md b/README.md index c764730..6f997ed 100644 --- a/README.md +++ b/README.md @@ -117,18 +117,19 @@ scotty beam listens deezer listenbrainz --timestamp "2023-12-06 14:26:24" ### Supported backends The following table lists the available backends and the currently supported features. -Backend | Listens Export | Listens Import | Loves Export | Loves Import -----------------|----------------|----------------|--------------|------------- -deezer | ✓ | ⨯ | ✓ | - -funkwhale | ✓ | ⨯ | ✓ | - -jspf | ✓ | ✓ | ✓ | ✓ -lastfm | ✓ | ✓ | ✓ | ✓ -listenbrainz | ✓ | ✓ | ✓ | ✓ -maloja | ✓ | ✓ | ⨯ | ⨯ -scrobbler-log | ✓ | ✓ | ⨯ | ⨯ -spotify | ✓ | ⨯ | ✓ | - -spotify-history | ✓ | ⨯ | ⨯ | ⨯ -subsonic | ⨯ | ⨯ | ✓ | - +Backend | Listens Export | Listens Import | Loves Export | Loves Import +---------------------|----------------|----------------|--------------|------------- +deezer | ✓ | ⨯ | ✓ | - +funkwhale | ✓ | ⨯ | ✓ | - +jspf | ✓ | ✓ | ✓ | ✓ +lastfm | ✓ | ✓ | ✓ | ✓ +listenbrainz | ✓ | ✓ | ✓ | ✓ +listenbrainz-archive | ✓ | - | - | - +maloja | ✓ | ✓ | ⨯ | ⨯ +scrobbler-log | ✓ | ✓ | ⨯ | ⨯ +spotify | ✓ | ⨯ | ✓ | - +spotify-history | ✓ | ⨯ | ⨯ | ⨯ +subsonic | ⨯ | ⨯ | ✓ | - **✓** implemented **-** not yet implemented **⨯** unavailable / not planned diff --git a/config.example.toml b/config.example.toml index 6b81bac..40ffd18 100644 --- a/config.example.toml +++ b/config.example.toml @@ -19,6 +19,13 @@ token = "" # not already exists in your ListenBrainz profile. check-duplicate-listens = false +[service.listenbrainz-archive] +# This backend supports listens from a ListenBrainz export archive +# (https://listenbrainz.org/settings/export/). +backend = "listenbrainz-archive" +# The file path to the ListenBrainz export archive. +file-path = "./listenbrainz_outsidecontext.zip" + [service.maloja] # Maloja is a self hosted listening service (https://github.com/krateng/maloja) backend = "maloja" diff --git a/go.mod b/go.mod index a00b416..ccdb6cc 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/go.sum b/go.sum index 3cd01a6..028515c 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFT github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 h1:cgqwZtnR+IQfUYDLJ3Kiy4aE+O/wExTzEIg8xwC4Qfs= github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0/go.mod h1:n3nudMl178cEvD44PaopxH9jhJaQzthSxUzLO5iKMy4= +github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 h1:CXJI+lliMiiEwzfgE8yt/38K0heYDgQ0L3f/3fxRnQU= +github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740/go.mod h1:G4w16caPmc6at7u4fmkj/8OAoOnM9mkmJr2fvL0vhaw= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= diff --git a/internal/backends/backends.go b/internal/backends/backends.go index a9c3292..a1cd407 100644 --- a/internal/backends/backends.go +++ b/internal/backends/backends.go @@ -27,6 +27,7 @@ import ( "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" "go.uploadedlobster.com/scotty/internal/backends/lastfm" + "go.uploadedlobster.com/scotty/internal/backends/lbarchive" "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/backends/maloja" "go.uploadedlobster.com/scotty/internal/backends/scrobblerlog" @@ -105,17 +106,18 @@ func GetBackends() BackendList { } var knownBackends = map[string]func() models.Backend{ - "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, - "dump": func() models.Backend { return &dump.DumpBackend{} }, - "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, - "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, - "lastfm": func() models.Backend { return &lastfm.LastfmApiBackend{} }, - "listenbrainz": func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} }, - "maloja": func() models.Backend { return &maloja.MalojaApiBackend{} }, - "scrobbler-log": func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} }, - "spotify": func() models.Backend { return &spotify.SpotifyApiBackend{} }, - "spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} }, - "subsonic": func() models.Backend { return &subsonic.SubsonicApiBackend{} }, + "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, + "dump": func() models.Backend { return &dump.DumpBackend{} }, + "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, + "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, + "lastfm": func() models.Backend { return &lastfm.LastfmApiBackend{} }, + "listenbrainz": func() models.Backend { return &listenbrainz.ListenBrainzApiBackend{} }, + "listenbrainz-archive": func() models.Backend { return &lbarchive.ListenBrainzArchiveBackend{} }, + "maloja": func() models.Backend { return &maloja.MalojaApiBackend{} }, + "scrobbler-log": func() models.Backend { return &scrobblerlog.ScrobblerLogBackend{} }, + "spotify": func() models.Backend { return &spotify.SpotifyApiBackend{} }, + "spotify-history": func() models.Backend { return &spotifyhistory.SpotifyHistoryBackend{} }, + "subsonic": func() models.Backend { return &subsonic.SubsonicApiBackend{} }, } func backendWithConfig(config config.ServiceConfig) (models.Backend, error) { diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index e115636..737c7e3 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -28,6 +28,7 @@ import ( "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" "go.uploadedlobster.com/scotty/internal/backends/lastfm" + "go.uploadedlobster.com/scotty/internal/backends/lbarchive" "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/backends/maloja" "go.uploadedlobster.com/scotty/internal/backends/scrobblerlog" @@ -103,6 +104,11 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &lastfm.LastfmApiBackend{}) expectInterface[models.LovesImport](t, &lastfm.LastfmApiBackend{}) + expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{}) + expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{}) expectInterface[models.ListensImport](t, &listenbrainz.ListenBrainzApiBackend{}) expectInterface[models.LovesExport](t, &listenbrainz.ListenBrainzApiBackend{}) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go new file mode 100644 index 0000000..143a674 --- /dev/null +++ b/internal/backends/lbarchive/lbarchive.go @@ -0,0 +1,121 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package lbarchive + +import ( + "context" + "time" + + lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" + "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/models" + "go.uploadedlobster.com/scotty/pkg/listenbrainz" +) + +const batchSize = 2000 + +type ListenBrainzArchiveBackend struct { + filePath string +} + +func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" } + +func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { + return []models.BackendOption{{ + Name: "file-path", + Label: i18n.Tr("Export ZIP file path"), + Type: models.String, + }} +} + +func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { + b.filePath = config.GetString("file-path") + return nil +} + +func (b *ListenBrainzArchiveBackend) ExportListens( + ctx context.Context, oldestTimestamp time.Time, + results chan models.ListensResult, progress chan models.TransferProgress) { + startTime := time.Now() + minTime := oldestTimestamp + if minTime.Unix() < 1 { + minTime = time.Unix(1, 0) + } + + totalDuration := startTime.Sub(oldestTimestamp) + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } + + archive, err := listenbrainz.OpenArchive(b.filePath) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + defer archive.Close() + + userInfo, err := archive.UserInfo() + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + listens := make(models.ListensList, 0, batchSize) + for rawListen, err := range archive.IterListens(oldestTimestamp) { + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + listen := lbapi.AsListen(rawListen) + if listen.UserName == "" { + listen.UserName = userInfo.Name + } + listens = append(listens, listen) + + // Update the progress + p.Export.TotalItems += 1 + remainingTime := startTime.Sub(listen.ListenedAt) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + + // Allow the importer to start processing the listens by + // sending them in batches. + if len(listens) >= batchSize { + results <- models.ListensResult{Items: listens} + progress <- p + listens = listens[:0] + } + } + + results <- models.ListensResult{Items: listens} + p.Export.Complete() + progress <- p +} diff --git a/internal/backends/lbarchive/lbarchive_test.go b/internal/backends/lbarchive/lbarchive_test.go new file mode 100644 index 0000000..b7e164a --- /dev/null +++ b/internal/backends/lbarchive/lbarchive_test.go @@ -0,0 +1,40 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package lbarchive_test + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "go.uploadedlobster.com/scotty/internal/backends/lbarchive" + "go.uploadedlobster.com/scotty/internal/config" +) + +func TestInitConfig(t *testing.T) { + c := viper.New() + c.Set("file-path", "/foo/lbarchive.zip") + service := config.NewServiceConfig("test", c) + backend := lbarchive.ListenBrainzArchiveBackend{} + err := backend.InitConfig(&service) + assert.NoError(t, err) +} diff --git a/pkg/listenbrainz/archive.go b/pkg/listenbrainz/archive.go new file mode 100644 index 0000000..668b7e1 --- /dev/null +++ b/pkg/listenbrainz/archive.go @@ -0,0 +1,267 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package listenbrainz + +import ( + "archive/zip" + "encoding/json" + "fmt" + "io" + "iter" + "os" + "regexp" + "sort" + "strconv" + "time" + + "github.com/simonfrey/jsonl" +) + +// Represents a ListenBrainz export archive. +// +// The export contains the user's listen history, favorite tracks and +// user information. +type Archive struct { + backend archiveBackend +} + +// Close the archive and release any resources. +func (a *Archive) Close() error { + return a.backend.Close() +} + +// Read the user information from the archive. +func (a *Archive) UserInfo() (UserInfo, error) { + f, err := a.backend.OpenUserInfoFile() + if err != nil { + return UserInfo{}, err + } + defer f.Close() + + userInfo := UserInfo{} + bytes, err := io.ReadAll(f) + if err != nil { + return userInfo, err + } + + json.Unmarshal(bytes, &userInfo) + return userInfo, nil +} + +// Yields all listens from the archive that are newer than the given timestamp. +// The listens are yielded in ascending order of their listened_at timestamp. +func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { + return func(yield func(Listen, error) bool) { + files, err := a.backend.ListListenExports() + if err != nil { + yield(Listen{}, err) + return + } + + sort.Slice(files, func(i, j int) bool { + return files[i].TimeRange.Start.Before(files[j].TimeRange.Start) + }) + + for _, file := range files { + if file.TimeRange.End.Before(minTimestamp) { + continue + } + + f := NewExportFile(file.f) + for l, err := range f.IterListens() { + if err != nil { + yield(Listen{}, err) + return + } + + if !time.Unix(l.ListenedAt, 0).After(minTimestamp) { + continue + } + if !yield(l, nil) { + break + } + } + } + } +} + +// Open a ListenBrainz archive from file path. +func OpenArchive(path string) (*Archive, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + switch mode := fi.Mode(); { + case mode.IsRegular(): + backend := &zipArchive{} + err := backend.Open(path) + if err != nil { + return nil, err + } + return &Archive{backend: backend}, nil + case mode.IsDir(): + // TODO: Implement directory mode + return nil, fmt.Errorf("directory mode not implemented") + default: + return nil, fmt.Errorf("unsupported file mode: %s", mode) + } +} + +type UserInfo struct { + ID string `json:"user_id"` + Name string `json:"username"` +} + +type archiveBackend interface { + Close() error + OpenUserInfoFile() (io.ReadCloser, error) + ListListenExports() ([]ListenExportFileInfo, error) +} + +type timeRange struct { + Start time.Time + End time.Time +} + +type openableFile interface { + Open() (io.ReadCloser, error) +} + +type ListenExportFileInfo struct { + Name string + TimeRange timeRange + f openableFile +} + +type zipArchive struct { + zip *zip.ReadCloser +} + +func (a *zipArchive) Open(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) OpenUserInfoFile() (io.ReadCloser, error) { + file, err := a.zip.Open("user.json") + if err != nil { + return nil, err + } + return file, nil +} + +func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) { + re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) + result := make([]ListenExportFileInfo, 0) + + for _, file := range a.zip.File { + match := re.FindStringSubmatch(file.Name) + if match == nil { + continue + } + + year := match[1] + month := match[2] + times, err := getMonthTimeRange(year, month) + if err != nil { + return nil, err + } + info := ListenExportFileInfo{ + Name: file.Name, + TimeRange: *times, + f: file, + } + result = append(result, info) + } + + return result, nil +} + +type ListenExportFile struct { + file openableFile +} + +func NewExportFile(f openableFile) ListenExportFile { + return ListenExportFile{file: f} +} + +func (f *ListenExportFile) openReader() (*jsonl.Reader, error) { + fio, err := f.file.Open() + if err != nil { + return nil, err + } + reader := jsonl.NewReader(fio) + return &reader, nil +} + +func (f *ListenExportFile) IterListens() iter.Seq2[Listen, error] { + return func(yield func(Listen, error) bool) { + reader, err := f.openReader() + if err != nil { + yield(Listen{}, err) + return + } + defer reader.Close() + + for { + listen := Listen{} + err := reader.ReadSingleLine(&listen) + if err != nil { + break + } + if !yield(listen, nil) { + break + } + } + } +} + +func getMonthTimeRange(year string, month string) (*timeRange, error) { + yearInt, err := strconv.Atoi(year) + if err != nil { + return nil, err + } + monthInt, err := strconv.Atoi(month) + if err != nil { + return nil, err + } + + r := &timeRange{} + r.Start = time.Date(yearInt, time.Month(monthInt), 1, 0, 0, 0, 0, time.UTC) + + // Get the end of the month + nextMonth := monthInt + 1 + r.End = time.Date( + yearInt, time.Month(nextMonth), 1, 0, 0, 0, 0, time.UTC).Add(-time.Second) + return r, nil +} diff --git a/pkg/listenbrainz/models.go b/pkg/listenbrainz/models.go index 2dac432..0b5f439 100644 --- a/pkg/listenbrainz/models.go +++ b/pkg/listenbrainz/models.go @@ -55,11 +55,11 @@ type ListenSubmission struct { } type Listen struct { - InsertedAt int64 `json:"inserted_at,omitempty"` - ListenedAt int64 `json:"listened_at"` - RecordingMSID string `json:"recording_msid,omitempty"` - UserName string `json:"user_name,omitempty"` - TrackMetadata Track `json:"track_metadata"` + InsertedAt float64 `json:"inserted_at,omitempty"` + ListenedAt int64 `json:"listened_at"` + RecordingMSID string `json:"recording_msid,omitempty"` + UserName string `json:"user_name,omitempty"` + TrackMetadata Track `json:"track_metadata"` } type Track struct { From 424305518b49d50107a57caf190a10b79af6dfc7 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 00:21:46 +0200 Subject: [PATCH 49/77] Implemented directory mode for listenbrainz-archive --- pkg/listenbrainz/archive.go | 139 +++++++++++++++++++++++++++++------- 1 file changed, 112 insertions(+), 27 deletions(-) diff --git a/pkg/listenbrainz/archive.go b/pkg/listenbrainz/archive.go index 668b7e1..de34ba8 100644 --- a/pkg/listenbrainz/archive.go +++ b/pkg/listenbrainz/archive.go @@ -28,6 +28,7 @@ import ( "io" "iter" "os" + "path/filepath" "regexp" "sort" "strconv" @@ -51,7 +52,7 @@ func (a *Archive) Close() error { // Read the user information from the archive. func (a *Archive) UserInfo() (UserInfo, error) { - f, err := a.backend.OpenUserInfoFile() + f, err := a.backend.OpenFile("user.json") if err != nil { return UserInfo{}, err } @@ -67,11 +68,43 @@ func (a *Archive) UserInfo() (UserInfo, error) { return userInfo, nil } +func (a *Archive) ListListenExports() ([]ListenExportFileInfo, error) { + re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) + result := make([]ListenExportFileInfo, 0) + + files, err := a.backend.Glob("listens/*/*.jsonl") + if err != nil { + return nil, err + } + + for _, file := range files { + match := re.FindStringSubmatch(file.Name) + if match == nil { + continue + } + + year := match[1] + month := match[2] + times, err := getMonthTimeRange(year, month) + if err != nil { + return nil, err + } + info := ListenExportFileInfo{ + Name: file.Name, + TimeRange: *times, + f: file.File, + } + result = append(result, info) + } + + return result, nil +} + // Yields all listens from the archive that are newer than the given timestamp. // The listens are yielded in ascending order of their listened_at timestamp. func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { return func(yield func(Listen, error) bool) { - files, err := a.backend.ListListenExports() + files, err := a.ListListenExports() if err != nil { yield(Listen{}, err) return @@ -119,8 +152,12 @@ func OpenArchive(path string) (*Archive, error) { } return &Archive{backend: backend}, nil case mode.IsDir(): - // TODO: Implement directory mode - return nil, fmt.Errorf("directory mode not implemented") + backend := &dirArchive{} + err := backend.Open(path) + if err != nil { + return nil, err + } + return &Archive{backend: backend}, nil default: return nil, fmt.Errorf("unsupported file mode: %s", mode) } @@ -133,8 +170,8 @@ type UserInfo struct { type archiveBackend interface { Close() error - OpenUserInfoFile() (io.ReadCloser, error) - ListListenExports() ([]ListenExportFileInfo, error) + OpenFile(path string) (io.ReadCloser, error) + Glob(pattern string) ([]FileInfo, error) } type timeRange struct { @@ -142,16 +179,30 @@ type timeRange struct { End time.Time } -type openableFile interface { +type OpenableFile interface { Open() (io.ReadCloser, error) } +type FileInfo struct { + Name string + File OpenableFile +} + +type FilesystemFile struct { + path string +} + +func (f *FilesystemFile) Open() (io.ReadCloser, error) { + return os.Open(f.path) +} + type ListenExportFileInfo struct { Name string TimeRange timeRange - f openableFile + f OpenableFile } +// An implementation of the archiveBackend interface for zip files. type zipArchive struct { zip *zip.ReadCloser } @@ -172,34 +223,68 @@ func (a *zipArchive) Close() error { return a.zip.Close() } -func (a *zipArchive) OpenUserInfoFile() (io.ReadCloser, error) { - file, err := a.zip.Open("user.json") +func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { + result := make([]FileInfo, 0) + for _, file := range a.zip.File { + if matched, err := filepath.Match(pattern, file.Name); matched { + if err != nil { + return nil, err + } + info := FileInfo{ + Name: file.Name, + File: file, + } + result = append(result, info) + } + } + + return result, nil +} + +func (a *zipArchive) OpenFile(path string) (io.ReadCloser, error) { + file, err := a.zip.Open(path) if err != nil { return nil, err } return file, nil } -func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) { - re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) - result := make([]ListenExportFileInfo, 0) +// An implementation of the archiveBackend interface for directories. +type dirArchive struct { + dir string +} - for _, file := range a.zip.File { - match := re.FindStringSubmatch(file.Name) - if match == nil { - continue - } +func (a *dirArchive) Open(path string) error { + a.dir = filepath.Clean(path) + return nil +} - year := match[1] - month := match[2] - times, err := getMonthTimeRange(year, month) +func (a *dirArchive) Close() error { + return nil +} + +func (a *dirArchive) OpenFile(path string) (io.ReadCloser, error) { + file, err := os.Open(filepath.Join(a.dir, path)) + if err != nil { + return nil, err + } + return file, nil +} + +func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { + files, err := filepath.Glob(filepath.Join(a.dir, pattern)) + if err != nil { + return nil, err + } + result := make([]FileInfo, 0) + for _, filename := range files { + name, err := filepath.Rel(a.dir, filename) if err != nil { return nil, err } - info := ListenExportFileInfo{ - Name: file.Name, - TimeRange: *times, - f: file, + info := FileInfo{ + Name: name, + File: &FilesystemFile{path: filename}, } result = append(result, info) } @@ -208,10 +293,10 @@ func (a *zipArchive) ListListenExports() ([]ListenExportFileInfo, error) { } type ListenExportFile struct { - file openableFile + file OpenableFile } -func NewExportFile(f openableFile) ListenExportFile { +func NewExportFile(f OpenableFile) ListenExportFile { return ListenExportFile{file: f} } From 1025277ba91c8e92ea8f4ec9935a723c4b19532b Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 00:35:55 +0200 Subject: [PATCH 50/77] Moved generic archive abstraction into separate package --- internal/archive/archive.go | 181 ++++++++++++++++++++++++++++++++++++ pkg/listenbrainz/archive.go | 162 +++----------------------------- 2 files changed, 196 insertions(+), 147 deletions(-) create mode 100644 internal/archive/archive.go diff --git a/internal/archive/archive.go b/internal/archive/archive.go new file mode 100644 index 0000000..604efe2 --- /dev/null +++ b/internal/archive/archive.go @@ -0,0 +1,181 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +// Implements generic access to files inside an archive. +// +// An archive in this context can be any container that holds files. +// In this implementation the archive can be a ZIP file or a directory. +package archive + +import ( + "archive/zip" + "fmt" + "io" + "os" + "path/filepath" +) + +// Generic archive interface. +type Archive interface { + Close() error + OpenFile(path string) (io.ReadCloser, error) + Glob(pattern string) ([]FileInfo, error) +} + +// Open an archive in path. +// The archive can be a ZIP file or a directory. The implementation +// will detect the type of archive and return the appropriate +// implementation of the Archive interface. +func OpenArchive(path string) (Archive, error) { + fi, err := os.Stat(path) + if err != nil { + return nil, err + } + switch mode := fi.Mode(); { + case mode.IsRegular(): + archive := &zipArchive{} + err := archive.Open(path) + if err != nil { + return nil, err + } + return archive, nil + case mode.IsDir(): + archive := &dirArchive{} + err := archive.Open(path) + if err != nil { + return nil, err + } + return archive, nil + default: + return nil, fmt.Errorf("unsupported file mode: %s", mode) + } +} + +// Interface for a file that can be opened when needed. +type OpenableFile interface { + Open() (io.ReadCloser, error) +} + +// Generic information about a file inside an archive. +type FileInfo struct { + Name string + File OpenableFile +} + +// A openable file in the filesystem. +type filesystemFile struct { + path string +} + +func (f *filesystemFile) Open() (io.ReadCloser, error) { + return os.Open(f.path) +} + +// An implementation of the archiveBackend interface for zip files. +type zipArchive struct { + zip *zip.ReadCloser +} + +func (a *zipArchive) Open(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { + result := make([]FileInfo, 0) + for _, file := range a.zip.File { + if matched, err := filepath.Match(pattern, file.Name); matched { + if err != nil { + return nil, err + } + info := FileInfo{ + Name: file.Name, + File: file, + } + result = append(result, info) + } + } + + return result, nil +} + +func (a *zipArchive) OpenFile(path string) (io.ReadCloser, error) { + file, err := a.zip.Open(path) + if err != nil { + return nil, err + } + return file, nil +} + +// An implementation of the archiveBackend interface for directories. +type dirArchive struct { + dir string +} + +func (a *dirArchive) Open(path string) error { + a.dir = filepath.Clean(path) + return nil +} + +func (a *dirArchive) Close() error { + return nil +} + +func (a *dirArchive) OpenFile(path string) (io.ReadCloser, error) { + file, err := os.Open(filepath.Join(a.dir, path)) + if err != nil { + return nil, err + } + return file, nil +} + +func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { + files, err := filepath.Glob(filepath.Join(a.dir, pattern)) + if err != nil { + return nil, err + } + result := make([]FileInfo, 0) + for _, filename := range files { + name, err := filepath.Rel(a.dir, filename) + if err != nil { + return nil, err + } + info := FileInfo{ + Name: name, + File: &filesystemFile{path: filename}, + } + result = append(result, info) + } + + return result, nil +} diff --git a/pkg/listenbrainz/archive.go b/pkg/listenbrainz/archive.go index de34ba8..a455d03 100644 --- a/pkg/listenbrainz/archive.go +++ b/pkg/listenbrainz/archive.go @@ -22,19 +22,16 @@ THE SOFTWARE. package listenbrainz import ( - "archive/zip" "encoding/json" - "fmt" "io" "iter" - "os" - "path/filepath" "regexp" "sort" "strconv" "time" "github.com/simonfrey/jsonl" + "go.uploadedlobster.com/scotty/internal/archive" ) // Represents a ListenBrainz export archive. @@ -42,7 +39,17 @@ import ( // The export contains the user's listen history, favorite tracks and // user information. type Archive struct { - backend archiveBackend + backend archive.Archive +} + +// Open a ListenBrainz archive from file path. +func OpenArchive(path string) (*Archive, error) { + backend, err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + + return &Archive{backend: backend}, nil } // Close the archive and release any resources. @@ -137,166 +144,27 @@ func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { } } -// Open a ListenBrainz archive from file path. -func OpenArchive(path string) (*Archive, error) { - fi, err := os.Stat(path) - if err != nil { - return nil, err - } - switch mode := fi.Mode(); { - case mode.IsRegular(): - backend := &zipArchive{} - err := backend.Open(path) - if err != nil { - return nil, err - } - return &Archive{backend: backend}, nil - case mode.IsDir(): - backend := &dirArchive{} - err := backend.Open(path) - if err != nil { - return nil, err - } - return &Archive{backend: backend}, nil - default: - return nil, fmt.Errorf("unsupported file mode: %s", mode) - } -} - type UserInfo struct { ID string `json:"user_id"` Name string `json:"username"` } -type archiveBackend interface { - Close() error - OpenFile(path string) (io.ReadCloser, error) - Glob(pattern string) ([]FileInfo, error) -} - type timeRange struct { Start time.Time End time.Time } -type OpenableFile interface { - Open() (io.ReadCloser, error) -} - -type FileInfo struct { - Name string - File OpenableFile -} - -type FilesystemFile struct { - path string -} - -func (f *FilesystemFile) Open() (io.ReadCloser, error) { - return os.Open(f.path) -} - type ListenExportFileInfo struct { Name string TimeRange timeRange - f OpenableFile -} - -// An implementation of the archiveBackend interface for zip files. -type zipArchive struct { - zip *zip.ReadCloser -} - -func (a *zipArchive) Open(path string) error { - zip, err := zip.OpenReader(path) - if err != nil { - return err - } - a.zip = zip - return nil -} - -func (a *zipArchive) Close() error { - if a.zip == nil { - return nil - } - return a.zip.Close() -} - -func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { - result := make([]FileInfo, 0) - for _, file := range a.zip.File { - if matched, err := filepath.Match(pattern, file.Name); matched { - if err != nil { - return nil, err - } - info := FileInfo{ - Name: file.Name, - File: file, - } - result = append(result, info) - } - } - - return result, nil -} - -func (a *zipArchive) OpenFile(path string) (io.ReadCloser, error) { - file, err := a.zip.Open(path) - if err != nil { - return nil, err - } - return file, nil -} - -// An implementation of the archiveBackend interface for directories. -type dirArchive struct { - dir string -} - -func (a *dirArchive) Open(path string) error { - a.dir = filepath.Clean(path) - return nil -} - -func (a *dirArchive) Close() error { - return nil -} - -func (a *dirArchive) OpenFile(path string) (io.ReadCloser, error) { - file, err := os.Open(filepath.Join(a.dir, path)) - if err != nil { - return nil, err - } - return file, nil -} - -func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { - files, err := filepath.Glob(filepath.Join(a.dir, pattern)) - if err != nil { - return nil, err - } - result := make([]FileInfo, 0) - for _, filename := range files { - name, err := filepath.Rel(a.dir, filename) - if err != nil { - return nil, err - } - info := FileInfo{ - Name: name, - File: &FilesystemFile{path: filename}, - } - result = append(result, info) - } - - return result, nil + f archive.OpenableFile } type ListenExportFile struct { - file OpenableFile + file archive.OpenableFile } -func NewExportFile(f OpenableFile) ListenExportFile { +func NewExportFile(f archive.OpenableFile) ListenExportFile { return ListenExportFile{file: f} } From 8462b9395e35ee51f7be317cb79a548ba22b0827 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 00:45:11 +0200 Subject: [PATCH 51/77] Keep listenbrainz package internal for now --- internal/backends/lbarchive/lbarchive.go | 2 +- internal/backends/listenbrainz/listenbrainz.go | 2 +- internal/backends/listenbrainz/listenbrainz_test.go | 2 +- {pkg => internal}/listenbrainz/archive.go | 0 {pkg => internal}/listenbrainz/client.go | 0 {pkg => internal}/listenbrainz/client_test.go | 2 +- {pkg => internal}/listenbrainz/models.go | 0 {pkg => internal}/listenbrainz/models_test.go | 2 +- {pkg => internal}/listenbrainz/testdata/feedback.json | 0 {pkg => internal}/listenbrainz/testdata/listen.json | 0 {pkg => internal}/listenbrainz/testdata/listens.json | 0 {pkg => internal}/listenbrainz/testdata/lookup.json | 0 12 files changed, 5 insertions(+), 5 deletions(-) rename {pkg => internal}/listenbrainz/archive.go (100%) rename {pkg => internal}/listenbrainz/client.go (100%) rename {pkg => internal}/listenbrainz/client_test.go (99%) rename {pkg => internal}/listenbrainz/models.go (100%) rename {pkg => internal}/listenbrainz/models_test.go (98%) rename {pkg => internal}/listenbrainz/testdata/feedback.json (100%) rename {pkg => internal}/listenbrainz/testdata/listen.json (100%) rename {pkg => internal}/listenbrainz/testdata/listens.json (100%) rename {pkg => internal}/listenbrainz/testdata/lookup.json (100%) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 143a674..88d8be7 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -28,8 +28,8 @@ import ( lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) const batchSize = 2000 diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 5e80a10..4f0ce2f 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -26,10 +26,10 @@ import ( "go.uploadedlobster.com/musicbrainzws2" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/internal/similarity" "go.uploadedlobster.com/scotty/internal/version" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" ) type ListenBrainzApiBackend struct { diff --git a/internal/backends/listenbrainz/listenbrainz_test.go b/internal/backends/listenbrainz/listenbrainz_test.go index dd3e1d3..f7151e5 100644 --- a/internal/backends/listenbrainz/listenbrainz_test.go +++ b/internal/backends/listenbrainz/listenbrainz_test.go @@ -26,7 +26,7 @@ import ( "go.uploadedlobster.com/mbtypes" lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestInitConfig(t *testing.T) { diff --git a/pkg/listenbrainz/archive.go b/internal/listenbrainz/archive.go similarity index 100% rename from pkg/listenbrainz/archive.go rename to internal/listenbrainz/archive.go diff --git a/pkg/listenbrainz/client.go b/internal/listenbrainz/client.go similarity index 100% rename from pkg/listenbrainz/client.go rename to internal/listenbrainz/client.go diff --git a/pkg/listenbrainz/client_test.go b/internal/listenbrainz/client_test.go similarity index 99% rename from pkg/listenbrainz/client_test.go rename to internal/listenbrainz/client_test.go index 3742ca9..9baf293 100644 --- a/pkg/listenbrainz/client_test.go +++ b/internal/listenbrainz/client_test.go @@ -31,7 +31,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestNewClient(t *testing.T) { diff --git a/pkg/listenbrainz/models.go b/internal/listenbrainz/models.go similarity index 100% rename from pkg/listenbrainz/models.go rename to internal/listenbrainz/models.go diff --git a/pkg/listenbrainz/models_test.go b/internal/listenbrainz/models_test.go similarity index 98% rename from pkg/listenbrainz/models_test.go rename to internal/listenbrainz/models_test.go index 8fb4994..404b87b 100644 --- a/pkg/listenbrainz/models_test.go +++ b/internal/listenbrainz/models_test.go @@ -29,7 +29,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/scotty/pkg/listenbrainz" + "go.uploadedlobster.com/scotty/internal/listenbrainz" ) func TestTrackDurationMillisecondsInt(t *testing.T) { diff --git a/pkg/listenbrainz/testdata/feedback.json b/internal/listenbrainz/testdata/feedback.json similarity index 100% rename from pkg/listenbrainz/testdata/feedback.json rename to internal/listenbrainz/testdata/feedback.json diff --git a/pkg/listenbrainz/testdata/listen.json b/internal/listenbrainz/testdata/listen.json similarity index 100% rename from pkg/listenbrainz/testdata/listen.json rename to internal/listenbrainz/testdata/listen.json diff --git a/pkg/listenbrainz/testdata/listens.json b/internal/listenbrainz/testdata/listens.json similarity index 100% rename from pkg/listenbrainz/testdata/listens.json rename to internal/listenbrainz/testdata/listens.json diff --git a/pkg/listenbrainz/testdata/lookup.json b/internal/listenbrainz/testdata/lookup.json similarity index 100% rename from pkg/listenbrainz/testdata/lookup.json rename to internal/listenbrainz/testdata/lookup.json From cf5319309a4c81e29df2b9eb8edfe26f363ab8c1 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 00:51:28 +0200 Subject: [PATCH 52/77] Renamed listenbrainz.Archive to listenbrainz.ExportArchive --- internal/backends/lbarchive/lbarchive.go | 2 +- internal/listenbrainz/archive.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 88d8be7..0848d38 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -69,7 +69,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( }, } - archive, err := listenbrainz.OpenArchive(b.filePath) + archive, err := listenbrainz.OpenExportArchive(b.filePath) if err != nil { p.Export.Abort() progress <- p diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index a455d03..1d3efa3 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -38,27 +38,27 @@ import ( // // The export contains the user's listen history, favorite tracks and // user information. -type Archive struct { +type ExportArchive struct { backend archive.Archive } // Open a ListenBrainz archive from file path. -func OpenArchive(path string) (*Archive, error) { +func OpenExportArchive(path string) (*ExportArchive, error) { backend, err := archive.OpenArchive(path) if err != nil { return nil, err } - return &Archive{backend: backend}, nil + return &ExportArchive{backend: backend}, nil } // Close the archive and release any resources. -func (a *Archive) Close() error { +func (a *ExportArchive) Close() error { return a.backend.Close() } // Read the user information from the archive. -func (a *Archive) UserInfo() (UserInfo, error) { +func (a *ExportArchive) UserInfo() (UserInfo, error) { f, err := a.backend.OpenFile("user.json") if err != nil { return UserInfo{}, err @@ -75,7 +75,7 @@ func (a *Archive) UserInfo() (UserInfo, error) { return userInfo, nil } -func (a *Archive) ListListenExports() ([]ListenExportFileInfo, error) { +func (a *ExportArchive) ListListenExports() ([]ListenExportFileInfo, error) { re := regexp.MustCompile(`^listens/(\d{4})/(\d{1,2})\.jsonl$`) result := make([]ListenExportFileInfo, 0) @@ -109,7 +109,7 @@ func (a *Archive) ListListenExports() ([]ListenExportFileInfo, error) { // Yields all listens from the archive that are newer than the given timestamp. // The listens are yielded in ascending order of their listened_at timestamp. -func (a *Archive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { +func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, error] { return func(yield func(Listen, error) bool) { files, err := a.ListListenExports() if err != nil { From 0231331209e6a79416ce87dce99a0d56f3503c48 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 01:23:12 +0200 Subject: [PATCH 53/77] Implemented listenrbainz.ExportArchive.IterFeedback --- internal/listenbrainz/archive.go | 55 +++++++++++++++++++++++++------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index 1d3efa3..eb7677c 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -23,6 +23,7 @@ package listenbrainz import ( "encoding/json" + "errors" "io" "iter" "regexp" @@ -54,6 +55,9 @@ func OpenExportArchive(path string) (*ExportArchive, error) { // Close the archive and release any resources. func (a *ExportArchive) Close() error { + if a.backend == nil { + return nil + } return a.backend.Close() } @@ -126,8 +130,8 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er continue } - f := NewExportFile(file.f) - for l, err := range f.IterListens() { + f := JSONLFile[Listen]{file: file.f} + for l, err := range f.IterItems() { if err != nil { yield(Listen{}, err) return @@ -144,6 +148,36 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er } } +// Yields all feedbacks from the archive that are newer than the given timestamp. +// The feedbacks are yielded in ascending order of their Created timestamp. +func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, error] { + return func(yield func(Feedback, error) bool) { + files, err := a.backend.Glob("feedback.jsonl") + if err != nil { + yield(Feedback{}, err) + return + } else if len(files) == 0 { + yield(Feedback{}, errors.New("no feedback.jsonl file found in archive")) + return + } + + j := JSONLFile[Feedback]{file: files[0].File} + for l, err := range j.IterItems() { + if err != nil { + yield(Feedback{}, err) + return + } + + if !time.Unix(l.Created, 0).After(minTimestamp) { + continue + } + if !yield(l, nil) { + break + } + } + } +} + type UserInfo struct { ID string `json:"user_id"` Name string `json:"username"` @@ -160,15 +194,11 @@ type ListenExportFileInfo struct { f archive.OpenableFile } -type ListenExportFile struct { +type JSONLFile[T any] struct { file archive.OpenableFile } -func NewExportFile(f archive.OpenableFile) ListenExportFile { - return ListenExportFile{file: f} -} - -func (f *ListenExportFile) openReader() (*jsonl.Reader, error) { +func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { fio, err := f.file.Open() if err != nil { return nil, err @@ -177,17 +207,18 @@ func (f *ListenExportFile) openReader() (*jsonl.Reader, error) { return &reader, nil } -func (f *ListenExportFile) IterListens() iter.Seq2[Listen, error] { - return func(yield func(Listen, error) bool) { +func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] { + return func(yield func(T, error) bool) { reader, err := f.openReader() if err != nil { - yield(Listen{}, err) + var listen T + yield(listen, err) return } defer reader.Close() for { - listen := Listen{} + var listen T err := reader.ReadSingleLine(&listen) if err != nil { break From 975e2082548060acc484f484886df2fee91e28e6 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 02:20:07 +0200 Subject: [PATCH 54/77] Simplify dirArchive by using os.dirFS and have Archive.Open return fs.File --- internal/archive/archive.go | 40 +++++++++++++++----------------- internal/listenbrainz/archive.go | 2 +- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 604efe2..7714552 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -30,6 +30,7 @@ import ( "archive/zip" "fmt" "io" + "io/fs" "os" "path/filepath" ) @@ -37,7 +38,7 @@ import ( // Generic archive interface. type Archive interface { Close() error - OpenFile(path string) (io.ReadCloser, error) + Open(path string) (fs.File, error) Glob(pattern string) ([]FileInfo, error) } @@ -53,14 +54,14 @@ func OpenArchive(path string) (Archive, error) { switch mode := fi.Mode(); { case mode.IsRegular(): archive := &zipArchive{} - err := archive.Open(path) + err := archive.OpenArchive(path) if err != nil { return nil, err } return archive, nil case mode.IsDir(): archive := &dirArchive{} - err := archive.Open(path) + err := archive.OpenArchive(path) if err != nil { return nil, err } @@ -95,7 +96,7 @@ type zipArchive struct { zip *zip.ReadCloser } -func (a *zipArchive) Open(path string) error { +func (a *zipArchive) OpenArchive(path string) error { zip, err := zip.OpenReader(path) if err != nil { return err @@ -129,7 +130,7 @@ func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { return result, nil } -func (a *zipArchive) OpenFile(path string) (io.ReadCloser, error) { +func (a *zipArchive) Open(path string) (fs.File, error) { file, err := a.zip.Open(path) if err != nil { return nil, err @@ -139,11 +140,13 @@ func (a *zipArchive) OpenFile(path string) (io.ReadCloser, error) { // An implementation of the archiveBackend interface for directories. type dirArchive struct { - dir string + path string + dirFS fs.FS } -func (a *dirArchive) Open(path string) error { - a.dir = filepath.Clean(path) +func (a *dirArchive) OpenArchive(path string) error { + a.path = filepath.Clean(path) + a.dirFS = os.DirFS(path) return nil } @@ -151,28 +154,23 @@ func (a *dirArchive) Close() error { return nil } -func (a *dirArchive) OpenFile(path string) (io.ReadCloser, error) { - file, err := os.Open(filepath.Join(a.dir, path)) - if err != nil { - return nil, err - } - return file, nil +// Open opens the named file in the archive. +// [fs.File.Close] must be called to release any associated resources. +func (a *dirArchive) Open(path string) (fs.File, error) { + return a.dirFS.Open(path) } func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { - files, err := filepath.Glob(filepath.Join(a.dir, pattern)) + files, err := fs.Glob(a.dirFS, pattern) if err != nil { return nil, err } result := make([]FileInfo, 0) - for _, filename := range files { - name, err := filepath.Rel(a.dir, filename) - if err != nil { - return nil, err - } + for _, name := range files { + fullPath := filepath.Join(a.path, name) info := FileInfo{ Name: name, - File: &filesystemFile{path: filename}, + File: &filesystemFile{path: fullPath}, } result = append(result, info) } diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index eb7677c..b7b5909 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -63,7 +63,7 @@ func (a *ExportArchive) Close() error { // Read the user information from the archive. func (a *ExportArchive) UserInfo() (UserInfo, error) { - f, err := a.backend.OpenFile("user.json") + f, err := a.backend.Open("user.json") if err != nil { return UserInfo{}, err } From d25095267876e1a6fdb19d950a061047da8a2c1d Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 11:33:10 +0200 Subject: [PATCH 55/77] Extend dump backend to be able to write to a file --- config.example.toml | 8 +++- internal/backends/dump/dump.go | 75 ++++++++++++++++++++++++++++++---- 2 files changed, 74 insertions(+), 9 deletions(-) diff --git a/config.example.toml b/config.example.toml index 40ffd18..ecbba9b 100644 --- a/config.example.toml +++ b/config.example.toml @@ -141,4 +141,10 @@ client-secret = "" [service.dump] # This backend allows writing listens and loves as console output. Useful for # debugging the export from other services. -backend = "dump" +backend = "dump" +# Path to a file where the listens and loves are written to. If not set, +# the output is written to stdout. +file-path = "" +# If true (default), new listens will be appended to the existing file. Set to +# false to overwrite the file on every run. +append = true diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index 1fcd864..4714bd6 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -17,25 +17,80 @@ Scotty. If not, see . package dump import ( + "bytes" "context" "fmt" + "io" + "os" + "strings" "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/i18n" "go.uploadedlobster.com/scotty/internal/models" ) -type DumpBackend struct{} +type DumpBackend struct { + buffer io.ReadWriter + print bool // Whether to print the output to stdout +} func (b *DumpBackend) Name() string { return "dump" } -func (b *DumpBackend) Options() []models.BackendOption { return nil } +func (b *DumpBackend) Options() []models.BackendOption { + return []models.BackendOption{{ + Name: "file-path", + Label: i18n.Tr("File path"), + Type: models.String, + }, { + Name: "append", + Label: i18n.Tr("Append to file"), + Type: models.Bool, + Default: "true", + }} +} func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error { + filePath := config.GetString("file-path") + append := config.GetBool("append", true) + if strings.TrimSpace(filePath) != "" { + mode := os.O_WRONLY | os.O_CREATE + if !append { + mode |= os.O_TRUNC // Truncate the file if not appending + } + f, err := os.OpenFile(filePath, mode, 0644) + if err != nil { + return err + } + b.buffer = f + b.print = false // If a file path is specified, we don't print to stdout + } else { + // If no file path is specified, use a bytes.Buffer for in-memory dumping + b.buffer = new(bytes.Buffer) + b.print = true // Print to stdout + } return nil } -func (b *DumpBackend) StartImport() error { return nil } -func (b *DumpBackend) FinishImport() error { return nil } +func (b *DumpBackend) StartImport() error { return nil } + +func (b *DumpBackend) FinishImport() error { + if b.print { + out := new(strings.Builder) + _, err := io.Copy(out, b.buffer) + if err != nil { + return err + } + fmt.Println(out.String()) + } + + // Close the io writer if it is closable + if closer, ok := b.buffer.(io.Closer); ok { + if err := closer.Close(); err != nil { + return fmt.Errorf("failed to close output file: %w", err) + } + } + return nil +} func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { @@ -45,9 +100,11 @@ func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensRe importResult.UpdateTimestamp(listen.ListenedAt) importResult.ImportCount += 1 - msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", + _, err := fmt.Fprintf(b.buffer, "🎶 %v: \"%v\" by %v (%v)\n", listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) - importResult.Log(models.Info, msg) + if err != nil { + return importResult, err + } progress <- models.TransferProgress{}.FromImportResult(importResult, false) } @@ -62,9 +119,11 @@ func (b *DumpBackend) ImportLoves(ctx context.Context, export models.LovesResult importResult.UpdateTimestamp(love.Created) importResult.ImportCount += 1 - msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", + _, err := fmt.Fprintf(b.buffer, "❤️ %v: \"%v\" by %v (%v)\n", love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) - importResult.Log(models.Info, msg) + if err != nil { + return importResult, err + } progress <- models.TransferProgress{}.FromImportResult(importResult, false) } From dddd2e4eec7d2f54d16ea59ace358bd2c5d85ebc Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 11:59:35 +0200 Subject: [PATCH 56/77] Implemented lbarchive loves export --- go.mod | 2 +- internal/backends/backends_test.go | 2 +- internal/backends/lbarchive/lbarchive.go | 95 ++++++++++++++- internal/backends/listenbrainz/helper.go | 115 ++++++++++++++++++ .../backends/listenbrainz/listenbrainz.go | 81 +----------- 5 files changed, 210 insertions(+), 85 deletions(-) create mode 100644 internal/backends/listenbrainz/helper.go diff --git a/go.mod b/go.mod index ccdb6cc..c4c2a65 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/pelletier/go-toml/v2 v2.2.4 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 + github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 github.com/spf13/cast v1.8.0 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.20.1 @@ -53,7 +54,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect - github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index 737c7e3..026e487 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -106,7 +106,7 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{}) // expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{}) - // expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) // expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{}) expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{}) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 0848d38..cff2a1f 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -25,17 +25,23 @@ import ( "context" "time" + "go.uploadedlobster.com/musicbrainzws2" lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" + "go.uploadedlobster.com/scotty/internal/version" ) -const batchSize = 2000 +const ( + listensBatchSize = 2000 + lovesBatchSize = 10 +) type ListenBrainzArchiveBackend struct { filePath string + mbClient musicbrainzws2.Client } func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" } @@ -50,6 +56,11 @@ func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { b.filePath = config.GetString("file-path") + b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ + Name: version.AppName, + Version: version.AppVersion, + URL: version.AppURL, + }) return nil } @@ -86,7 +97,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( return } - listens := make(models.ListensList, 0, batchSize) + listens := make(models.ListensList, 0, listensBatchSize) for rawListen, err := range archive.IterListens(oldestTimestamp) { if err != nil { p.Export.Abort() @@ -108,7 +119,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( // Allow the importer to start processing the listens by // sending them in batches. - if len(listens) >= batchSize { + if len(listens) >= listensBatchSize { results <- models.ListensResult{Items: listens} progress <- p listens = listens[:0] @@ -119,3 +130,81 @@ func (b *ListenBrainzArchiveBackend) ExportListens( p.Export.Complete() progress <- p } + +func (b *ListenBrainzArchiveBackend) ExportLoves( + ctx context.Context, oldestTimestamp time.Time, + results chan models.LovesResult, progress chan models.TransferProgress) { + startTime := time.Now() + minTime := oldestTimestamp + if minTime.Unix() < 1 { + minTime = time.Unix(1, 0) + } + + totalDuration := startTime.Sub(oldestTimestamp) + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } + + archive, err := listenbrainz.OpenExportArchive(b.filePath) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + defer archive.Close() + + userInfo, err := archive.UserInfo() + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + + loves := make(models.LovesList, 0, lovesBatchSize) + for feedback, err := range archive.IterFeedback(oldestTimestamp) { + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + + // The export file does not include track metadata. Try fetching details + // from MusicBrainz. + if feedback.TrackMetadata == nil { + track, err := lbapi.LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) + if err == nil { + feedback.TrackMetadata = track + } + } + + love := lbapi.AsLove(feedback) + if love.UserName == "" { + love.UserName = userInfo.Name + } + // TODO: The dump does not contain TrackMetadata for feedback. + // We need to look it up in the archive. + loves = append(loves, love) + + // Update the progress + p.Export.TotalItems += 1 + remainingTime := startTime.Sub(love.Created) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + + // Allow the importer to start processing the listens by + // sending them in batches. + if len(loves) >= lovesBatchSize { + results <- models.LovesResult{Items: loves} + progress <- p + loves = loves[:0] + } + } + + results <- models.LovesResult{Items: loves} + p.Export.Complete() + progress <- p +} diff --git a/internal/backends/listenbrainz/helper.go b/internal/backends/listenbrainz/helper.go new file mode 100644 index 0000000..f39a2df --- /dev/null +++ b/internal/backends/listenbrainz/helper.go @@ -0,0 +1,115 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package listenbrainz + +import ( + "context" + "time" + + "go.uploadedlobster.com/mbtypes" + "go.uploadedlobster.com/musicbrainzws2" + "go.uploadedlobster.com/scotty/internal/listenbrainz" + "go.uploadedlobster.com/scotty/internal/models" +) + +func LookupRecording( + ctx context.Context, + mb *musicbrainzws2.Client, + mbid mbtypes.MBID, +) (*listenbrainz.Track, error) { + filter := musicbrainzws2.IncludesFilter{ + Includes: []string{"artist-credits"}, + } + recording, err := mb.LookupRecording(ctx, mbid, filter) + if err != nil { + return nil, err + } + + artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) + for _, artist := range recording.ArtistCredit { + artistMBIDs = append(artistMBIDs, artist.Artist.ID) + } + track := listenbrainz.Track{ + TrackName: recording.Title, + ArtistName: recording.ArtistCredit.String(), + MBIDMapping: &listenbrainz.MBIDMapping{ + // In case of redirects this MBID differs from the looked up MBID + RecordingMBID: recording.ID, + ArtistMBIDs: artistMBIDs, + }, + } + return &track, nil +} + +func AsListen(lbListen listenbrainz.Listen) models.Listen { + listen := models.Listen{ + ListenedAt: time.Unix(lbListen.ListenedAt, 0), + UserName: lbListen.UserName, + Track: AsTrack(lbListen.TrackMetadata), + } + return listen +} + +func AsLove(f listenbrainz.Feedback) models.Love { + recordingMBID := f.RecordingMBID + track := f.TrackMetadata + if track == nil { + track = &listenbrainz.Track{} + } + love := models.Love{ + UserName: f.UserName, + RecordingMBID: recordingMBID, + Created: time.Unix(f.Created, 0), + Track: AsTrack(*track), + } + + if love.Track.RecordingMBID == "" { + love.Track.RecordingMBID = love.RecordingMBID + } + + return love +} + +func AsTrack(t listenbrainz.Track) models.Track { + track := models.Track{ + TrackName: t.TrackName, + ReleaseName: t.ReleaseName, + ArtistNames: []string{t.ArtistName}, + Duration: t.Duration(), + TrackNumber: t.TrackNumber(), + DiscNumber: t.DiscNumber(), + RecordingMBID: t.RecordingMBID(), + ReleaseMBID: t.ReleaseMBID(), + ReleaseGroupMBID: t.ReleaseGroupMBID(), + ISRC: t.ISRC(), + AdditionalInfo: t.AdditionalInfo, + } + + if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 { + for _, artistMBID := range t.MBIDMapping.ArtistMBIDs { + track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID) + } + } + + return track +} diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 4f0ce2f..8035b22 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -249,7 +249,7 @@ out: // longer available and might have been merged. Try fetching details // from MusicBrainz. if feedback.TrackMetadata == nil { - track, err := b.lookupRecording(ctx, feedback.RecordingMBID) + track, err := LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) if err == nil { feedback.TrackMetadata = track } @@ -375,82 +375,3 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste return false, nil } - -func (b *ListenBrainzApiBackend) lookupRecording( - ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) { - filter := musicbrainzws2.IncludesFilter{ - Includes: []string{"artist-credits"}, - } - recording, err := b.mbClient.LookupRecording(ctx, mbid, filter) - if err != nil { - return nil, err - } - - artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) - for _, artist := range recording.ArtistCredit { - artistMBIDs = append(artistMBIDs, artist.Artist.ID) - } - track := listenbrainz.Track{ - TrackName: recording.Title, - ArtistName: recording.ArtistCredit.String(), - MBIDMapping: &listenbrainz.MBIDMapping{ - // In case of redirects this MBID differs from the looked up MBID - RecordingMBID: recording.ID, - ArtistMBIDs: artistMBIDs, - }, - } - return &track, nil -} - -func AsListen(lbListen listenbrainz.Listen) models.Listen { - listen := models.Listen{ - ListenedAt: time.Unix(lbListen.ListenedAt, 0), - UserName: lbListen.UserName, - Track: AsTrack(lbListen.TrackMetadata), - } - return listen -} - -func AsLove(f listenbrainz.Feedback) models.Love { - recordingMBID := f.RecordingMBID - track := f.TrackMetadata - if track == nil { - track = &listenbrainz.Track{} - } - love := models.Love{ - UserName: f.UserName, - RecordingMBID: recordingMBID, - Created: time.Unix(f.Created, 0), - Track: AsTrack(*track), - } - - if love.Track.RecordingMBID == "" { - love.Track.RecordingMBID = love.RecordingMBID - } - - return love -} - -func AsTrack(t listenbrainz.Track) models.Track { - track := models.Track{ - TrackName: t.TrackName, - ReleaseName: t.ReleaseName, - ArtistNames: []string{t.ArtistName}, - Duration: t.Duration(), - TrackNumber: t.TrackNumber(), - DiscNumber: t.DiscNumber(), - RecordingMBID: t.RecordingMBID(), - ReleaseMBID: t.ReleaseMBID(), - ReleaseGroupMBID: t.ReleaseGroupMBID(), - ISRC: t.ISRC(), - AdditionalInfo: t.AdditionalInfo, - } - - if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 { - for _, artistMBID := range t.MBIDMapping.ArtistMBIDs { - track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID) - } - } - - return track -} From 7542657925b1f6a253898d541d64c6408639af9d Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 16:46:10 +0200 Subject: [PATCH 57/77] Use LB API to lookup missing metadata for loves This is faster than using the MBID API individually --- internal/backends/lbarchive/lbarchive.go | 46 ++++---- internal/backends/listenbrainz/helper.go | 133 ++++++++++++++++++----- internal/listenbrainz/client.go | 22 ++++ internal/listenbrainz/models.go | 44 +++++++- 4 files changed, 194 insertions(+), 51 deletions(-) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index cff2a1f..6e2f349 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -36,11 +36,12 @@ import ( const ( listensBatchSize = 2000 - lovesBatchSize = 10 + lovesBatchSize = listenbrainz.MaxItemsPerGet ) type ListenBrainzArchiveBackend struct { filePath string + lbClient listenbrainz.Client mbClient musicbrainzws2.Client } @@ -56,6 +57,7 @@ func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { b.filePath = config.GetString("file-path") + b.lbClient = listenbrainz.NewClient("", version.UserAgent()) b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: version.AppName, Version: version.AppVersion, @@ -164,7 +166,7 @@ func (b *ListenBrainzArchiveBackend) ExportLoves( return } - loves := make(models.LovesList, 0, lovesBatchSize) + batch := make([]listenbrainz.Feedback, 0, lovesBatchSize) for feedback, err := range archive.IterFeedback(oldestTimestamp) { if err != nil { p.Export.Abort() @@ -173,37 +175,43 @@ func (b *ListenBrainzArchiveBackend) ExportLoves( return } - // The export file does not include track metadata. Try fetching details - // from MusicBrainz. - if feedback.TrackMetadata == nil { - track, err := lbapi.LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) - if err == nil { - feedback.TrackMetadata = track - } + if feedback.UserName == "" { + feedback.UserName = userInfo.Name } - love := lbapi.AsLove(feedback) - if love.UserName == "" { - love.UserName = userInfo.Name - } - // TODO: The dump does not contain TrackMetadata for feedback. - // We need to look it up in the archive. - loves = append(loves, love) + batch = append(batch, feedback) // Update the progress p.Export.TotalItems += 1 - remainingTime := startTime.Sub(love.Created) + remainingTime := startTime.Sub(time.Unix(feedback.Created, 0)) p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) // Allow the importer to start processing the listens by // sending them in batches. - if len(loves) >= lovesBatchSize { + if len(batch) >= lovesBatchSize { + // The dump does not contain track metadata. Extend it with additional + // lookups + loves, err := lbapi.ExtendTrackMetadata(ctx, &b.lbClient, &b.mbClient, &batch) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + results <- models.LovesResult{Items: loves} progress <- p - loves = loves[:0] + batch = batch[:0] } } + loves, err := lbapi.ExtendTrackMetadata(ctx, &b.lbClient, &b.mbClient, &batch) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } results <- models.LovesResult{Items: loves} p.Export.Complete() progress <- p diff --git a/internal/backends/listenbrainz/helper.go b/internal/backends/listenbrainz/helper.go index f39a2df..d6572d0 100644 --- a/internal/backends/listenbrainz/helper.go +++ b/internal/backends/listenbrainz/helper.go @@ -32,35 +32,6 @@ import ( "go.uploadedlobster.com/scotty/internal/models" ) -func LookupRecording( - ctx context.Context, - mb *musicbrainzws2.Client, - mbid mbtypes.MBID, -) (*listenbrainz.Track, error) { - filter := musicbrainzws2.IncludesFilter{ - Includes: []string{"artist-credits"}, - } - recording, err := mb.LookupRecording(ctx, mbid, filter) - if err != nil { - return nil, err - } - - artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) - for _, artist := range recording.ArtistCredit { - artistMBIDs = append(artistMBIDs, artist.Artist.ID) - } - track := listenbrainz.Track{ - TrackName: recording.Title, - ArtistName: recording.ArtistCredit.String(), - MBIDMapping: &listenbrainz.MBIDMapping{ - // In case of redirects this MBID differs from the looked up MBID - RecordingMBID: recording.ID, - ArtistMBIDs: artistMBIDs, - }, - } - return &track, nil -} - func AsListen(lbListen listenbrainz.Listen) models.Listen { listen := models.Listen{ ListenedAt: time.Unix(lbListen.ListenedAt, 0), @@ -113,3 +84,107 @@ func AsTrack(t listenbrainz.Track) models.Track { return track } + +func LookupRecording( + ctx context.Context, + mb *musicbrainzws2.Client, + mbid mbtypes.MBID, +) (*listenbrainz.Track, error) { + filter := musicbrainzws2.IncludesFilter{ + Includes: []string{"artist-credits"}, + } + recording, err := mb.LookupRecording(ctx, mbid, filter) + if err != nil { + return nil, err + } + + artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) + for _, artist := range recording.ArtistCredit { + artistMBIDs = append(artistMBIDs, artist.Artist.ID) + } + track := listenbrainz.Track{ + TrackName: recording.Title, + ArtistName: recording.ArtistCredit.String(), + MBIDMapping: &listenbrainz.MBIDMapping{ + // In case of redirects this MBID differs from the looked up MBID + RecordingMBID: recording.ID, + ArtistMBIDs: artistMBIDs, + }, + } + return &track, nil +} + +func ExtendTrackMetadata( + ctx context.Context, + lb *listenbrainz.Client, + mb *musicbrainzws2.Client, + feedbacks *[]listenbrainz.Feedback, +) ([]models.Love, error) { + mbids := make([]mbtypes.MBID, 0, len(*feedbacks)) + for _, feedback := range *feedbacks { + if feedback.TrackMetadata == nil && feedback.RecordingMBID != "" { + mbids = append(mbids, feedback.RecordingMBID) + } + } + result, err := lb.MetadataRecordings(ctx, mbids) + if err != nil { + return nil, err + } + + loves := make([]models.Love, 0, len(*feedbacks)) + for _, feedback := range *feedbacks { + if feedback.TrackMetadata == nil && feedback.RecordingMBID != "" { + metadata, ok := result[feedback.RecordingMBID] + if ok { + feedback.TrackMetadata = trackFromMetadataLookup( + feedback.RecordingMBID, metadata) + } else { + // MBID not in result. This is probably a MBID redirect, get + // data from MB instead (slower). + // If this also fails, just leave the metadata empty. + track, err := LookupRecording(ctx, mb, feedback.RecordingMBID) + if err == nil { + feedback.TrackMetadata = track + } + } + } + + loves = append(loves, AsLove(feedback)) + } + + return loves, nil +} + +func trackFromMetadataLookup( + recordingMBID mbtypes.MBID, + metadata listenbrainz.RecordingMetadata, +) *listenbrainz.Track { + artistMBIDs := make([]mbtypes.MBID, 0, len(metadata.Artist.Artists)) + artists := make([]listenbrainz.Artist, 0, len(metadata.Artist.Artists)) + for _, artist := range metadata.Artist.Artists { + artistMBIDs = append(artistMBIDs, artist.ArtistMBID) + artists = append(artists, listenbrainz.Artist{ + ArtistCreditName: artist.Name, + ArtistMBID: artist.ArtistMBID, + JoinPhrase: artist.JoinPhrase, + }) + } + + return &listenbrainz.Track{ + TrackName: metadata.Recording.Name, + ArtistName: metadata.Artist.Name, + ReleaseName: metadata.Release.Name, + AdditionalInfo: map[string]any{ + "duration_ms": metadata.Recording.Length, + "release_group_mbid": metadata.Release.ReleaseGroupMBID, + }, + MBIDMapping: &listenbrainz.MBIDMapping{ + RecordingMBID: recordingMBID, + ReleaseMBID: metadata.Release.MBID, + ArtistMBIDs: artistMBIDs, + Artists: artists, + CAAID: metadata.Release.CAAID, + CAAReleaseMBID: metadata.Release.CAAReleaseMBID, + }, + } +} diff --git a/internal/listenbrainz/client.go b/internal/listenbrainz/client.go index 957a946..270bf4b 100644 --- a/internal/listenbrainz/client.go +++ b/internal/listenbrainz/client.go @@ -28,6 +28,7 @@ import ( "time" "github.com/go-resty/resty/v2" + "go.uploadedlobster.com/mbtypes" "go.uploadedlobster.com/scotty/pkg/ratelimit" ) @@ -158,3 +159,24 @@ func (c Client) Lookup(ctx context.Context, recordingName string, artistName str } return } + +func (c Client) MetadataRecordings(ctx context.Context, mbids []mbtypes.MBID) (result RecordingMetadataResult, err error) { + const path = "/metadata/recording/" + errorResult := ErrorResult{} + body := RecordingMetadataRequest{ + RecordingMBIDs: mbids, + Includes: "artist release", + } + response, err := c.HTTPClient.R(). + SetContext(ctx). + SetBody(body). + SetResult(&result). + SetError(&errorResult). + Post(path) + + if !response.IsSuccess() { + err = errors.New(errorResult.Error) + return + } + return +} diff --git a/internal/listenbrainz/models.go b/internal/listenbrainz/models.go index 0b5f439..5e0d0e1 100644 --- a/internal/listenbrainz/models.go +++ b/internal/listenbrainz/models.go @@ -82,9 +82,9 @@ type MBIDMapping struct { } type Artist struct { - ArtistCreditName string `json:"artist_credit_name,omitempty"` - ArtistMBID string `json:"artist_mbid,omitempty"` - JoinPhrase string `json:"join_phrase,omitempty"` + ArtistCreditName string `json:"artist_credit_name,omitempty"` + ArtistMBID mbtypes.MBID `json:"artist_mbid,omitempty"` + JoinPhrase string `json:"join_phrase,omitempty"` } type GetFeedbackResult struct { @@ -112,6 +112,44 @@ type LookupResult struct { ArtistMBIDs []mbtypes.MBID `json:"artist_mbids"` } +type RecordingMetadataRequest struct { + RecordingMBIDs []mbtypes.MBID `json:"recording_mbids"` + Includes string `json:"inc,omitempty"` +} + +// Result for a recording metadata lookup +type RecordingMetadataResult map[mbtypes.MBID]RecordingMetadata + +type RecordingMetadata struct { + Artist struct { + Name string `json:"name"` + ArtistCreditID int `json:"artist_credit_id"` + Artists []struct { + Name string `json:"name"` + Area string `json:"area"` + ArtistMBID mbtypes.MBID `json:"artist_mbid"` + JoinPhrase string `json:"join_phrase"` + BeginYear int `json:"begin_year"` + Type string `json:"type"` + // todo rels + } `json:"artists"` + } `json:"artist"` + Recording struct { + Name string `json:"name"` + Length int `json:"length"` + // TODO rels + } `json:"recording"` + Release struct { + Name string `json:"name"` + AlbumArtistName string `json:"album_artist_name"` + Year int `json:"year"` + MBID mbtypes.MBID `json:"mbid"` + ReleaseGroupMBID mbtypes.MBID `json:"release_group_mbid"` + CAAID int `json:"caa_id"` + CAAReleaseMBID mbtypes.MBID `json:"caa_release_mbid"` + } `json:"release"` +} + type StatusResult struct { Status string `json:"status"` } From 4ad89d287d2f241641b6ef06bd281205afaaac50 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 16:47:13 +0200 Subject: [PATCH 58/77] Rework ratelimit code Simplify variables and avoid potential error if retry header reading fails --- pkg/ratelimit/httpheader.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/pkg/ratelimit/httpheader.go b/pkg/ratelimit/httpheader.go index dba5e30..617c3b8 100644 --- a/pkg/ratelimit/httpheader.go +++ b/pkg/ratelimit/httpheader.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Scotty is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -25,9 +25,9 @@ import ( ) const ( - RetryCount = 5 - DefaultRateLimitWaitSeconds = 5 - MaxWaitTimeSeconds = 60 + RetryCount = 5 + DefaultRateLimitWait = 5 * time.Second + MaxWaitTime = 60 * time.Second ) // Implements rate HTTP header based limiting for resty. @@ -47,16 +47,15 @@ func EnableHTTPHeaderRateLimit(client *resty.Client, resetInHeader string) { return code == http.StatusTooManyRequests || code >= http.StatusInternalServerError }, ) - client.SetRetryMaxWaitTime(time.Duration(MaxWaitTimeSeconds * time.Second)) + client.SetRetryMaxWaitTime(MaxWaitTime) client.SetRetryAfter(func(client *resty.Client, resp *resty.Response) (time.Duration, error) { - var err error - var retryAfter int = DefaultRateLimitWaitSeconds + retryAfter := DefaultRateLimitWait if resp.StatusCode() == http.StatusTooManyRequests { - retryAfter, err = strconv.Atoi(resp.Header().Get(resetInHeader)) - if err != nil { - retryAfter = DefaultRateLimitWaitSeconds + retryAfterHeader, err := strconv.Atoi(resp.Header().Get(resetInHeader)) + if err == nil { + retryAfter = time.Duration(retryAfterHeader) * time.Second } } - return time.Duration(retryAfter * int(time.Second)), err + return retryAfter, nil }) } From f70b6248b6e733ecaef35b6fe89324ab774e33c9 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 16:48:38 +0200 Subject: [PATCH 59/77] Update musicbrainzws2 to fix rate limit issues --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c4c2a65..c5c3511 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d github.com/vbauerster/mpb/v8 v8.10.1 go.uploadedlobster.com/mbtypes v0.4.0 - go.uploadedlobster.com/musicbrainzws2 v0.15.0 + go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 golang.org/x/oauth2 v0.30.0 golang.org/x/text v0.25.0 diff --git a/go.sum b/go.sum index 028515c..6d34a6d 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= go.uploadedlobster.com/musicbrainzws2 v0.15.0 h1:njJeyf1dDwfz2toEHaZSuockVsn1fg+967/tVfLHhwQ= go.uploadedlobster.com/musicbrainzws2 v0.15.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= +go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 h1:bir8kas9u0A+T54sfzj3il7SUAV5KQtb5QzDtwvslxI= +go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= From ef6780701ad506aad79cea397ed6d758ab1c033a Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 17:08:15 +0200 Subject: [PATCH 60/77] Use ExtendTrackMetadata also for LB API loves export --- .../backends/listenbrainz/listenbrainz.go | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 8035b22..9e1c9f3 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -32,6 +32,8 @@ import ( "go.uploadedlobster.com/scotty/internal/version" ) +const lovesBatchSize = listenbrainz.MaxItemsPerGet + type ListenBrainzApiBackend struct { client listenbrainz.Client mbClient musicbrainzws2.Client @@ -229,7 +231,8 @@ func (b *ListenBrainzApiBackend) ExportLoves(ctx context.Context, oldestTimestam func (b *ListenBrainzApiBackend) exportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult) { offset := 0 defer close(results) - loves := make(models.LovesList, 0, 2*listenbrainz.MaxItemsPerGet) + allLoves := make(models.LovesList, 0, 2*listenbrainz.MaxItemsPerGet) + batch := make([]listenbrainz.Feedback, 0, lovesBatchSize) out: for { @@ -245,31 +248,45 @@ out: } for _, feedback := range result.Feedback { - // Missing track metadata indicates that the recording MBID is no - // longer available and might have been merged. Try fetching details - // from MusicBrainz. - if feedback.TrackMetadata == nil { - track, err := LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) - if err == nil { - feedback.TrackMetadata = track - } - } - - love := AsLove(feedback) - if love.Created.After(oldestTimestamp) { - loves = append(loves, love) + if time.Unix(feedback.Created, 0).After(oldestTimestamp) { + batch = append(batch, feedback) } else { break out } + + if len(batch) >= lovesBatchSize { + // Missing track metadata indicates that the recording MBID is no + // longer available and might have been merged. Try fetching details + // from MusicBrainz. + lovesBatch, err := ExtendTrackMetadata(ctx, &b.client, &b.mbClient, &batch) + if err != nil { + results <- models.LovesResult{Error: err} + return + } + + for _, l := range lovesBatch { + allLoves = append(allLoves, l) + } + } } offset += listenbrainz.MaxItemsPerGet } - sort.Sort(loves) + lovesBatch, err := ExtendTrackMetadata(ctx, &b.client, &b.mbClient, &batch) + if err != nil { + results <- models.LovesResult{Error: err} + return + } + + for _, l := range lovesBatch { + allLoves = append(allLoves, l) + } + + sort.Sort(allLoves) results <- models.LovesResult{ - Total: len(loves), - Items: loves, + Total: len(allLoves), + Items: allLoves, } } From 7fb77da135d74577f76c4e1045988197e6f55f52 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 17:35:19 +0200 Subject: [PATCH 61/77] Allow reading Spotify history directly from ZIP file --- config.example.toml | 8 +- internal/backends/spotifyhistory/archive.go | 82 +++++++++++++++++++ .../backends/spotifyhistory/spotifyhistory.go | 51 +++++------- 3 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 internal/backends/spotifyhistory/archive.go diff --git a/config.example.toml b/config.example.toml index ecbba9b..28c37ad 100644 --- a/config.example.toml +++ b/config.example.toml @@ -105,9 +105,11 @@ client-secret = "" [service.spotify-history] # Read listens from a Spotify extended history export backend = "spotify-history" -# Directory where the extended history JSON files are located. The files must -# follow the naming scheme "Streaming_History_Audio_*.json". -dir-path = "./my_spotify_data_extended/Spotify Extended Streaming History" +# Path to the Spotify extended history archive. This can either point directly +# to the "my_spotify_data_extended.zip" ZIP file provided by Spotify or a +# directory where this file has been extracted to. The history files are +# expected to follow the naming pattern "Streaming_History_Audio_*.json". +archive-path = "./my_spotify_data_extended.zip" # If true (default), ignore listens from a Spotify "private session". ignore-incognito = true # If true, ignore listens marked as skipped. Default is false. diff --git a/internal/backends/spotifyhistory/archive.go b/internal/backends/spotifyhistory/archive.go new file mode 100644 index 0000000..1d596bd --- /dev/null +++ b/internal/backends/spotifyhistory/archive.go @@ -0,0 +1,82 @@ +/* +Copyright © 2025 Philipp Wolfer + +This file is part of Scotty. + +Scotty is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later version. + +Scotty is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +Scotty. If not, see . +*/ + +package spotifyhistory + +import ( + "errors" + "sort" + + "go.uploadedlobster.com/scotty/internal/archive" +) + +var historyFileGlobs = []string{ + "Spotify Extended Streaming History/Streaming_History_Audio_*.json", + "Streaming_History_Audio_*.json", +} + +// Access a Spotify history archive. +// This can be either the ZIP file as provided by Spotify +// or a directory where this was extracted to. +type HistoryArchive struct { + backend archive.Archive +} + +// Open a Spotify history archive from file path. +func OpenHistoryArchive(path string) (*HistoryArchive, error) { + backend, err := archive.OpenArchive(path) + if err != nil { + return nil, err + } + + return &HistoryArchive{backend: backend}, nil +} + +func (h *HistoryArchive) GetHistoryFiles() ([]archive.FileInfo, error) { + for _, glob := range historyFileGlobs { + files, err := h.backend.Glob(glob) + if err != nil { + return nil, err + } + + if len(files) > 0 { + sort.Slice(files, func(i, j int) bool { + return files[i].Name < files[j].Name + }) + return files, nil + } + } + + // Found no files, fail + return nil, errors.New("found no history files in archive") +} + +func readHistoryFile(f archive.OpenableFile) (StreamingHistory, error) { + file, err := f.Open() + if err != nil { + return nil, err + } + + defer file.Close() + history := StreamingHistory{} + err = history.Read(file) + if err != nil { + return nil, err + } + + return history, nil +} diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index ce470ff..90ee8ff 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -19,9 +19,6 @@ package spotifyhistory import ( "context" - "os" - "path/filepath" - "slices" "sort" "time" @@ -30,10 +27,8 @@ import ( "go.uploadedlobster.com/scotty/internal/models" ) -const historyFileGlob = "Streaming_History_Audio_*.json" - type SpotifyHistoryBackend struct { - dirPath string + archivePath string ignoreIncognito bool ignoreSkipped bool skippedMinSeconds int @@ -43,9 +38,10 @@ func (b *SpotifyHistoryBackend) Name() string { return "spotify-history" } func (b *SpotifyHistoryBackend) Options() []models.BackendOption { return []models.BackendOption{{ - Name: "dir-path", - Label: i18n.Tr("Directory path"), - Type: models.String, + Name: "archive-path", + Label: i18n.Tr("Archive path"), + Type: models.String, + Default: "./my_spotify_data_extended.zip", }, { Name: "ignore-incognito", Label: i18n.Tr("Ignore listens in incognito mode"), @@ -65,7 +61,11 @@ func (b *SpotifyHistoryBackend) Options() []models.BackendOption { } func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { - b.dirPath = config.GetString("dir-path") + b.archivePath = config.GetString("archive-path") + // Backward compatibility + if b.archivePath == "" { + b.archivePath = config.GetString("dir-path") + } b.ignoreIncognito = config.GetBool("ignore-incognito", true) b.ignoreSkipped = config.GetBool("ignore-skipped", false) b.skippedMinSeconds = config.GetInt("ignore-min-duration-seconds", 30) @@ -73,11 +73,19 @@ func (b *SpotifyHistoryBackend) InitConfig(config *config.ServiceConfig) error { } func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { - files, err := filepath.Glob(filepath.Join(b.dirPath, historyFileGlob)) p := models.TransferProgress{ Export: &models.Progress{}, } + archive, err := OpenHistoryArchive(b.archivePath) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + files, err := archive.GetHistoryFiles() if err != nil { p.Export.Abort() progress <- p @@ -85,10 +93,9 @@ func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimesta return } - slices.Sort(files) fileCount := int64(len(files)) p.Export.Total = fileCount - for i, filePath := range files { + for i, f := range files { if err := ctx.Err(); err != nil { results <- models.ListensResult{Error: err} p.Export.Abort() @@ -96,7 +103,7 @@ func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimesta return } - history, err := readHistoryFile(filePath) + history, err := readHistoryFile(f.File) if err != nil { results <- models.ListensResult{Error: err} p.Export.Abort() @@ -118,19 +125,3 @@ func (b *SpotifyHistoryBackend) ExportListens(ctx context.Context, oldestTimesta p.Export.Complete() progress <- p } - -func readHistoryFile(filePath string) (StreamingHistory, error) { - file, err := os.Open(filePath) - if err != nil { - return nil, err - } - - defer file.Close() - history := StreamingHistory{} - err = history.Read(file) - if err != nil { - return nil, err - } - - return history, nil -} From 1ef498943b6a1db48e60ef250b4b567a6613bc33 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 17:38:19 +0200 Subject: [PATCH 62/77] Renamed parameter for lbarchive also to "archive-file" --- config.example.toml | 6 ++++-- internal/backends/lbarchive/lbarchive.go | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/config.example.toml b/config.example.toml index 28c37ad..d01a51a 100644 --- a/config.example.toml +++ b/config.example.toml @@ -23,8 +23,10 @@ check-duplicate-listens = false # This backend supports listens from a ListenBrainz export archive # (https://listenbrainz.org/settings/export/). backend = "listenbrainz-archive" -# The file path to the ListenBrainz export archive. -file-path = "./listenbrainz_outsidecontext.zip" +# The file path to the ListenBrainz export archive. The path can either point +# to the ZIP file as downloaded from ListenBrainz or a directory were the +# ZIP was extracted to. +archive-path = "./listenbrainz_outsidecontext.zip" [service.maloja] # Maloja is a self hosted listening service (https://github.com/krateng/maloja) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 6e2f349..a91c0a5 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -49,14 +49,14 @@ func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archiv func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { return []models.BackendOption{{ - Name: "file-path", - Label: i18n.Tr("Export ZIP file path"), + Name: "archive-path", + Label: i18n.Tr("Archive path"), Type: models.String, }} } func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { - b.filePath = config.GetString("file-path") + b.filePath = config.GetString("archive-path") b.lbClient = listenbrainz.NewClient("", version.UserAgent()) b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: version.AppName, From 93767df5679f39063114616958381c3ed1760409 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 17:54:24 +0200 Subject: [PATCH 63/77] Allow editing config option after renaming --- internal/backends/spotifyhistory/spotifyhistory.go | 9 +++++---- internal/cli/services.go | 6 ++++++ internal/models/options.go | 11 ++++++----- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 90ee8ff..5f67604 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -38,10 +38,11 @@ func (b *SpotifyHistoryBackend) Name() string { return "spotify-history" } func (b *SpotifyHistoryBackend) Options() []models.BackendOption { return []models.BackendOption{{ - Name: "archive-path", - Label: i18n.Tr("Archive path"), - Type: models.String, - Default: "./my_spotify_data_extended.zip", + Name: "archive-path", + Label: i18n.Tr("Archive path"), + Type: models.String, + Default: "./my_spotify_data_extended.zip", + MigrateFrom: "dir-path", }, { Name: "ignore-incognito", Label: i18n.Tr("Ignore listens in incognito mode"), diff --git a/internal/cli/services.go b/internal/cli/services.go index df27833..65e4337 100644 --- a/internal/cli/services.go +++ b/internal/cli/services.go @@ -83,6 +83,12 @@ func PromptExtraOptions(config config.ServiceConfig) (config.ServiceConfig, erro current, exists := config.ConfigValues[opt.Name] if exists { opt.Default = fmt.Sprintf("%v", current) + } else if opt.MigrateFrom != "" { + // If there is an old value to migrate from, try that + fallback, exists := config.ConfigValues[opt.MigrateFrom] + if exists { + opt.Default = fmt.Sprintf("%v", fallback) + } } val, err := Prompt(opt) diff --git a/internal/models/options.go b/internal/models/options.go index ffa3ae6..0e09dd7 100644 --- a/internal/models/options.go +++ b/internal/models/options.go @@ -25,9 +25,10 @@ const ( ) type BackendOption struct { - Name string - Label string - Type OptionType - Default string - Validate func(string) error + Name string + Label string + Type OptionType + Default string + Validate func(string) error + MigrateFrom string } From c29b2e20cd0d30b361267db98c92db2fc3c71e7b Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 18:22:42 +0200 Subject: [PATCH 64/77] deezer: fixed endless export loop if user's listen history is empty --- internal/backends/deezer/deezer.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index c38f4e7..a6eaec2 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -105,6 +105,11 @@ out: return } + // No result, break immediately + if result.Total == 0 { + break out + } + // The offset was higher then the actual number of tracks. Adjust the offset // and continue. if offset >= result.Total { From b18a6c210427a97d93a0aab2bf3b8c4710e7398d Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 18:30:26 +0200 Subject: [PATCH 65/77] Update changelog and README Clarify that some services are not suited for full listen history export --- CHANGES.md | 17 +++++++++++++++++ README.md | 8 +++++++- config.example.toml | 6 +++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 486d0ff..5ccf6d0 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,22 @@ # Scotty Changelog +## 0.7.0 - WIP +- listenbrainz-archive: new backend to load listens and loves from a + ListenBrainz export. The data can be read from the downloaded ZIP archive + or a directory where the contents of the archive have been extracted to. +- listenbrainz: faster loading of missing loves metadata using the ListenBrainz + API instead of MusicBrainz. Fallback to slower MusicBrainz query, if + ListenBrainz does not provide the data. +- spotify-history: it is now possible to specify the path directly to the + `my_spotify_data_extended.zip` ZIP file as downloaded from Spotify. +- spotify-history: the parameter to the export archive path has been renamed to + `archive-path`. For backward compatibility the old `dir-path` parameter is + still read. +- deezer: fixed endless export loop if the user's listen history was empty. +- dump: it is now possible to specify a file to write the text output to. +- Fixed potential issues with MusicBrainz rate limiting. + + ## 0.6.0 - 2025-05-23 - Fully reworked progress report - Cancel both export and import on error diff --git a/README.md b/README.md index 6f997ed..b10a030 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ funkwhale | ✓ | ⨯ | ✓ | - jspf | ✓ | ✓ | ✓ | ✓ lastfm | ✓ | ✓ | ✓ | ✓ listenbrainz | ✓ | ✓ | ✓ | ✓ -listenbrainz-archive | ✓ | - | - | - +listenbrainz-archive | ✓ | - | ✓ | - maloja | ✓ | ✓ | ⨯ | ⨯ scrobbler-log | ✓ | ✓ | ⨯ | ⨯ spotify | ✓ | ⨯ | ✓ | - @@ -135,6 +135,12 @@ subsonic | ⨯ | ⨯ | ✓ | - See the comments in [config.example.toml](./config.example.toml) for a description of each backend's available configuration options. +**NOTE:** Some services, e.g. the Spotify and Deezer API, do not provide access +to the user's full listening history. Hence the API integrations are not suited +to do a full history export. They can however be well used for continuously +transfer recent listens to other services when running scotty frequently, e.g. +as a cron job. + ## Contribute The source code for Scotty is available on [SourceHut](https://sr.ht/~phw/scotty/). To report issues or feature requests please [create a ticket](https://todo.sr.ht/~phw/scotty). diff --git a/config.example.toml b/config.example.toml index d01a51a..3acdf88 100644 --- a/config.example.toml +++ b/config.example.toml @@ -106,6 +106,8 @@ client-secret = "" [service.spotify-history] # Read listens from a Spotify extended history export +# NOTE: The Spotify API does not allow access to the full listen history, +# but only to recent listens. backend = "spotify-history" # Path to the Spotify extended history archive. This can either point directly # to the "my_spotify_data_extended.zip" ZIP file provided by Spotify or a @@ -122,7 +124,9 @@ ignore-skipped = false ignore-min-duration-seconds = 30 [service.deezer] -# Read listens and loves from a Deezer account +# Read listens and loves from a Deezer account. +# NOTE: The Deezer API does not allow access to the full listen history, +# but only to recent listens. backend = "deezer" # You need to register an application on https://developers.deezer.com/myapps # and set the client ID and client secret below. From b1b0df7763f00323c5740fe691084ec623edbcb5 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 18:52:15 +0200 Subject: [PATCH 66/77] listenbrainz: fixed timestamp update with duplicates --- CHANGES.md | 2 ++ internal/backends/listenbrainz/listenbrainz.go | 1 + 2 files changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 5ccf6d0..40d2f73 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,8 @@ - listenbrainz: faster loading of missing loves metadata using the ListenBrainz API instead of MusicBrainz. Fallback to slower MusicBrainz query, if ListenBrainz does not provide the data. +- listenbrainz: fixed issue were timestamp was not updated properly if + duplicate listens where detected during import. - spotify-history: it is now possible to specify the path directly to the `my_spotify_data_extended.zip` ZIP file as downloaded from Spotify. - spotify-history: the parameter to the export archive path has been renamed to diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 9e1c9f3..dcc28fa 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -165,6 +165,7 @@ func (b *ListenBrainzApiBackend) ImportListens(ctx context.Context, export model msg := i18n.Tr("Ignored duplicate listen %v: \"%v\" by %v (%v)", l.ListenedAt, l.TrackName, l.ArtistName(), l.RecordingMBID) importResult.Log(models.Info, msg) + importResult.UpdateTimestamp(l.ListenedAt) continue } } From 312d9860cf84f075f6bf5a6392fc2223a7fb3fd3 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 20:43:02 +0200 Subject: [PATCH 67/77] Fixed import log output duplicating --- CHANGES.md | 1 + internal/backends/import.go | 5 +++-- internal/models/models.go | 20 +++++++++++++++----- internal/models/models_test.go | 22 ++++++++++++++++++++-- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 40d2f73..2257aaa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -17,6 +17,7 @@ - deezer: fixed endless export loop if the user's listen history was empty. - dump: it is now possible to specify a file to write the text output to. - Fixed potential issues with MusicBrainz rate limiting. +- Fixed import log output duplicating. ## 0.6.0 - 2025-05-23 diff --git a/internal/backends/import.go b/internal/backends/import.go index e7a6add..97912dd 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -112,8 +112,9 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( return } - importResult, err := processor.Import(ctx, exportResult, result, out, progress) - result.Update(importResult) + importResult, err := processor.Import( + ctx, exportResult, result.Copy(), out, progress) + result.Update(&importResult) if err != nil { processor.ImportBackend().FinishImport() out <- handleError(result, err, progress) diff --git a/internal/models/models.go b/internal/models/models.go index 78d9965..a93a043 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -196,11 +196,21 @@ func (i *ImportResult) UpdateTimestamp(newTime time.Time) { } } -func (i *ImportResult) Update(from ImportResult) { - i.TotalCount = from.TotalCount - i.ImportCount = from.ImportCount - i.UpdateTimestamp(from.LastTimestamp) - i.ImportLog = append(i.ImportLog, from.ImportLog...) +func (i *ImportResult) Update(from *ImportResult) { + if i != from { + i.TotalCount = from.TotalCount + i.ImportCount = from.ImportCount + i.UpdateTimestamp(from.LastTimestamp) + i.ImportLog = append(i.ImportLog, from.ImportLog...) + } +} + +func (i *ImportResult) Copy() ImportResult { + return ImportResult{ + TotalCount: i.TotalCount, + ImportCount: i.ImportCount, + LastTimestamp: i.LastTimestamp, + } } func (i *ImportResult) Log(t LogEntryType, msg string) { diff --git a/internal/models/models_test.go b/internal/models/models_test.go index 5395610..47ef86f 100644 --- a/internal/models/models_test.go +++ b/internal/models/models_test.go @@ -1,5 +1,5 @@ /* -Copyright © 2023 Philipp Wolfer +Copyright © 2023-2025 Philipp Wolfer Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -138,13 +138,31 @@ func TestImportResultUpdate(t *testing.T) { LastTimestamp: time.Now().Add(1 * time.Hour), ImportLog: []models.LogEntry{logEntry2}, } - result.Update(newResult) + result.Update(&newResult) assert.Equal(t, 120, result.TotalCount) assert.Equal(t, 50, result.ImportCount) assert.Equal(t, newResult.LastTimestamp, result.LastTimestamp) assert.Equal(t, []models.LogEntry{logEntry1, logEntry2}, result.ImportLog) } +func TestImportResultCopy(t *testing.T) { + logEntry := models.LogEntry{ + Type: models.Warning, + Message: "foo", + } + result := models.ImportResult{ + TotalCount: 100, + ImportCount: 20, + LastTimestamp: time.Now(), + ImportLog: []models.LogEntry{logEntry}, + } + copy := result.Copy() + assert.Equal(t, result.TotalCount, copy.TotalCount) + assert.Equal(t, result.ImportCount, copy.ImportCount) + assert.Equal(t, result.LastTimestamp, copy.LastTimestamp) + assert.Empty(t, copy.ImportLog) +} + func TestImportResultLog(t *testing.T) { result := models.ImportResult{} result.Log(models.Warning, "foo") From 4da569743555301b18ffc993f5bcfffb4d625f98 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 20:54:20 +0200 Subject: [PATCH 68/77] If dump does no write to file, output the result as log --- internal/backends/dump/dump.go | 7 +++++-- internal/backends/import.go | 6 +++--- internal/backends/jspf/jspf.go | 2 +- internal/backends/lastfm/lastfm.go | 6 ++++-- internal/backends/listenbrainz/listenbrainz.go | 6 ++++-- internal/backends/maloja/maloja.go | 6 ++++-- internal/backends/scrobblerlog/scrobblerlog.go | 2 +- internal/cli/transfer.go | 6 +++++- internal/models/interfaces.go | 2 +- internal/models/models.go | 1 + 10 files changed, 29 insertions(+), 15 deletions(-) diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index 4714bd6..8d7c641 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -73,14 +73,17 @@ func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error { func (b *DumpBackend) StartImport() error { return nil } -func (b *DumpBackend) FinishImport() error { +func (b *DumpBackend) FinishImport(result *models.ImportResult) error { if b.print { out := new(strings.Builder) _, err := io.Copy(out, b.buffer) if err != nil { return err } - fmt.Println(out.String()) + + if result != nil { + result.Log(models.Output, out.String()) + } } // Close the io writer if it is closable diff --git a/internal/backends/import.go b/internal/backends/import.go index 97912dd..ae6da92 100644 --- a/internal/backends/import.go +++ b/internal/backends/import.go @@ -107,7 +107,7 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( for exportResult := range results { if err := ctx.Err(); err != nil { - processor.ImportBackend().FinishImport() + processor.ImportBackend().FinishImport(&result) out <- handleError(result, err, progress) return } @@ -116,14 +116,14 @@ func process[R models.LovesResult | models.ListensResult, P ImportProcessor[R]]( ctx, exportResult, result.Copy(), out, progress) result.Update(&importResult) if err != nil { - processor.ImportBackend().FinishImport() + processor.ImportBackend().FinishImport(&result) out <- handleError(result, err, progress) return } progress <- p.FromImportResult(result, false) } - if err := processor.ImportBackend().FinishImport(); err != nil { + if err := processor.ImportBackend().FinishImport(&result); err != nil { out <- handleError(result, err, progress) return } diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index e2bcde1..887fd72 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -90,7 +90,7 @@ func (b *JSPFBackend) StartImport() error { return b.readJSPF() } -func (b *JSPFBackend) FinishImport() error { +func (b *JSPFBackend) FinishImport(result *models.ImportResult) error { return b.writeJSPF() } diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index b34452e..186a631 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -70,8 +70,10 @@ func (b *LastfmApiBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *LastfmApiBackend) StartImport() error { return nil } -func (b *LastfmApiBackend) FinishImport() error { return nil } +func (b *LastfmApiBackend) StartImport() error { return nil } +func (b *LastfmApiBackend) FinishImport(result *models.ImportResult) error { + return nil +} func (b *LastfmApiBackend) OAuth2Strategy(redirectURL *url.URL) auth.OAuth2Strategy { return lastfmStrategy{ diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index dcc28fa..98d1525 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -73,8 +73,10 @@ func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error return nil } -func (b *ListenBrainzApiBackend) StartImport() error { return nil } -func (b *ListenBrainzApiBackend) FinishImport() error { return nil } +func (b *ListenBrainzApiBackend) StartImport() error { return nil } +func (b *ListenBrainzApiBackend) FinishImport(result *models.ImportResult) error { + return nil +} func (b *ListenBrainzApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { startTime := time.Now() diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index f082d9b..d85309f 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -61,8 +61,10 @@ func (b *MalojaApiBackend) InitConfig(config *config.ServiceConfig) error { return nil } -func (b *MalojaApiBackend) StartImport() error { return nil } -func (b *MalojaApiBackend) FinishImport() error { return nil } +func (b *MalojaApiBackend) StartImport() error { return nil } +func (b *MalojaApiBackend) FinishImport(result *models.ImportResult) error { + return nil +} func (b *MalojaApiBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { page := 0 diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 6d42f3c..13aecba 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -126,7 +126,7 @@ func (b *ScrobblerLogBackend) StartImport() error { return nil } -func (b *ScrobblerLogBackend) FinishImport() error { +func (b *ScrobblerLogBackend) FinishImport(result *models.ImportResult) error { return b.file.Close() } diff --git a/internal/cli/transfer.go b/internal/cli/transfer.go index 3aabb4b..7c5ecc0 100644 --- a/internal/cli/transfer.go +++ b/internal/cli/transfer.go @@ -157,7 +157,11 @@ func (c *TransferCmd[E, I, R]) Transfer(exp backends.ExportProcessor[R], imp bac fmt.Println() fmt.Println(i18n.Tr("Import log:")) for _, entry := range result.ImportLog { - fmt.Println(i18n.Tr("%v: %v", entry.Type, entry.Message)) + if entry.Type != models.Output { + fmt.Println(i18n.Tr("%v: %v", entry.Type, entry.Message)) + } else { + fmt.Println(entry.Message) + } } } diff --git a/internal/models/interfaces.go b/internal/models/interfaces.go index 2f4beaf..79a4c6c 100644 --- a/internal/models/interfaces.go +++ b/internal/models/interfaces.go @@ -46,7 +46,7 @@ type ImportBackend interface { // The implementation can perform all steps here to finalize the // export/import and free used resources. - FinishImport() error + FinishImport(result *ImportResult) error } // Must be implemented by services supporting the export of listens. diff --git a/internal/models/models.go b/internal/models/models.go index a93a043..69280b3 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -169,6 +169,7 @@ type LovesResult ExportResult[LovesList] type LogEntryType string const ( + Output LogEntryType = "" Info LogEntryType = "Info" Warning LogEntryType = "Warning" Error LogEntryType = "Error" From 28c618ffcebce1a973d45fdfcd2e3b18a2f64094 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 12:46:44 +0200 Subject: [PATCH 69/77] Implemented tests and added documentation for archive --- internal/archive/archive.go | 31 +++- internal/archive/archive_test.go | 189 ++++++++++++++++++++ internal/archive/testdata/archive.zip | Bin 0 -> 835 bytes internal/archive/testdata/archive/a/1.txt | 1 + internal/archive/testdata/archive/b/1.txt | 1 + internal/archive/testdata/archive/b/2.txt | 1 + internal/backends/spotifyhistory/archive.go | 2 +- internal/listenbrainz/archive.go | 2 +- 8 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 internal/archive/archive_test.go create mode 100644 internal/archive/testdata/archive.zip create mode 100644 internal/archive/testdata/archive/a/1.txt create mode 100644 internal/archive/testdata/archive/b/1.txt create mode 100644 internal/archive/testdata/archive/b/2.txt diff --git a/internal/archive/archive.go b/internal/archive/archive.go index 7714552..3c2ee86 100644 --- a/internal/archive/archive.go +++ b/internal/archive/archive.go @@ -35,10 +35,17 @@ import ( "path/filepath" ) -// Generic archive interface. -type Archive interface { - Close() error +// Generic interface to access files inside an archive. +type ArchiveReader interface { + io.Closer + + // Open the file inside the archive identified by the given path. + // The path is relative to the archive's root. + // The caller must call [fs.File.Close] when finished using the file. Open(path string) (fs.File, error) + + // List files inside the archive which satisfy the given glob pattern. + // This method only returns files, not directories. Glob(pattern string) ([]FileInfo, error) } @@ -46,7 +53,7 @@ type Archive interface { // The archive can be a ZIP file or a directory. The implementation // will detect the type of archive and return the appropriate // implementation of the Archive interface. -func OpenArchive(path string) (Archive, error) { +func OpenArchive(path string) (ArchiveReader, error) { fi, err := os.Stat(path) if err != nil { return nil, err @@ -73,10 +80,14 @@ func OpenArchive(path string) (Archive, error) { // Interface for a file that can be opened when needed. type OpenableFile interface { + // Open the file for reading. + // The caller is responsible to call [io.ReadCloser.Close] when + // finished reading the file. Open() (io.ReadCloser, error) } // Generic information about a file inside an archive. +// This provides the filename and allows opening the file for reading. type FileInfo struct { Name string File OpenableFile @@ -115,6 +126,10 @@ func (a *zipArchive) Close() error { func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { result := make([]FileInfo, 0) for _, file := range a.zip.File { + if file.FileInfo().IsDir() { + continue + } + if matched, err := filepath.Match(pattern, file.Name); matched { if err != nil { return nil, err @@ -167,6 +182,14 @@ func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { } result := make([]FileInfo, 0) for _, name := range files { + stat, err := fs.Stat(a.dirFS, name) + if err != nil { + return nil, err + } + if stat.IsDir() { + continue + } + fullPath := filepath.Join(a.path, name) info := FileInfo{ Name: name, diff --git a/internal/archive/archive_test.go b/internal/archive/archive_test.go new file mode 100644 index 0000000..6f3f6db --- /dev/null +++ b/internal/archive/archive_test.go @@ -0,0 +1,189 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package archive_test + +import ( + "fmt" + "io" + "log" + "slices" + "testing" + + "go.uploadedlobster.com/scotty/internal/archive" +) + +func ExampleOpenArchive() { + a, err := archive.OpenArchive("testdata/archive.zip") + if err != nil { + log.Fatal(err) + } + + defer a.Close() + + files, err := a.Glob("a/*.txt") + for _, fi := range files { + fmt.Println(fi.Name) + f, err := fi.File.Open() + if err != nil { + log.Fatal(err) + } + + defer f.Close() + data, err := io.ReadAll(f) + if err != nil { + log.Fatal(err) + } + fmt.Println(string(data)) + } + + // Output: a/1.txt + // a1 +} + +var testArchives = []string{ + "testdata/archive", + "testdata/archive.zip", +} + +func TestGlob(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + files, err := a.Glob("[ab]/1.txt") + if err != nil { + t.Fatal(err) + } + + if len(files) != 2 { + t.Errorf("Expected 2 files, got %d", len(files)) + } + + expectedName := "b/1.txt" + var fileInfo *archive.FileInfo = nil + for _, file := range files { + if file.Name == expectedName { + fileInfo = &file + } + } + + if fileInfo == nil { + t.Fatalf("Expected file %q to be found", expectedName) + } + + if fileInfo.File == nil { + t.Fatalf("Expected FileInfo to hold an openable File") + } + + f, err := fileInfo.File.Open() + if err != nil { + t.Fatal(err) + } + + expectedData := "b1\n" + data, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(data) != expectedData { + fmt.Printf("%s: Expected file content to be %q, got %q", + path, expectedData, string(data)) + } + } +} + +func TestGlobAll(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + files, err := a.Glob("*/*") + if err != nil { + t.Fatal(err) + } + + filenames := make([]string, 0, len(files)) + for _, f := range files { + fmt.Printf("%v: %v\n", path, f.Name) + filenames = append(filenames, f.Name) + } + + slices.Sort(filenames) + + expectedFilenames := []string{ + "a/1.txt", + "b/1.txt", + "b/2.txt", + } + if !slices.Equal(filenames, expectedFilenames) { + t.Errorf("%s: Expected filenames to be %q, got %q", + path, expectedFilenames, filenames) + } + } +} + +func TestOpen(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + f, err := a.Open("b/2.txt") + if err != nil { + t.Fatal(err) + } + + expectedData := "b2\n" + data, err := io.ReadAll(f) + if err != nil { + t.Fatal(err) + } + if string(data) != expectedData { + fmt.Printf("%s: Expected file content to be %q, got %q", + path, expectedData, string(data)) + } + } +} + +func TestOpenError(t *testing.T) { + for _, path := range testArchives { + a, err := archive.OpenArchive(path) + if err != nil { + t.Fatal(err) + } + defer a.Close() + + _, err = a.Open("b/3.txt") + if err == nil { + t.Errorf("%s: Expected the Open command to fail", path) + } + } +} diff --git a/internal/archive/testdata/archive.zip b/internal/archive/testdata/archive.zip new file mode 100644 index 0000000000000000000000000000000000000000..19923f60d9aae15128d24d3ca3918eeeb96f18d9 GIT binary patch literal 835 zcmWIWW@Zs#00GYUol#&0lwe{|U`W&t4dG>AcY0}*0mG#g+zgB?FPIq^z(fF8KL-N` z0~3;dpezGBSihlONks|R5L2KbFpOr1x5*PGh5&DNj+N8yOkf4-1?h*F3~~%FlF1-G z$Yn_omw5vX^#x)Ojbt51CW`h zk8n9Gra&~B%LBX-j>HxY3=9C;X`jac literal 0 HcmV?d00001 diff --git a/internal/archive/testdata/archive/a/1.txt b/internal/archive/testdata/archive/a/1.txt new file mode 100644 index 0000000..da0f8ed --- /dev/null +++ b/internal/archive/testdata/archive/a/1.txt @@ -0,0 +1 @@ +a1 diff --git a/internal/archive/testdata/archive/b/1.txt b/internal/archive/testdata/archive/b/1.txt new file mode 100644 index 0000000..c9c6af7 --- /dev/null +++ b/internal/archive/testdata/archive/b/1.txt @@ -0,0 +1 @@ +b1 diff --git a/internal/archive/testdata/archive/b/2.txt b/internal/archive/testdata/archive/b/2.txt new file mode 100644 index 0000000..e6bfff5 --- /dev/null +++ b/internal/archive/testdata/archive/b/2.txt @@ -0,0 +1 @@ +b2 diff --git a/internal/backends/spotifyhistory/archive.go b/internal/backends/spotifyhistory/archive.go index 1d596bd..cb53772 100644 --- a/internal/backends/spotifyhistory/archive.go +++ b/internal/backends/spotifyhistory/archive.go @@ -33,7 +33,7 @@ var historyFileGlobs = []string{ // This can be either the ZIP file as provided by Spotify // or a directory where this was extracted to. type HistoryArchive struct { - backend archive.Archive + backend archive.ArchiveReader } // Open a Spotify history archive from file path. diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index b7b5909..57c3ad8 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -40,7 +40,7 @@ import ( // The export contains the user's listen history, favorite tracks and // user information. type ExportArchive struct { - backend archive.Archive + backend archive.ArchiveReader } // Open a ListenBrainz archive from file path. From 1244405747c1f720dccd52b9f79a10fc3fc6a024 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 12:51:36 +0200 Subject: [PATCH 70/77] Moved archive package to public pkg/ --- internal/backends/spotifyhistory/archive.go | 2 +- internal/listenbrainz/archive.go | 2 +- {internal => pkg}/archive/archive.go | 101 ------------------ {internal => pkg}/archive/archive_test.go | 2 +- pkg/archive/dir.go | 77 +++++++++++++ .../archive/testdata/archive.zip | Bin .../archive/testdata/archive/a/1.txt | 0 .../archive/testdata/archive/b/1.txt | 0 .../archive/testdata/archive/b/2.txt | 0 pkg/archive/zip.go | 80 ++++++++++++++ 10 files changed, 160 insertions(+), 104 deletions(-) rename {internal => pkg}/archive/archive.go (61%) rename {internal => pkg}/archive/archive_test.go (98%) create mode 100644 pkg/archive/dir.go rename {internal => pkg}/archive/testdata/archive.zip (100%) rename {internal => pkg}/archive/testdata/archive/a/1.txt (100%) rename {internal => pkg}/archive/testdata/archive/b/1.txt (100%) rename {internal => pkg}/archive/testdata/archive/b/2.txt (100%) create mode 100644 pkg/archive/zip.go diff --git a/internal/backends/spotifyhistory/archive.go b/internal/backends/spotifyhistory/archive.go index cb53772..2f9a2ec 100644 --- a/internal/backends/spotifyhistory/archive.go +++ b/internal/backends/spotifyhistory/archive.go @@ -21,7 +21,7 @@ import ( "errors" "sort" - "go.uploadedlobster.com/scotty/internal/archive" + "go.uploadedlobster.com/scotty/pkg/archive" ) var historyFileGlobs = []string{ diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index 57c3ad8..68740aa 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -32,7 +32,7 @@ import ( "time" "github.com/simonfrey/jsonl" - "go.uploadedlobster.com/scotty/internal/archive" + "go.uploadedlobster.com/scotty/pkg/archive" ) // Represents a ListenBrainz export archive. diff --git a/internal/archive/archive.go b/pkg/archive/archive.go similarity index 61% rename from internal/archive/archive.go rename to pkg/archive/archive.go index 3c2ee86..41c954f 100644 --- a/internal/archive/archive.go +++ b/pkg/archive/archive.go @@ -27,12 +27,10 @@ THE SOFTWARE. package archive import ( - "archive/zip" "fmt" "io" "io/fs" "os" - "path/filepath" ) // Generic interface to access files inside an archive. @@ -101,102 +99,3 @@ type filesystemFile struct { func (f *filesystemFile) Open() (io.ReadCloser, error) { return os.Open(f.path) } - -// An implementation of the archiveBackend interface for zip files. -type zipArchive struct { - zip *zip.ReadCloser -} - -func (a *zipArchive) OpenArchive(path string) error { - zip, err := zip.OpenReader(path) - if err != nil { - return err - } - a.zip = zip - return nil -} - -func (a *zipArchive) Close() error { - if a.zip == nil { - return nil - } - return a.zip.Close() -} - -func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { - result := make([]FileInfo, 0) - for _, file := range a.zip.File { - if file.FileInfo().IsDir() { - continue - } - - if matched, err := filepath.Match(pattern, file.Name); matched { - if err != nil { - return nil, err - } - info := FileInfo{ - Name: file.Name, - File: file, - } - result = append(result, info) - } - } - - return result, nil -} - -func (a *zipArchive) Open(path string) (fs.File, error) { - file, err := a.zip.Open(path) - if err != nil { - return nil, err - } - return file, nil -} - -// An implementation of the archiveBackend interface for directories. -type dirArchive struct { - path string - dirFS fs.FS -} - -func (a *dirArchive) OpenArchive(path string) error { - a.path = filepath.Clean(path) - a.dirFS = os.DirFS(path) - return nil -} - -func (a *dirArchive) Close() error { - return nil -} - -// Open opens the named file in the archive. -// [fs.File.Close] must be called to release any associated resources. -func (a *dirArchive) Open(path string) (fs.File, error) { - return a.dirFS.Open(path) -} - -func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { - files, err := fs.Glob(a.dirFS, pattern) - if err != nil { - return nil, err - } - result := make([]FileInfo, 0) - for _, name := range files { - stat, err := fs.Stat(a.dirFS, name) - if err != nil { - return nil, err - } - if stat.IsDir() { - continue - } - - fullPath := filepath.Join(a.path, name) - info := FileInfo{ - Name: name, - File: &filesystemFile{path: fullPath}, - } - result = append(result, info) - } - - return result, nil -} diff --git a/internal/archive/archive_test.go b/pkg/archive/archive_test.go similarity index 98% rename from internal/archive/archive_test.go rename to pkg/archive/archive_test.go index 6f3f6db..f1bbd07 100644 --- a/internal/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -29,7 +29,7 @@ import ( "slices" "testing" - "go.uploadedlobster.com/scotty/internal/archive" + "go.uploadedlobster.com/scotty/pkg/archive" ) func ExampleOpenArchive() { diff --git a/pkg/archive/dir.go b/pkg/archive/dir.go new file mode 100644 index 0000000..166e70b --- /dev/null +++ b/pkg/archive/dir.go @@ -0,0 +1,77 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package archive + +import ( + "io/fs" + "os" + "path/filepath" +) + +// An implementation of the [ArchiveReader] interface for directories. +type dirArchive struct { + path string + dirFS fs.FS +} + +func (a *dirArchive) OpenArchive(path string) error { + a.path = filepath.Clean(path) + a.dirFS = os.DirFS(path) + return nil +} + +func (a *dirArchive) Close() error { + return nil +} + +// Open opens the named file in the archive. +// [fs.File.Close] must be called to release any associated resources. +func (a *dirArchive) Open(path string) (fs.File, error) { + return a.dirFS.Open(path) +} + +func (a *dirArchive) Glob(pattern string) ([]FileInfo, error) { + files, err := fs.Glob(a.dirFS, pattern) + if err != nil { + return nil, err + } + result := make([]FileInfo, 0) + for _, name := range files { + stat, err := fs.Stat(a.dirFS, name) + if err != nil { + return nil, err + } + if stat.IsDir() { + continue + } + + fullPath := filepath.Join(a.path, name) + info := FileInfo{ + Name: name, + File: &filesystemFile{path: fullPath}, + } + result = append(result, info) + } + + return result, nil +} diff --git a/internal/archive/testdata/archive.zip b/pkg/archive/testdata/archive.zip similarity index 100% rename from internal/archive/testdata/archive.zip rename to pkg/archive/testdata/archive.zip diff --git a/internal/archive/testdata/archive/a/1.txt b/pkg/archive/testdata/archive/a/1.txt similarity index 100% rename from internal/archive/testdata/archive/a/1.txt rename to pkg/archive/testdata/archive/a/1.txt diff --git a/internal/archive/testdata/archive/b/1.txt b/pkg/archive/testdata/archive/b/1.txt similarity index 100% rename from internal/archive/testdata/archive/b/1.txt rename to pkg/archive/testdata/archive/b/1.txt diff --git a/internal/archive/testdata/archive/b/2.txt b/pkg/archive/testdata/archive/b/2.txt similarity index 100% rename from internal/archive/testdata/archive/b/2.txt rename to pkg/archive/testdata/archive/b/2.txt diff --git a/pkg/archive/zip.go b/pkg/archive/zip.go new file mode 100644 index 0000000..0054cf3 --- /dev/null +++ b/pkg/archive/zip.go @@ -0,0 +1,80 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package archive + +import ( + "archive/zip" + "io/fs" + "path/filepath" +) + +// An implementation of the [ArchiveReader] interface for zip files. +type zipArchive struct { + zip *zip.ReadCloser +} + +func (a *zipArchive) OpenArchive(path string) error { + zip, err := zip.OpenReader(path) + if err != nil { + return err + } + a.zip = zip + return nil +} + +func (a *zipArchive) Close() error { + if a.zip == nil { + return nil + } + return a.zip.Close() +} + +func (a *zipArchive) Glob(pattern string) ([]FileInfo, error) { + result := make([]FileInfo, 0) + for _, file := range a.zip.File { + if file.FileInfo().IsDir() { + continue + } + + if matched, err := filepath.Match(pattern, file.Name); matched { + if err != nil { + return nil, err + } + info := FileInfo{ + Name: file.Name, + File: file, + } + result = append(result, info) + } + } + + return result, nil +} + +func (a *zipArchive) Open(path string) (fs.File, error) { + file, err := a.zip.Open(path) + if err != nil { + return nil, err + } + return file, nil +} From e85090fe4a1143c5b1c3734d06cb29a9f35669c5 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 15:34:56 +0200 Subject: [PATCH 71/77] Implemented deezer-history backend listen import --- README.md | 1 + config.example.toml | 13 +- go.mod | 7 + go.sum | 15 +- internal/backends/backends.go | 2 + internal/backends/backends_test.go | 3 + .../backends/deezerhistory/deezerhistory.go | 134 ++++++++++++++++++ 7 files changed, 171 insertions(+), 4 deletions(-) create mode 100644 internal/backends/deezerhistory/deezerhistory.go diff --git a/README.md b/README.md index b10a030..6f8e378 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,7 @@ The following table lists the available backends and the currently supported fea Backend | Listens Export | Listens Import | Loves Export | Loves Import ---------------------|----------------|----------------|--------------|------------- deezer | ✓ | ⨯ | ✓ | - +deezer-history | ✓ | ⨯ | - | ⨯ funkwhale | ✓ | ⨯ | ✓ | - jspf | ✓ | ✓ | ✓ | ✓ lastfm | ✓ | ✓ | ✓ | ✓ diff --git a/config.example.toml b/config.example.toml index 3acdf88..91d5318 100644 --- a/config.example.toml +++ b/config.example.toml @@ -96,6 +96,8 @@ identifier = "" [service.spotify] # Read listens and loves from a Spotify account +# NOTE: The Spotify API does not allow access to the full listen history, +# but only to recent listens. backend = "spotify" # You need to register an application on https://developer.spotify.com/ # and set the client ID and client secret below. @@ -106,8 +108,6 @@ client-secret = "" [service.spotify-history] # Read listens from a Spotify extended history export -# NOTE: The Spotify API does not allow access to the full listen history, -# but only to recent listens. backend = "spotify-history" # Path to the Spotify extended history archive. This can either point directly # to the "my_spotify_data_extended.zip" ZIP file provided by Spotify or a @@ -135,6 +135,15 @@ backend = "deezer" client-id = "" client-secret = "" +[service.deezer-history] +# Read listens from a Deezer data export. +# You can request a download of all your Deezer data, including the complete +# listen history, in the section "My information" in your Deezer +# "Account settings". +backend = "deezer-history" +# Path to XLSX file provided by Deezer, e.g. "deezer-data_520704045.xlsx". +file-path = "" + [service.lastfm] backend = "lastfm" # Your Last.fm username diff --git a/go.mod b/go.mod index c5c3511..5991ecd 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/stretchr/testify v1.10.0 github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d github.com/vbauerster/mpb/v8 v8.10.1 + github.com/xuri/excelize/v2 v2.9.1 go.uploadedlobster.com/mbtypes v0.4.0 go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 @@ -52,13 +53,19 @@ require ( github.com/ncruces/go-strftime v0.1.9 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/richardlehane/mscfb v1.0.4 // indirect + github.com/richardlehane/msoleps v1.0.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tiendc/go-deepcopy v1.6.0 // indirect + github.com/xuri/efp v0.0.1 // indirect + github.com/xuri/nfp v0.0.1 // indirect go.uber.org/multierr v1.11.0 // indirect + golang.org/x/crypto v0.38.0 // indirect golang.org/x/image v0.27.0 // indirect golang.org/x/mod v0.24.0 // indirect golang.org/x/net v0.40.0 // indirect diff --git a/go.sum b/go.sum index 6d34a6d..fefdfa4 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,11 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/richardlehane/mscfb v1.0.4 h1:WULscsljNPConisD5hR0+OyZjwK46Pfyr6mPu5ZawpM= +github.com/richardlehane/mscfb v1.0.4/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= +github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -127,15 +132,21 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4= +github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= +github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= +github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= +github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= +github.com/xuri/excelize/v2 v2.9.1/go.mod h1:x7L6pKz2dvo9ejrRuD8Lnl98z4JLt0TGAwjhW+EiP8s= +github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= +github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s= go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= -go.uploadedlobster.com/musicbrainzws2 v0.15.0 h1:njJeyf1dDwfz2toEHaZSuockVsn1fg+967/tVfLHhwQ= -go.uploadedlobster.com/musicbrainzws2 v0.15.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 h1:bir8kas9u0A+T54sfzj3il7SUAV5KQtb5QzDtwvslxI= go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/internal/backends/backends.go b/internal/backends/backends.go index a1cd407..97a78c2 100644 --- a/internal/backends/backends.go +++ b/internal/backends/backends.go @@ -23,6 +23,7 @@ import ( "strings" "go.uploadedlobster.com/scotty/internal/backends/deezer" + "go.uploadedlobster.com/scotty/internal/backends/deezerhistory" "go.uploadedlobster.com/scotty/internal/backends/dump" "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" @@ -107,6 +108,7 @@ func GetBackends() BackendList { var knownBackends = map[string]func() models.Backend{ "deezer": func() models.Backend { return &deezer.DeezerApiBackend{} }, + "deezer-history": func() models.Backend { return &deezerhistory.DeezerHistoryBackend{} }, "dump": func() models.Backend { return &dump.DumpBackend{} }, "funkwhale": func() models.Backend { return &funkwhale.FunkwhaleApiBackend{} }, "jspf": func() models.Backend { return &jspf.JSPFBackend{} }, diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index 026e487..b30eb95 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/assert" "go.uploadedlobster.com/scotty/internal/backends" "go.uploadedlobster.com/scotty/internal/backends/deezer" + "go.uploadedlobster.com/scotty/internal/backends/deezerhistory" "go.uploadedlobster.com/scotty/internal/backends/dump" "go.uploadedlobster.com/scotty/internal/backends/funkwhale" "go.uploadedlobster.com/scotty/internal/backends/jspf" @@ -86,6 +87,8 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.LovesExport](t, &deezer.DeezerApiBackend{}) // expectInterface[models.LovesImport](t, &deezer.DeezerApiBackend{}) + expectInterface[models.ListensExport](t, &deezerhistory.DeezerHistoryBackend{}) + expectInterface[models.ListensImport](t, &dump.DumpBackend{}) expectInterface[models.LovesImport](t, &dump.DumpBackend{}) diff --git a/internal/backends/deezerhistory/deezerhistory.go b/internal/backends/deezerhistory/deezerhistory.go new file mode 100644 index 0000000..fb8a464 --- /dev/null +++ b/internal/backends/deezerhistory/deezerhistory.go @@ -0,0 +1,134 @@ +/* +Copyright © 2025 Philipp Wolfer + +This file is part of Scotty. + +Scotty is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later version. + +Scotty is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +Scotty. If not, see . +*/ + +package deezerhistory + +import ( + "context" + "fmt" + "sort" + "strconv" + "time" + + "github.com/xuri/excelize/v2" + "go.uploadedlobster.com/mbtypes" + "go.uploadedlobster.com/scotty/internal/config" + "go.uploadedlobster.com/scotty/internal/i18n" + "go.uploadedlobster.com/scotty/internal/models" +) + +const ( + sheetListeningHistory = "10_listeningHistory" + sheetfavoriteSongs = "8_favoriteSong" +) + +type DeezerHistoryBackend struct { + filePath string +} + +func (b *DeezerHistoryBackend) Name() string { return "deezer-history" } + +func (b *DeezerHistoryBackend) Options() []models.BackendOption { + return []models.BackendOption{{ + Name: "file-path", + Label: i18n.Tr("File path"), + Type: models.String, + Default: "", + }} +} + +func (b *DeezerHistoryBackend) InitConfig(config *config.ServiceConfig) error { + b.filePath = config.GetString("file-path") + return nil +} + +func (b *DeezerHistoryBackend) ExportListens(ctx context.Context, oldestTimestamp time.Time, results chan models.ListensResult, progress chan models.TransferProgress) { + p := models.TransferProgress{ + Export: &models.Progress{}, + } + + rows, err := ReadXLSXSheet(b.filePath, sheetListeningHistory) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + + count := len(rows) - 1 // Exclude the header row + p.Export.TotalItems = count + p.Export.Total = int64(count) + + listens := make(models.ListensList, 0, count) + for i, row := range models.IterExportProgress(rows, &p, progress) { + // Skip header row + if i == 0 { + continue + } + + l, err := RowAsListen(row) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.ListensResult{Error: err} + return + } + listens = append(listens, *l) + } + + sort.Sort(listens) + results <- models.ListensResult{Items: listens} + p.Export.Complete() + progress <- p +} + +func ReadXLSXSheet(path string, sheet string) ([][]string, error) { + exc, err := excelize.OpenFile(path) + if err != nil { + return nil, err + } + + // Get all the rows in the Sheet1. + return exc.GetRows(sheetListeningHistory) +} + +func RowAsListen(row []string) (*models.Listen, error) { + if len(row) < 9 { + err := fmt.Errorf("Invalid row, expected 9 columns, got %d", len(row)) + return nil, err + } + + listenedAt, err := time.Parse(time.DateTime, row[8]) + if err != nil { + return nil, err + } + listen := models.Listen{ + ListenedAt: listenedAt, + Track: models.Track{ + TrackName: row[0], + ArtistNames: []string{row[1]}, + ReleaseName: row[3], + ISRC: mbtypes.ISRC(row[2]), + }, + } + + if duration, err := strconv.Atoi(row[5]); err == nil { + listen.PlaybackDuration = time.Duration(duration) * time.Second + } + + return &listen, nil +} From 78a05e9f541e2e41bff637df541c9ca6176c8a5e Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 15:49:06 +0200 Subject: [PATCH 72/77] Implemented deezer-history loves export --- README.md | 2 +- .../backends/deezerhistory/deezerhistory.go | 76 ++++++++++++++++++- 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 6f8e378..3c004c0 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ The following table lists the available backends and the currently supported fea Backend | Listens Export | Listens Import | Loves Export | Loves Import ---------------------|----------------|----------------|--------------|------------- deezer | ✓ | ⨯ | ✓ | - -deezer-history | ✓ | ⨯ | - | ⨯ +deezer-history | ✓ | ⨯ | ✓ | ⨯ funkwhale | ✓ | ⨯ | ✓ | - jspf | ✓ | ✓ | ✓ | ✓ lastfm | ✓ | ✓ | ✓ | ✓ diff --git a/internal/backends/deezerhistory/deezerhistory.go b/internal/backends/deezerhistory/deezerhistory.go index fb8a464..5574a5d 100644 --- a/internal/backends/deezerhistory/deezerhistory.go +++ b/internal/backends/deezerhistory/deezerhistory.go @@ -22,6 +22,7 @@ import ( "fmt" "sort" "strconv" + "strings" "time" "github.com/xuri/excelize/v2" @@ -33,7 +34,7 @@ import ( const ( sheetListeningHistory = "10_listeningHistory" - sheetfavoriteSongs = "8_favoriteSong" + sheetFavoriteSongs = "8_favoriteSong" ) type DeezerHistoryBackend struct { @@ -96,6 +97,46 @@ func (b *DeezerHistoryBackend) ExportListens(ctx context.Context, oldestTimestam progress <- p } +func (b *DeezerHistoryBackend) ExportLoves(ctx context.Context, oldestTimestamp time.Time, results chan models.LovesResult, progress chan models.TransferProgress) { + p := models.TransferProgress{ + Export: &models.Progress{}, + } + + rows, err := ReadXLSXSheet(b.filePath, sheetFavoriteSongs) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + + count := len(rows) - 1 // Exclude the header row + p.Export.TotalItems = count + p.Export.Total = int64(count) + + love := make(models.LovesList, 0, count) + for i, row := range models.IterExportProgress(rows, &p, progress) { + // Skip header row + if i == 0 { + continue + } + + l, err := RowAsLove(row) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + love = append(love, *l) + } + + sort.Sort(love) + results <- models.LovesResult{Items: love} + p.Export.Complete() + progress <- p +} + func ReadXLSXSheet(path string, sheet string) ([][]string, error) { exc, err := excelize.OpenFile(path) if err != nil { @@ -103,7 +144,7 @@ func ReadXLSXSheet(path string, sheet string) ([][]string, error) { } // Get all the rows in the Sheet1. - return exc.GetRows(sheetListeningHistory) + return exc.GetRows(sheet) } func RowAsListen(row []string) (*models.Listen, error) { @@ -123,6 +164,9 @@ func RowAsListen(row []string) (*models.Listen, error) { ArtistNames: []string{row[1]}, ReleaseName: row[3], ISRC: mbtypes.ISRC(row[2]), + AdditionalInfo: map[string]any{ + "music_service": "deezer.com", + }, }, } @@ -132,3 +176,31 @@ func RowAsListen(row []string) (*models.Listen, error) { return &listen, nil } + +func RowAsLove(row []string) (*models.Love, error) { + if len(row) < 5 { + err := fmt.Errorf("Invalid row, expected 5 columns, got %d", len(row)) + return nil, err + } + + url := row[4] + if !strings.HasPrefix(url, "http://") || !strings.HasPrefix(url, "https") { + url = "https://" + url + } + + love := models.Love{ + Track: models.Track{ + TrackName: row[0], + ArtistNames: []string{row[1]}, + ReleaseName: row[2], + ISRC: mbtypes.ISRC(row[3]), + AdditionalInfo: map[string]any{ + "music_service": "deezer.com", + "origin_url": url, + "deezer_id": url, + }, + }, + } + + return &love, nil +} From 0115eca1c6867e185aaa6813432e08b5dbfcf428 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 15:53:01 +0200 Subject: [PATCH 73/77] Minor code cleanup when creation time.Duration --- internal/backends/deezer/deezer.go | 2 +- internal/backends/funkwhale/funkwhale.go | 2 +- internal/backends/spotify/spotify.go | 2 +- internal/backends/spotifyhistory/models.go | 2 +- internal/backends/subsonic/subsonic.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index a6eaec2..9d622df 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -251,7 +251,7 @@ func (t Track) AsTrack() models.Track { TrackName: t.Title, ReleaseName: t.Album.Title, ArtistNames: []string{t.Artist.Name}, - Duration: time.Duration(t.Duration * int(time.Second)), + Duration: time.Duration(t.Duration) * time.Second, AdditionalInfo: map[string]any{}, } diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index d9632a6..8874c70 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -220,7 +220,7 @@ func (t Track) AsTrack() models.Track { } if len(t.Uploads) > 0 { - track.Duration = time.Duration(t.Uploads[0].Duration * int(time.Second)) + track.Duration = time.Duration(t.Uploads[0].Duration) * time.Second } return track diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index b00ebba..fbfe821 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -258,7 +258,7 @@ func (t Track) AsTrack() models.Track { TrackName: t.Name, ReleaseName: t.Album.Name, ArtistNames: make([]string, 0, len(t.Artists)), - Duration: time.Duration(t.DurationMs * int(time.Millisecond)), + Duration: time.Duration(t.DurationMs) * time.Millisecond, TrackNumber: t.TrackNumber, DiscNumber: t.DiscNumber, ISRC: t.ExternalIDs.ISRC, diff --git a/internal/backends/spotifyhistory/models.go b/internal/backends/spotifyhistory/models.go index a2eba23..3efaa38 100644 --- a/internal/backends/spotifyhistory/models.go +++ b/internal/backends/spotifyhistory/models.go @@ -89,7 +89,7 @@ func (i HistoryItem) AsListen() models.Listen { AdditionalInfo: models.AdditionalInfo{}, }, ListenedAt: i.Timestamp, - PlaybackDuration: time.Duration(i.MillisecondsPlayed * int(time.Millisecond)), + PlaybackDuration: time.Duration(i.MillisecondsPlayed) * time.Millisecond, UserName: i.UserName, } if trackURL, err := formatSpotifyUri(i.SpotifyTrackUri); err != nil { diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index aa1b1e3..f75366d 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -121,7 +121,7 @@ func SongAsLove(song subsonic.Child, username string) models.Love { AdditionalInfo: map[string]any{ "subsonic_id": song.ID, }, - Duration: time.Duration(song.Duration * int(time.Second)), + Duration: time.Duration(song.Duration) * time.Second, }, } From ed0c31c00f617dfc8eaf51a587c32b92739a39ca Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 25 May 2025 15:54:25 +0200 Subject: [PATCH 74/77] Update changelog --- CHANGES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGES.md b/CHANGES.md index 2257aaa..228b101 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -14,6 +14,7 @@ - spotify-history: the parameter to the export archive path has been renamed to `archive-path`. For backward compatibility the old `dir-path` parameter is still read. +- deezer-history: new backend to import listens and loves from Deezer data export. - deezer: fixed endless export loop if the user's listen history was empty. - dump: it is now possible to specify a file to write the text output to. - Fixed potential issues with MusicBrainz rate limiting. From 0c0246639953f301c43cee23e2253db556f594f5 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 26 May 2025 17:38:50 +0200 Subject: [PATCH 75/77] Moved JSONLFile to models package --- internal/listenbrainz/archive.go | 42 ++------------------- internal/models/jsonl.go | 65 ++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 39 deletions(-) create mode 100644 internal/models/jsonl.go diff --git a/internal/listenbrainz/archive.go b/internal/listenbrainz/archive.go index 68740aa..b263ca9 100644 --- a/internal/listenbrainz/archive.go +++ b/internal/listenbrainz/archive.go @@ -31,7 +31,7 @@ import ( "strconv" "time" - "github.com/simonfrey/jsonl" + "go.uploadedlobster.com/scotty/internal/models" "go.uploadedlobster.com/scotty/pkg/archive" ) @@ -130,7 +130,7 @@ func (a *ExportArchive) IterListens(minTimestamp time.Time) iter.Seq2[Listen, er continue } - f := JSONLFile[Listen]{file: file.f} + f := models.JSONLFile[Listen]{File: file.f} for l, err := range f.IterItems() { if err != nil { yield(Listen{}, err) @@ -161,7 +161,7 @@ func (a *ExportArchive) IterFeedback(minTimestamp time.Time) iter.Seq2[Feedback, return } - j := JSONLFile[Feedback]{file: files[0].File} + j := models.JSONLFile[Feedback]{File: files[0].File} for l, err := range j.IterItems() { if err != nil { yield(Feedback{}, err) @@ -194,42 +194,6 @@ type ListenExportFileInfo struct { f archive.OpenableFile } -type JSONLFile[T any] struct { - file archive.OpenableFile -} - -func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { - fio, err := f.file.Open() - if err != nil { - return nil, err - } - reader := jsonl.NewReader(fio) - return &reader, nil -} - -func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] { - return func(yield func(T, error) bool) { - reader, err := f.openReader() - if err != nil { - var listen T - yield(listen, err) - return - } - defer reader.Close() - - for { - var listen T - err := reader.ReadSingleLine(&listen) - if err != nil { - break - } - if !yield(listen, nil) { - break - } - } - } -} - func getMonthTimeRange(year string, month string) (*timeRange, error) { yearInt, err := strconv.Atoi(year) if err != nil { diff --git a/internal/models/jsonl.go b/internal/models/jsonl.go new file mode 100644 index 0000000..2bb1ea1 --- /dev/null +++ b/internal/models/jsonl.go @@ -0,0 +1,65 @@ +/* +Copyright © 2025 Philipp Wolfer + +This file is part of Scotty. + +Scotty is free software: you can redistribute it and/or modify it under the +terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later version. + +Scotty is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +Scotty. If not, see . +*/ + +package models + +import ( + "errors" + "iter" + + "github.com/simonfrey/jsonl" + "go.uploadedlobster.com/scotty/pkg/archive" +) + +type JSONLFile[T any] struct { + File archive.OpenableFile +} + +func (f *JSONLFile[T]) openReader() (*jsonl.Reader, error) { + if f.File == nil { + return nil, errors.New("file not set") + } + fio, err := f.File.Open() + if err != nil { + return nil, err + } + reader := jsonl.NewReader(fio) + return &reader, nil +} + +func (f *JSONLFile[T]) IterItems() iter.Seq2[T, error] { + return func(yield func(T, error) bool) { + reader, err := f.openReader() + if err != nil { + var listen T + yield(listen, err) + return + } + defer reader.Close() + + for { + var listen T + err := reader.ReadSingleLine(&listen) + if err != nil { + break + } + if !yield(listen, nil) { + break + } + } + } +} From c1a480a1a6e511c87ecf907b16c2cdd6e40ec88f Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Tue, 10 Jun 2025 08:09:44 +0200 Subject: [PATCH 76/77] Update dependencies --- go.mod | 32 +++++++++++++-------------- go.sum | 70 ++++++++++++++++++++++++++++------------------------------ 2 files changed, 50 insertions(+), 52 deletions(-) diff --git a/go.mod b/go.mod index 5991ecd..aec95c7 100644 --- a/go.mod +++ b/go.mod @@ -16,20 +16,20 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 - github.com/spf13/cast v1.8.0 + github.com/spf13/cast v1.9.2 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d - github.com/vbauerster/mpb/v8 v8.10.1 + github.com/vbauerster/mpb/v8 v8.10.2 github.com/xuri/excelize/v2 v2.9.1 go.uploadedlobster.com/mbtypes v0.4.0 - go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 - golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 + go.uploadedlobster.com/musicbrainzws2 v0.16.0 + golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 golang.org/x/oauth2 v0.30.0 - golang.org/x/text v0.25.0 + golang.org/x/text v0.26.0 gorm.io/datatypes v1.2.5 - gorm.io/gorm v1.26.1 + gorm.io/gorm v1.30.0 ) require ( @@ -61,23 +61,23 @@ require ( github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tiendc/go-deepcopy v1.6.0 // indirect + github.com/tiendc/go-deepcopy v1.6.1 // indirect github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.38.0 // indirect - golang.org/x/image v0.27.0 // indirect - golang.org/x/mod v0.24.0 // indirect - golang.org/x/net v0.40.0 // indirect - golang.org/x/sync v0.14.0 // indirect + golang.org/x/crypto v0.39.0 // indirect + golang.org/x/image v0.28.0 // indirect + golang.org/x/mod v0.25.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sync v0.15.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/tools v0.33.0 // indirect + golang.org/x/tools v0.34.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/driver/mysql v1.5.7 // indirect - modernc.org/libc v1.65.7 // indirect + gorm.io/driver/mysql v1.6.0 // indirect + modernc.org/libc v1.65.10 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.37.1 // indirect + modernc.org/sqlite v1.38.0 // indirect ) tool golang.org/x/text/cmd/gotext diff --git a/go.sum b/go.sum index fefdfa4..2483a83 100644 --- a/go.sum +++ b/go.sum @@ -40,7 +40,6 @@ github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GM github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= github.com/go-resty/resty/v2 v2.16.5 h1:hBKqmWrr7uRc3euHVqmh1HTHcKn99Smr7o5spptdhTM= github.com/go-resty/resty/v2 v2.16.5/go.mod h1:hkJtXbA2iKHzJheXYvQ8snQES5ZLGKMwQ07xAwp/fiA= -github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU= github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= @@ -118,8 +117,8 @@ github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9yS github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= -github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE= +github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= @@ -132,10 +131,10 @@ github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8 github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d h1:70+Nn7yh+cfeKqqXVTdpneFqXuvrBLyP7U6GVUsjTU4= github.com/supersonic-app/go-subsonic v0.0.0-20241224013245-9b2841f3711d/go.mod h1:D+OWPXeD9owcdcoXATv5YPBGWxxVvn5k98rt5B4wMc4= -github.com/tiendc/go-deepcopy v1.6.0 h1:0UtfV/imoCwlLxVsyfUd4hNHnB3drXsfle+wzSCA5Wo= -github.com/tiendc/go-deepcopy v1.6.0/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= -github.com/vbauerster/mpb/v8 v8.10.1 h1:t/ZFv/NYgoBUy2LrmkD5Vc25r+JhoS4+gRkjVbolO2Y= -github.com/vbauerster/mpb/v8 v8.10.1/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= +github.com/tiendc/go-deepcopy v1.6.1 h1:uVRTItFeNHkMcLueHS7OCsxgxT9P8MzGB/taUa2Y4Tk= +github.com/tiendc/go-deepcopy v1.6.1/go.mod h1:toXoeQoUqXOOS/X4sKuiAoSk6elIdqc0pN7MTgOOo2I= +github.com/vbauerster/mpb/v8 v8.10.2 h1:2uBykSHAYHekE11YvJhKxYmLATKHAGorZwFlyNw4hHM= +github.com/vbauerster/mpb/v8 v8.10.2/go.mod h1:+Ja4P92E3/CorSZgfDtK46D7AVbDqmBQRTmyTqPElo0= github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8= github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= github.com/xuri/excelize/v2 v2.9.1 h1:VdSGk+rraGmgLHGFaGG9/9IWu1nj4ufjJ7uwMDtj8Qw= @@ -147,34 +146,34 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uploadedlobster.com/mbtypes v0.4.0 h1:D5asCgHsRWufj4Yn5u0IuH2J9z1UuYImYkYIp1Z1Q7s= go.uploadedlobster.com/mbtypes v0.4.0/go.mod h1:Bu1K1Hl77QTAE2Z7QKiW/JAp9KqYWQebkRRfG02dlZM= -go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064 h1:bir8kas9u0A+T54sfzj3il7SUAV5KQtb5QzDtwvslxI= -go.uploadedlobster.com/musicbrainzws2 v0.15.1-0.20250524094913-01f007ad1064/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= +go.uploadedlobster.com/musicbrainzws2 v0.16.0 h1:Boux1cZg5S559G/pbQC35BoF+1H7I56oxhBwg8Nzhs0= +go.uploadedlobster.com/musicbrainzws2 v0.16.0/go.mod h1:T6sYE7ZHRH3mJWT3g9jdSUPKJLZubnBjKyjMPNdkgao= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= -golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4= +golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= golang.org/x/image v0.13.0/go.mod h1:6mmbMOeV28HuMTgA6OSRkdXKYw/t5W9Uwn2Yv1r3Yxk= -golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w= -golang.org/x/image v0.27.0/go.mod h1:xbdrClrAUway1MUTEZDq9mz/UpRwYAkFFNUslZtcB+g= +golang.org/x/image v0.28.0 h1:gdem5JW1OLS4FbkWgLO+7ZeFzYtL3xClb97GaUzYMFE= +golang.org/x/image v0.28.0/go.mod h1:GUJYXtnGKEUgggyzh+Vxt+AviiCcyiwpsl8iQ8MvwGY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= -golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w= +golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= -golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -194,16 +193,16 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= @@ -212,27 +211,26 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/datatypes v1.2.5 h1:9UogU3jkydFVW1bIVVeoYsTpLRgwDVW3rHfJG6/Ek9I= gorm.io/datatypes v1.2.5/go.mod h1:I5FUdlKpLb5PMqeMQhm30CQ6jXP8Rj89xkTeCSAaAD4= -gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= -gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg= +gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo= gorm.io/driver/postgres v1.5.0 h1:u2FXTy14l45qc3UeCJ7QaAXZmZfDDv0YrthvmRq1l0U= gorm.io/driver/postgres v1.5.0/go.mod h1:FUZXzO+5Uqg5zzwzv4KK49R8lvGIyscBOqYrtI1Ce9A= gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= gorm.io/driver/sqlserver v1.5.4 h1:xA+Y1KDNspv79q43bPyjDMUgHoYHLhXYmdFcYPobg8g= gorm.io/driver/sqlserver v1.5.4/go.mod h1:+frZ/qYmuna11zHPlh5oc2O6ZA/lS88Keb0XSH1Zh/g= -gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= -gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw= -gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= +gorm.io/gorm v1.30.0 h1:qbT5aPv1UH8gI99OsRlvDToLxW5zR7FzS9acZDOZcgs= +gorm.io/gorm v1.30.0/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE= modernc.org/cc/v4 v4.26.1 h1:+X5NtzVBn0KgsBCBe+xkDC7twLb/jNVj9FPgiwSQO3s= modernc.org/cc/v4 v4.26.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v4 v4.28.0 h1:rjznn6WWehKq7dG4JtLRKxb52Ecv8OUGah8+Z/SfpNU= modernc.org/ccgo/v4 v4.28.0/go.mod h1:JygV3+9AV6SmPhDasu4JgquwU81XAKLd3OKTUDNOiKE= -modernc.org/fileutil v1.3.1 h1:8vq5fe7jdtEvoCf3Zf9Nm0Q05sH6kGx0Op2CPx1wTC8= -modernc.org/fileutil v1.3.1/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/fileutil v1.3.3 h1:3qaU+7f7xxTUmvU1pJTZiDLAIoJVdUSSauJNHg9yXoA= +modernc.org/fileutil v1.3.3/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/libc v1.65.7 h1:Ia9Z4yzZtWNtUIuiPuQ7Qf7kxYrxP1/jeHZzG8bFu00= -modernc.org/libc v1.65.7/go.mod h1:011EQibzzio/VX3ygj1qGFt5kMjP0lHb0qCW5/D/pQU= +modernc.org/libc v1.65.10 h1:ZwEk8+jhW7qBjHIT+wd0d9VjitRyQef9BnzlzGwMODc= +modernc.org/libc v1.65.10/go.mod h1:StFvYpx7i/mXtBAfVOjaU0PWZOvIRoZSgXhrwXzr8Po= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= @@ -241,8 +239,8 @@ modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= -modernc.org/sqlite v1.37.1 h1:EgHJK/FPoqC+q2YBXg7fUmES37pCHFc97sI7zSayBEs= -modernc.org/sqlite v1.37.1/go.mod h1:XwdRtsE1MpiBcL54+MbKcaDvcuej+IYSMfLN6gSKV8g= +modernc.org/sqlite v1.38.0 h1:+4OrfPQ8pxHKuWG4md1JpR/EYAh3Md7TdejuuzE7EUI= +modernc.org/sqlite v1.38.0/go.mod h1:1Bj+yES4SVvBZ4cBOpVZ6QgesMCKpJZDq0nxYzOpmNE= modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= From 499786cab984be42033c56c52ada7c5b5fa64365 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Tue, 10 Jun 2025 08:30:26 +0200 Subject: [PATCH 77/77] Introduced Backend.Close method This allows Backend implementations to free used resources. Currently used for musicbrainzws2.Client --- internal/backends/deezer/deezer.go | 2 ++ internal/backends/deezerhistory/deezerhistory.go | 2 ++ internal/backends/dump/dump.go | 2 ++ internal/backends/funkwhale/funkwhale.go | 2 ++ internal/backends/jspf/jspf.go | 2 ++ internal/backends/lastfm/lastfm.go | 2 ++ internal/backends/lbarchive/lbarchive.go | 14 ++++++++++---- internal/backends/listenbrainz/listenbrainz.go | 14 ++++++++++---- internal/backends/maloja/maloja.go | 2 ++ internal/backends/scrobblerlog/scrobblerlog.go | 2 ++ internal/backends/spotify/spotify.go | 2 ++ internal/backends/spotifyhistory/spotifyhistory.go | 2 ++ internal/backends/subsonic/subsonic.go | 2 ++ internal/models/interfaces.go | 3 +++ 14 files changed, 45 insertions(+), 8 deletions(-) diff --git a/internal/backends/deezer/deezer.go b/internal/backends/deezer/deezer.go index 9d622df..f70a2c9 100644 --- a/internal/backends/deezer/deezer.go +++ b/internal/backends/deezer/deezer.go @@ -38,6 +38,8 @@ type DeezerApiBackend struct { func (b *DeezerApiBackend) Name() string { return "deezer" } +func (b *DeezerApiBackend) Close() {} + func (b *DeezerApiBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "client-id", diff --git a/internal/backends/deezerhistory/deezerhistory.go b/internal/backends/deezerhistory/deezerhistory.go index 5574a5d..9c74368 100644 --- a/internal/backends/deezerhistory/deezerhistory.go +++ b/internal/backends/deezerhistory/deezerhistory.go @@ -43,6 +43,8 @@ type DeezerHistoryBackend struct { func (b *DeezerHistoryBackend) Name() string { return "deezer-history" } +func (b *DeezerHistoryBackend) Close() {} + func (b *DeezerHistoryBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "file-path", diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index 8d7c641..b342ba5 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -36,6 +36,8 @@ type DumpBackend struct { func (b *DumpBackend) Name() string { return "dump" } +func (b *DumpBackend) Close() {} + func (b *DumpBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "file-path", diff --git a/internal/backends/funkwhale/funkwhale.go b/internal/backends/funkwhale/funkwhale.go index 8874c70..8039ec2 100644 --- a/internal/backends/funkwhale/funkwhale.go +++ b/internal/backends/funkwhale/funkwhale.go @@ -36,6 +36,8 @@ type FunkwhaleApiBackend struct { func (b *FunkwhaleApiBackend) Name() string { return "funkwhale" } +func (b *FunkwhaleApiBackend) Close() {} + func (b *FunkwhaleApiBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "server-url", diff --git a/internal/backends/jspf/jspf.go b/internal/backends/jspf/jspf.go index 887fd72..cdbb23e 100644 --- a/internal/backends/jspf/jspf.go +++ b/internal/backends/jspf/jspf.go @@ -46,6 +46,8 @@ type JSPFBackend struct { func (b *JSPFBackend) Name() string { return "jspf" } +func (b *JSPFBackend) Close() {} + func (b *JSPFBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "file-path", diff --git a/internal/backends/lastfm/lastfm.go b/internal/backends/lastfm/lastfm.go index 186a631..ebe226a 100644 --- a/internal/backends/lastfm/lastfm.go +++ b/internal/backends/lastfm/lastfm.go @@ -46,6 +46,8 @@ type LastfmApiBackend struct { func (b *LastfmApiBackend) Name() string { return "lastfm" } +func (b *LastfmApiBackend) Close() {} + func (b *LastfmApiBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "username", diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index a91c0a5..ce23795 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -42,11 +42,17 @@ const ( type ListenBrainzArchiveBackend struct { filePath string lbClient listenbrainz.Client - mbClient musicbrainzws2.Client + mbClient *musicbrainzws2.Client } func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" } +func (b *ListenBrainzArchiveBackend) Close() { + if b.mbClient != nil { + b.mbClient.Close() + } +} + func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "archive-path", @@ -58,7 +64,7 @@ func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { b.filePath = config.GetString("archive-path") b.lbClient = listenbrainz.NewClient("", version.UserAgent()) - b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ + b.mbClient = musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: version.AppName, Version: version.AppVersion, URL: version.AppURL, @@ -191,7 +197,7 @@ func (b *ListenBrainzArchiveBackend) ExportLoves( if len(batch) >= lovesBatchSize { // The dump does not contain track metadata. Extend it with additional // lookups - loves, err := lbapi.ExtendTrackMetadata(ctx, &b.lbClient, &b.mbClient, &batch) + loves, err := lbapi.ExtendTrackMetadata(ctx, &b.lbClient, b.mbClient, &batch) if err != nil { p.Export.Abort() progress <- p @@ -205,7 +211,7 @@ func (b *ListenBrainzArchiveBackend) ExportLoves( } } - loves, err := lbapi.ExtendTrackMetadata(ctx, &b.lbClient, &b.mbClient, &batch) + loves, err := lbapi.ExtendTrackMetadata(ctx, &b.lbClient, b.mbClient, &batch) if err != nil { p.Export.Abort() progress <- p diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 98d1525..b809b47 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -36,12 +36,18 @@ const lovesBatchSize = listenbrainz.MaxItemsPerGet type ListenBrainzApiBackend struct { client listenbrainz.Client - mbClient musicbrainzws2.Client + mbClient *musicbrainzws2.Client username string checkDuplicates bool existingMBIDs map[mbtypes.MBID]bool } +func (b *ListenBrainzApiBackend) Close() { + if b.mbClient != nil { + b.mbClient.Close() + } +} + func (b *ListenBrainzApiBackend) Name() string { return "listenbrainz" } func (b *ListenBrainzApiBackend) Options() []models.BackendOption { @@ -62,7 +68,7 @@ func (b *ListenBrainzApiBackend) Options() []models.BackendOption { func (b *ListenBrainzApiBackend) InitConfig(config *config.ServiceConfig) error { b.client = listenbrainz.NewClient(config.GetString("token"), version.UserAgent()) - b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ + b.mbClient = musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ Name: version.AppName, Version: version.AppVersion, URL: version.AppURL, @@ -261,7 +267,7 @@ out: // Missing track metadata indicates that the recording MBID is no // longer available and might have been merged. Try fetching details // from MusicBrainz. - lovesBatch, err := ExtendTrackMetadata(ctx, &b.client, &b.mbClient, &batch) + lovesBatch, err := ExtendTrackMetadata(ctx, &b.client, b.mbClient, &batch) if err != nil { results <- models.LovesResult{Error: err} return @@ -276,7 +282,7 @@ out: offset += listenbrainz.MaxItemsPerGet } - lovesBatch, err := ExtendTrackMetadata(ctx, &b.client, &b.mbClient, &batch) + lovesBatch, err := ExtendTrackMetadata(ctx, &b.client, b.mbClient, &batch) if err != nil { results <- models.LovesResult{Error: err} return diff --git a/internal/backends/maloja/maloja.go b/internal/backends/maloja/maloja.go index d85309f..e5537df 100644 --- a/internal/backends/maloja/maloja.go +++ b/internal/backends/maloja/maloja.go @@ -35,6 +35,8 @@ type MalojaApiBackend struct { func (b *MalojaApiBackend) Name() string { return "maloja" } +func (b *MalojaApiBackend) Close() {} + func (b *MalojaApiBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "server-url", diff --git a/internal/backends/scrobblerlog/scrobblerlog.go b/internal/backends/scrobblerlog/scrobblerlog.go index 13aecba..3a91e92 100644 --- a/internal/backends/scrobblerlog/scrobblerlog.go +++ b/internal/backends/scrobblerlog/scrobblerlog.go @@ -41,6 +41,8 @@ type ScrobblerLogBackend struct { func (b *ScrobblerLogBackend) Name() string { return "scrobbler-log" } +func (b *ScrobblerLogBackend) Close() {} + func (b *ScrobblerLogBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "file-path", diff --git a/internal/backends/spotify/spotify.go b/internal/backends/spotify/spotify.go index fbfe821..85b40dd 100644 --- a/internal/backends/spotify/spotify.go +++ b/internal/backends/spotify/spotify.go @@ -41,6 +41,8 @@ type SpotifyApiBackend struct { func (b *SpotifyApiBackend) Name() string { return "spotify" } +func (b *SpotifyApiBackend) Close() {} + func (b *SpotifyApiBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "client-id", diff --git a/internal/backends/spotifyhistory/spotifyhistory.go b/internal/backends/spotifyhistory/spotifyhistory.go index 5f67604..985469f 100644 --- a/internal/backends/spotifyhistory/spotifyhistory.go +++ b/internal/backends/spotifyhistory/spotifyhistory.go @@ -36,6 +36,8 @@ type SpotifyHistoryBackend struct { func (b *SpotifyHistoryBackend) Name() string { return "spotify-history" } +func (b *SpotifyHistoryBackend) Close() {} + func (b *SpotifyHistoryBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "archive-path", diff --git a/internal/backends/subsonic/subsonic.go b/internal/backends/subsonic/subsonic.go index f75366d..1ffa510 100644 --- a/internal/backends/subsonic/subsonic.go +++ b/internal/backends/subsonic/subsonic.go @@ -37,6 +37,8 @@ type SubsonicApiBackend struct { func (b *SubsonicApiBackend) Name() string { return "subsonic" } +func (b *SubsonicApiBackend) Close() {} + func (b *SubsonicApiBackend) Options() []models.BackendOption { return []models.BackendOption{{ Name: "server-url", diff --git a/internal/models/interfaces.go b/internal/models/interfaces.go index 79a4c6c..0f287bf 100644 --- a/internal/models/interfaces.go +++ b/internal/models/interfaces.go @@ -35,6 +35,9 @@ type Backend interface { // Return configuration options Options() []BackendOption + + // Free all resources of the backend + Close() } type ImportBackend interface {