From ab04eb1123fded32ab9720f2cf1aed3a33634460 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Wed, 15 Nov 2023 19:24:12 +0100 Subject: [PATCH] Updated all import/export interfaces --- backends/dump/dump.go | 4 - backends/funkwhale/funkwhale.go | 29 ++++--- backends/jspf/jspf.go | 87 +++++++++++-------- backends/listenbrainz/listenbrainz.go | 117 +++++++++++++++----------- backends/maloja/client.go | 2 + backends/maloja/maloja.go | 75 +++++++++-------- backends/scrobblerlog/parser.go | 15 ++-- backends/scrobblerlog/parser_test.go | 4 +- backends/scrobblerlog/scrobblerlog.go | 49 +++++++---- backends/subsonic/subsonic.go | 5 +- models/models.go | 11 +++ models/models_test.go | 16 +++- 12 files changed, 247 insertions(+), 167 deletions(-) diff --git a/backends/dump/dump.go b/backends/dump/dump.go index db0bc66..feb8c8d 100644 --- a/backends/dump/dump.go +++ b/backends/dump/dump.go @@ -37,8 +37,6 @@ func (b DumpBackend) FromConfig(config *viper.Viper) models.Backend { func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) { importResult := models.ImportResult{ - TotalCount: 0, - ImportCount: 0, LastTimestamp: oldestTimestamp, } for result := range results { @@ -59,8 +57,6 @@ func (b DumpBackend) ImportListens(results chan models.ListensResult, oldestTime func (b DumpBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) { importResult := models.ImportResult{ - TotalCount: 0, - ImportCount: 0, LastTimestamp: oldestTimestamp, } for result := range results { diff --git a/backends/funkwhale/funkwhale.go b/backends/funkwhale/funkwhale.go index ccac342..613b076 100644 --- a/backends/funkwhale/funkwhale.go +++ b/backends/funkwhale/funkwhale.go @@ -22,7 +22,7 @@ THE SOFTWARE. package funkwhale import ( - "slices" + "sort" "time" "github.com/spf13/viper" @@ -45,17 +45,19 @@ func (b FunkwhaleApiBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) { +func (b FunkwhaleApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { page := 1 perPage := MaxItemsPerGet - listens := make([]models.Listen, 0, 2*MaxItemsPerGet) + // We need to gather the full list of listens in order to sort them + listens := make(models.ListensList, 0, 2*MaxItemsPerGet) out: for { result, err := b.client.GetHistoryListenings(b.username, page, perPage) if err != nil { - return nil, err + results <- models.ListensResult{Error: err} + close(results) } count := len(result.Results) @@ -80,21 +82,26 @@ out: page += 1 } - slices.Reverse(listens) - return listens, nil + sort.Sort(listens) + results <- models.ListensResult{Listens: listens} + close(results) } -func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time) ([]models.Love, error) { +func (b FunkwhaleApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { page := 1 perPage := MaxItemsPerGet - loves := make([]models.Love, 0, 2*MaxItemsPerGet) + defer close(results) + + // We need to gather the full list of listens in order to sort them + loves := make(models.LovesList, 0, 2*MaxItemsPerGet) out: for { result, err := b.client.GetFavoriteTracks(page, perPage) if err != nil { - return nil, err + results <- models.LovesResult{Error: err} + return } count := len(result.Results) @@ -119,8 +126,8 @@ out: page += 1 } - slices.Reverse(loves) - return loves, nil + sort.Sort(loves) + results <- models.LovesResult{Loves: loves} } func (l Listening) ToListen() models.Listen { diff --git a/backends/jspf/jspf.go b/backends/jspf/jspf.go index a9af9f4..d91b8b1 100644 --- a/backends/jspf/jspf.go +++ b/backends/jspf/jspf.go @@ -46,52 +46,65 @@ func (b JspfBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b JspfBackend) ImportLoves(loves []models.Love, oldestTimestamp time.Time) (models.ImportResult, error) { - result := models.ImportResult{ - TotalCount: len(loves), - ImportCount: 0, +func (b JspfBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) { + importResult := models.ImportResult{ LastTimestamp: oldestTimestamp, } - tracks := make([]Track, 0, result.TotalCount) - for _, love := range loves { - extension := MusicBrainzTrackExtension{ - AddedAt: love.Created, - AddedBy: love.UserName, - AdditionalMetadata: love.AdditionalInfo, - ArtistIdentifiers: make([]string, len(love.ArtistMbids)), + tracks := make([]Track, 0, importResult.TotalCount) + for result := range results { + if result.Error != nil { + return importResult, result.Error } - for i, mbid := range love.ArtistMbids { - extension.ArtistIdentifiers[i] = "https://musicbrainz.org/artist/" + string(mbid) + importResult.TotalCount += len(result.Loves) + for _, love := range result.Loves { + track := loveToTrack(love) + tracks = append(tracks, track) + oldestTimestamp = love.Created + importResult.ImportCount += 1 } - - if love.ReleaseMbid != "" { - extension.ReleaseIdentifier = "https://musicbrainz.org/release/" + string(love.ReleaseMbid) - } - - track := Track{ - Title: love.TrackName, - Album: love.ReleaseName, - Creator: love.ArtistName(), - TrackNum: love.TrackNumber, - Extension: map[string]any{ - "https://musicbrainz.org/doc/jspf#track": extension, - }, - } - - if love.RecordingMbid != "" { - track.Identifier = append(track.Identifier, "https://musicbrainz.org/recording/"+string(love.RecordingMbid)) - } - - tracks = append(tracks, track) - - result.UpdateTimestamp(love.Created) - result.ImportCount += 1 } err := b.writeJspf(tracks) - return result, err + if err != nil { + importResult.UpdateTimestamp(oldestTimestamp) + importResult.ImportCount = len(tracks) + } + return importResult, err +} + +func loveToTrack(love models.Love) Track { + extension := MusicBrainzTrackExtension{ + AddedAt: love.Created, + AddedBy: love.UserName, + AdditionalMetadata: love.AdditionalInfo, + ArtistIdentifiers: make([]string, len(love.ArtistMbids)), + } + + for i, mbid := range love.ArtistMbids { + extension.ArtistIdentifiers[i] = "https://musicbrainz.org/artist/" + string(mbid) + } + + if love.ReleaseMbid != "" { + extension.ReleaseIdentifier = "https://musicbrainz.org/release/" + string(love.ReleaseMbid) + } + + track := Track{ + Title: love.TrackName, + Album: love.ReleaseName, + Creator: love.ArtistName(), + TrackNum: love.TrackNumber, + Extension: map[string]any{ + "https://musicbrainz.org/doc/jspf#track": extension, + }, + } + + if love.RecordingMbid != "" { + track.Identifier = append(track.Identifier, "https://musicbrainz.org/recording/"+string(love.RecordingMbid)) + } + + return track } func (b JspfBackend) writeJspf(tracks []Track) error { diff --git a/backends/listenbrainz/listenbrainz.go b/backends/listenbrainz/listenbrainz.go index 5d8d146..d18f4af 100644 --- a/backends/listenbrainz/listenbrainz.go +++ b/backends/listenbrainz/listenbrainz.go @@ -23,7 +23,7 @@ package listenbrainz import ( "fmt" - "slices" + "sort" "time" "github.com/spf13/viper" @@ -42,16 +42,21 @@ func (b ListenBrainzApiBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) { +func (b ListenBrainzApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { maxTime := time.Now() minTime := time.Unix(0, 0) - listens := make([]models.Listen, 0, 2*MaxItemsPerGet) + + defer close(results) + + // FIXME: Optimize by fetching the listens in reverse listen time order + listens := make(models.ListensList, 0, 2*MaxItemsPerGet) out: for { result, err := b.client.GetListens(b.username, maxTime, minTime) if err != nil { - return nil, err + results <- models.ListensResult{Error: err} + return } count := len(result.Payload.Listens) @@ -73,19 +78,21 @@ out: } } - slices.Reverse(listens) - return listens, nil + sort.Sort(listens) + results <- models.ListensResult{Listens: listens} } -func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time) ([]models.Love, error) { +func (b ListenBrainzApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { offset := 0 - loves := make([]models.Love, 0, 2*MaxItemsPerGet) + defer close(results) + loves := make(models.LovesList, 0, 2*MaxItemsPerGet) out: for { result, err := b.client.GetFeedback(b.username, 1, offset) if err != nil { - return nil, err + results <- models.LovesResult{Error: err} + return } count := len(result.Feedback) @@ -105,69 +112,79 @@ out: offset += MaxItemsPerGet } - slices.Reverse(loves) - return loves, nil + sort.Sort(loves) + results <- models.LovesResult{Loves: loves} } -func (b ListenBrainzApiBackend) ImportLoves(loves []models.Love, oldestTimestamp time.Time) (models.ImportResult, error) { - result := models.ImportResult{ - TotalCount: len(loves), - ImportCount: 0, +func (b ListenBrainzApiBackend) ImportLoves(results chan models.LovesResult, oldestTimestamp time.Time) (models.ImportResult, error) { + importResult := models.ImportResult{ LastTimestamp: oldestTimestamp, ImportErrors: make([]string, 0), } - existingLoves, err := b.ExportLoves(time.Unix(0, 0)) - if err != nil { - return result, err + existingLovesChan := make(chan models.LovesResult) + go b.ExportLoves(time.Unix(0, 0), existingLovesChan) + existingLoves := <-existingLovesChan + if existingLoves.Error != nil { + results <- models.LovesResult{Error: existingLoves.Error} + close(results) } - existingMbids := make(map[string]bool, len(existingLoves)) - for _, love := range existingLoves { + existingMbids := make(map[string]bool, len(existingLoves.Loves)) + for _, love := range existingLoves.Loves { existingMbids[string(love.RecordingMbid)] = true } - for _, love := range loves { - if love.Created.Unix() <= oldestTimestamp.Unix() { - continue + for result := range results { + if result.Error != nil { + return importResult, result.Error } - recordingMbid := string(love.RecordingMbid) + importResult.TotalCount += len(result.Loves) - if recordingMbid == "" { - lookup, err := b.client.Lookup(love.TrackName, love.ArtistName()) - if err == nil { - recordingMbid = lookup.RecordingMbid + for _, love := range result.Loves { + if love.Created.Unix() <= oldestTimestamp.Unix() { + continue } - } - if recordingMbid != "" { - ok := false - errMsg := "" - if existingMbids[recordingMbid] { - ok = true - } else { - resp, err := b.client.SendFeedback(Feedback{ - RecordingMbid: recordingMbid, - Score: 1, - }) - ok = err == nil && resp.Status == "ok" - if err != nil { - errMsg = err.Error() + recordingMbid := string(love.RecordingMbid) + + if recordingMbid == "" { + lookup, err := b.client.Lookup(love.TrackName, love.ArtistName()) + if err == nil { + recordingMbid = lookup.RecordingMbid } } - if ok { - result.UpdateTimestamp(love.Created) - result.ImportCount += 1 - } else { - msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v", - love.TrackName, love.ArtistName(), errMsg) - result.ImportErrors = append(result.ImportErrors, msg) + if recordingMbid != "" { + ok := false + errMsg := "" + if existingMbids[recordingMbid] { + ok = true + } else { + resp, err := b.client.SendFeedback(Feedback{ + RecordingMbid: recordingMbid, + Score: 1, + }) + ok = err == nil && resp.Status == "ok" + if err != nil { + errMsg = err.Error() + } + } + + if ok { + importResult.UpdateTimestamp(love.Created) + importResult.ImportCount += 1 + } else { + msg := fmt.Sprintf("Failed import of \"%s\" by %s: %v", + love.TrackName, love.ArtistName(), errMsg) + importResult.ImportErrors = append(importResult.ImportErrors, msg) + } } } } - return result, nil + + return importResult, nil } func (lbListen Listen) ToListen() models.Listen { diff --git a/backends/maloja/client.go b/backends/maloja/client.go index 893f5f8..5e3ea31 100644 --- a/backends/maloja/client.go +++ b/backends/maloja/client.go @@ -28,6 +28,8 @@ import ( "github.com/go-resty/resty/v2" ) +const MaxItemsPerGet = 1000 + type Client struct { HttpClient *resty.Client token string diff --git a/backends/maloja/maloja.go b/backends/maloja/maloja.go index e82adda..42864a9 100644 --- a/backends/maloja/maloja.go +++ b/backends/maloja/maloja.go @@ -23,7 +23,7 @@ package maloja import ( "errors" - "slices" + "sort" "strings" "time" @@ -45,17 +45,21 @@ func (b MalojaApiBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) { +func (b MalojaApiBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { page := 0 - perPage := 1000 + perPage := MaxItemsPerGet - listens := make([]models.Listen, 0) + defer close(results) + + // We need to gather the full list of listens in order to sort them + listens := make(models.ListensList, 0, 2*perPage) out: for { result, err := b.client.GetScrobbles(page, perPage) if err != nil { - return nil, err + results <- models.ListensResult{Error: err} + return } count := len(result.List) @@ -74,44 +78,49 @@ out: page += 1 } - slices.Reverse(listens) - return listens, nil + sort.Sort(listens) + results <- models.ListensResult{Listens: listens} } -func (b MalojaApiBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (models.ImportResult, error) { - result := models.ImportResult{ - TotalCount: len(listens), - ImportCount: 0, +func (b MalojaApiBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) { + importResult := models.ImportResult{ LastTimestamp: oldestTimestamp, } - for _, listen := range listens { - if listen.ListenedAt.Unix() <= oldestTimestamp.Unix() { - continue + + for result := range results { + if result.Error != nil { + return importResult, result.Error } - scrobble := NewScrobble{ - Title: listen.TrackName, - Artists: listen.ArtistNames, - Album: listen.ReleaseName, - Duration: int64(listen.PlaybackDuration.Seconds()), - Length: int64(listen.Duration.Seconds()), - Time: listen.ListenedAt.Unix(), - Nofix: b.nofix, - } + importResult.TotalCount += len(result.Listens) + for _, listen := range result.Listens { + if listen.ListenedAt.Unix() <= oldestTimestamp.Unix() { + break + } - resp, err := b.client.NewScrobble(scrobble) - if err != nil { - return result, err - } else if resp.Status != "success" { - return result, errors.New(resp.Error.Description) - } + scrobble := NewScrobble{ + Title: listen.TrackName, + Artists: listen.ArtistNames, + Album: listen.ReleaseName, + Duration: int64(listen.PlaybackDuration.Seconds()), + Length: int64(listen.Duration.Seconds()), + Time: listen.ListenedAt.Unix(), + Nofix: b.nofix, + } - if listen.ListenedAt.Unix() > result.LastTimestamp.Unix() { - result.LastTimestamp = listen.ListenedAt + resp, err := b.client.NewScrobble(scrobble) + if err != nil { + return importResult, err + } else if resp.Status != "success" { + return importResult, errors.New(resp.Error.Description) + } + + importResult.UpdateTimestamp(listen.ListenedAt) + importResult.ImportCount += 1 } - result.ImportCount += 1 } - return result, nil + + return importResult, nil } func (s Scrobble) ToListen() models.Listen { diff --git a/backends/scrobblerlog/parser.go b/backends/scrobblerlog/parser.go index b93eac1..ccda522 100644 --- a/backends/scrobblerlog/parser.go +++ b/backends/scrobblerlog/parser.go @@ -37,12 +37,12 @@ import ( type ScrobblerLog struct { Timezone string Client string - Listens []models.Listen + Listens models.ListensList } func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) { result := ScrobblerLog{ - Listens: make([]models.Listen, 0), + Listens: make(models.ListensList, 0), } reader := bufio.NewReader(data) @@ -92,16 +92,11 @@ func Parse(data io.Reader, includeSkipped bool) (ScrobblerLog, error) { return result, nil } -func Write(data io.Writer, log *ScrobblerLog) (lastTimestamp time.Time, err error) { - err = writeHeader(data, log) - if err != nil { - return - } - +func Write(data io.Writer, listens models.ListensList) (lastTimestamp time.Time, err error) { tsvWriter := csv.NewWriter(data) tsvWriter.Comma = '\t' - for _, listen := range log.Listens { + for _, listen := range listens { if listen.ListenedAt.Unix() > lastTimestamp.Unix() { lastTimestamp = listen.ListenedAt } @@ -162,7 +157,7 @@ func readHeader(reader *bufio.Reader, log *ScrobblerLog) error { return nil } -func writeHeader(writer io.Writer, log *ScrobblerLog) error { +func WriteHeader(writer io.Writer, log *ScrobblerLog) error { headers := []string{ "#AUDIOSCROBBLER/1.1\n", "#TZ/" + log.Timezone + "\n", diff --git a/backends/scrobblerlog/parser_test.go b/backends/scrobblerlog/parser_test.go index 9cfe480..af6af09 100644 --- a/backends/scrobblerlog/parser_test.go +++ b/backends/scrobblerlog/parser_test.go @@ -98,7 +98,9 @@ func TestWrite(t *testing.T) { }, }, } - lastTimestamp, err := scrobblerlog.Write(buffer, &log) + err := scrobblerlog.WriteHeader(buffer, &log) + require.NoError(t, err) + lastTimestamp, err := scrobblerlog.Write(buffer, log.Listens) require.NoError(t, err) result := string(buffer.Bytes()) lines := strings.Split(result, "\n") diff --git a/backends/scrobblerlog/scrobblerlog.go b/backends/scrobblerlog/scrobblerlog.go index 412816a..e871602 100644 --- a/backends/scrobblerlog/scrobblerlog.go +++ b/backends/scrobblerlog/scrobblerlog.go @@ -23,6 +23,7 @@ package scrobblerlog import ( "os" + "sort" "time" "github.com/spf13/viper" @@ -40,31 +41,36 @@ func (b ScrobblerLogBackend) FromConfig(config *viper.Viper) models.Backend { return b } -func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time) ([]models.Listen, error) { +func (b ScrobblerLogBackend) ExportListens(oldestTimestamp time.Time, results chan models.ListensResult) { + defer close(results) file, err := os.Open(b.filePath) if err != nil { - return nil, err + results <- models.ListensResult{Error: err} + return } defer file.Close() - result, err := Parse(file, b.includeSkipped) + log, err := Parse(file, b.includeSkipped) if err != nil { - return nil, err + results <- models.ListensResult{Error: err} + close(results) + return } - return result.Listens, nil + listens := log.Listens.NewerThan(oldestTimestamp) + sort.Sort(listens) + results <- models.ListensResult{Listens: listens} } -func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimestamp time.Time) (models.ImportResult, error) { - result := models.ImportResult{ - TotalCount: len(listens), +func (b ScrobblerLogBackend) ImportListens(results chan models.ListensResult, oldestTimestamp time.Time) (models.ImportResult, error) { + importResult := models.ImportResult{ LastTimestamp: oldestTimestamp, } file, err := os.Create(b.filePath) if err != nil { - return result, err + return importResult, err } defer file.Close() @@ -72,16 +78,27 @@ func (b ScrobblerLogBackend) ImportListens(listens []models.Listen, oldestTimest log := ScrobblerLog{ Timezone: "UNKNOWN", Client: "Rockbox unknown $Revision$", - Listens: listens, } - lastTimestamp, err := Write(file, &log) - + err = WriteHeader(file, &log) if err != nil { - return result, err + return importResult, err } - result.LastTimestamp = lastTimestamp - result.ImportCount = len(listens) - return result, nil + for result := range results { + if result.Error != nil { + return importResult, result.Error + } + + importResult.TotalCount += len(result.Listens) + lastTimestamp, err := Write(file, result.Listens) + if err != nil { + return importResult, err + } + + importResult.UpdateTimestamp(lastTimestamp) + importResult.ImportCount += len(result.Listens) + } + + return importResult, nil } diff --git a/backends/subsonic/subsonic.go b/backends/subsonic/subsonic.go index 9bbd8e4..0a24567 100644 --- a/backends/subsonic/subsonic.go +++ b/backends/subsonic/subsonic.go @@ -48,23 +48,20 @@ func (b SubsonicApiBackend) FromConfig(config *viper.Viper) models.Backend { } func (b SubsonicApiBackend) ExportLoves(oldestTimestamp time.Time, results chan models.LovesResult) { + defer close(results) err := b.client.Authenticate(b.password) if err != nil { results <- models.LovesResult{Error: err} - close(results) return } starred, err := b.client.GetStarred2(map[string]string{}) if err != nil { results <- models.LovesResult{Error: err} - close(results) return } results <- models.LovesResult{Loves: b.filterSongs(starred.Song, oldestTimestamp)} - close(results) - return } func (b SubsonicApiBackend) filterSongs(songs []*subsonic.Child, oldestTimestamp time.Time) models.LovesList { diff --git a/models/models.go b/models/models.go index f2b9eb0..6fc3373 100644 --- a/models/models.go +++ b/models/models.go @@ -67,6 +67,17 @@ type Love struct { type ListensList []Listen +// Returns a new ListensList with only elements that are newer than t. +func (l ListensList) NewerThan(t time.Time) ListensList { + result := make(ListensList, 0, len(l)) + for _, item := range l { + if item.ListenedAt.Unix() > t.Unix() { + result = append(result, item) + } + } + return result +} + func (l ListensList) Len() int { return len(l) } diff --git a/models/models_test.go b/models/models_test.go index c777259..2529e83 100644 --- a/models/models_test.go +++ b/models/models_test.go @@ -27,6 +27,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uploadedlobster.com/scotty/models" ) @@ -41,7 +42,7 @@ func TestTrackArtistName(t *testing.T) { assert.Equal(t, "Foo, Bar, Baz", track.ArtistName()) } -func TestListensListSort(t *testing.T) { +func TestListensListNewerThan(t *testing.T) { listen1 := models.Listen{ListenedAt: time.Unix(3, 0)} listen2 := models.Listen{ListenedAt: time.Unix(0, 0)} listen3 := models.Listen{ListenedAt: time.Unix(2, 0)} @@ -52,6 +53,19 @@ func TestListensListSort(t *testing.T) { assert.Equal(t, listen3, list[1]) } +func TestListensListSort(t *testing.T) { + now := time.Now() + listen1 := models.Listen{UserName: "l1", ListenedAt: now.Add(-1 * time.Hour)} + listen2 := models.Listen{UserName: "l2", ListenedAt: now} + listen3 := models.Listen{UserName: "l3", ListenedAt: now.Add(1 * time.Hour)} + listen4 := models.Listen{UserName: "l4", ListenedAt: now.Add(2 * time.Hour)} + list := models.ListensList{listen1, listen2, listen3, listen4} + newList := list.NewerThan(now) + require.Len(t, newList, 2) + assert.Equal(t, listen3, newList[0]) + assert.Equal(t, listen4, newList[1]) +} + func TestLovesListSort(t *testing.T) { love1 := models.Love{Created: time.Unix(3, 0)} love2 := models.Love{Created: time.Unix(0, 0)}