From dddd2e4eec7d2f54d16ea59ace358bd2c5d85ebc Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 24 May 2025 11:59:35 +0200 Subject: [PATCH] Implemented lbarchive loves export --- go.mod | 2 +- internal/backends/backends_test.go | 2 +- internal/backends/lbarchive/lbarchive.go | 95 ++++++++++++++- internal/backends/listenbrainz/helper.go | 115 ++++++++++++++++++ .../backends/listenbrainz/listenbrainz.go | 81 +----------- 5 files changed, 210 insertions(+), 85 deletions(-) create mode 100644 internal/backends/listenbrainz/helper.go diff --git a/go.mod b/go.mod index ccdb6cc..c4c2a65 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/pelletier/go-toml/v2 v2.2.4 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 + github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 github.com/spf13/cast v1.8.0 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.20.1 @@ -53,7 +54,6 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect - github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index 737c7e3..026e487 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -106,7 +106,7 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{}) // expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{}) - // expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) // expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{}) expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{}) diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index 0848d38..cff2a1f 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -25,17 +25,23 @@ import ( "context" "time" + "go.uploadedlobster.com/musicbrainzws2" lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" + "go.uploadedlobster.com/scotty/internal/version" ) -const batchSize = 2000 +const ( + listensBatchSize = 2000 + lovesBatchSize = 10 +) type ListenBrainzArchiveBackend struct { filePath string + mbClient musicbrainzws2.Client } func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" } @@ -50,6 +56,11 @@ func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { b.filePath = config.GetString("file-path") + b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ + Name: version.AppName, + Version: version.AppVersion, + URL: version.AppURL, + }) return nil } @@ -86,7 +97,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( return } - listens := make(models.ListensList, 0, batchSize) + listens := make(models.ListensList, 0, listensBatchSize) for rawListen, err := range archive.IterListens(oldestTimestamp) { if err != nil { p.Export.Abort() @@ -108,7 +119,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( // Allow the importer to start processing the listens by // sending them in batches. - if len(listens) >= batchSize { + if len(listens) >= listensBatchSize { results <- models.ListensResult{Items: listens} progress <- p listens = listens[:0] @@ -119,3 +130,81 @@ func (b *ListenBrainzArchiveBackend) ExportListens( p.Export.Complete() progress <- p } + +func (b *ListenBrainzArchiveBackend) ExportLoves( + ctx context.Context, oldestTimestamp time.Time, + results chan models.LovesResult, progress chan models.TransferProgress) { + startTime := time.Now() + minTime := oldestTimestamp + if minTime.Unix() < 1 { + minTime = time.Unix(1, 0) + } + + totalDuration := startTime.Sub(oldestTimestamp) + p := models.TransferProgress{ + Export: &models.Progress{ + Total: int64(totalDuration.Seconds()), + }, + } + + archive, err := listenbrainz.OpenExportArchive(b.filePath) + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + defer archive.Close() + + userInfo, err := archive.UserInfo() + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + + loves := make(models.LovesList, 0, lovesBatchSize) + for feedback, err := range archive.IterFeedback(oldestTimestamp) { + if err != nil { + p.Export.Abort() + progress <- p + results <- models.LovesResult{Error: err} + return + } + + // The export file does not include track metadata. Try fetching details + // from MusicBrainz. + if feedback.TrackMetadata == nil { + track, err := lbapi.LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) + if err == nil { + feedback.TrackMetadata = track + } + } + + love := lbapi.AsLove(feedback) + if love.UserName == "" { + love.UserName = userInfo.Name + } + // TODO: The dump does not contain TrackMetadata for feedback. + // We need to look it up in the archive. + loves = append(loves, love) + + // Update the progress + p.Export.TotalItems += 1 + remainingTime := startTime.Sub(love.Created) + p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) + + // Allow the importer to start processing the listens by + // sending them in batches. + if len(loves) >= lovesBatchSize { + results <- models.LovesResult{Items: loves} + progress <- p + loves = loves[:0] + } + } + + results <- models.LovesResult{Items: loves} + p.Export.Complete() + progress <- p +} diff --git a/internal/backends/listenbrainz/helper.go b/internal/backends/listenbrainz/helper.go new file mode 100644 index 0000000..f39a2df --- /dev/null +++ b/internal/backends/listenbrainz/helper.go @@ -0,0 +1,115 @@ +/* +Copyright © 2025 Philipp Wolfer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package listenbrainz + +import ( + "context" + "time" + + "go.uploadedlobster.com/mbtypes" + "go.uploadedlobster.com/musicbrainzws2" + "go.uploadedlobster.com/scotty/internal/listenbrainz" + "go.uploadedlobster.com/scotty/internal/models" +) + +func LookupRecording( + ctx context.Context, + mb *musicbrainzws2.Client, + mbid mbtypes.MBID, +) (*listenbrainz.Track, error) { + filter := musicbrainzws2.IncludesFilter{ + Includes: []string{"artist-credits"}, + } + recording, err := mb.LookupRecording(ctx, mbid, filter) + if err != nil { + return nil, err + } + + artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) + for _, artist := range recording.ArtistCredit { + artistMBIDs = append(artistMBIDs, artist.Artist.ID) + } + track := listenbrainz.Track{ + TrackName: recording.Title, + ArtistName: recording.ArtistCredit.String(), + MBIDMapping: &listenbrainz.MBIDMapping{ + // In case of redirects this MBID differs from the looked up MBID + RecordingMBID: recording.ID, + ArtistMBIDs: artistMBIDs, + }, + } + return &track, nil +} + +func AsListen(lbListen listenbrainz.Listen) models.Listen { + listen := models.Listen{ + ListenedAt: time.Unix(lbListen.ListenedAt, 0), + UserName: lbListen.UserName, + Track: AsTrack(lbListen.TrackMetadata), + } + return listen +} + +func AsLove(f listenbrainz.Feedback) models.Love { + recordingMBID := f.RecordingMBID + track := f.TrackMetadata + if track == nil { + track = &listenbrainz.Track{} + } + love := models.Love{ + UserName: f.UserName, + RecordingMBID: recordingMBID, + Created: time.Unix(f.Created, 0), + Track: AsTrack(*track), + } + + if love.Track.RecordingMBID == "" { + love.Track.RecordingMBID = love.RecordingMBID + } + + return love +} + +func AsTrack(t listenbrainz.Track) models.Track { + track := models.Track{ + TrackName: t.TrackName, + ReleaseName: t.ReleaseName, + ArtistNames: []string{t.ArtistName}, + Duration: t.Duration(), + TrackNumber: t.TrackNumber(), + DiscNumber: t.DiscNumber(), + RecordingMBID: t.RecordingMBID(), + ReleaseMBID: t.ReleaseMBID(), + ReleaseGroupMBID: t.ReleaseGroupMBID(), + ISRC: t.ISRC(), + AdditionalInfo: t.AdditionalInfo, + } + + if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 { + for _, artistMBID := range t.MBIDMapping.ArtistMBIDs { + track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID) + } + } + + return track +} diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 4f0ce2f..8035b22 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -249,7 +249,7 @@ out: // longer available and might have been merged. Try fetching details // from MusicBrainz. if feedback.TrackMetadata == nil { - track, err := b.lookupRecording(ctx, feedback.RecordingMBID) + track, err := LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) if err == nil { feedback.TrackMetadata = track } @@ -375,82 +375,3 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste return false, nil } - -func (b *ListenBrainzApiBackend) lookupRecording( - ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) { - filter := musicbrainzws2.IncludesFilter{ - Includes: []string{"artist-credits"}, - } - recording, err := b.mbClient.LookupRecording(ctx, mbid, filter) - if err != nil { - return nil, err - } - - artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) - for _, artist := range recording.ArtistCredit { - artistMBIDs = append(artistMBIDs, artist.Artist.ID) - } - track := listenbrainz.Track{ - TrackName: recording.Title, - ArtistName: recording.ArtistCredit.String(), - MBIDMapping: &listenbrainz.MBIDMapping{ - // In case of redirects this MBID differs from the looked up MBID - RecordingMBID: recording.ID, - ArtistMBIDs: artistMBIDs, - }, - } - return &track, nil -} - -func AsListen(lbListen listenbrainz.Listen) models.Listen { - listen := models.Listen{ - ListenedAt: time.Unix(lbListen.ListenedAt, 0), - UserName: lbListen.UserName, - Track: AsTrack(lbListen.TrackMetadata), - } - return listen -} - -func AsLove(f listenbrainz.Feedback) models.Love { - recordingMBID := f.RecordingMBID - track := f.TrackMetadata - if track == nil { - track = &listenbrainz.Track{} - } - love := models.Love{ - UserName: f.UserName, - RecordingMBID: recordingMBID, - Created: time.Unix(f.Created, 0), - Track: AsTrack(*track), - } - - if love.Track.RecordingMBID == "" { - love.Track.RecordingMBID = love.RecordingMBID - } - - return love -} - -func AsTrack(t listenbrainz.Track) models.Track { - track := models.Track{ - TrackName: t.TrackName, - ReleaseName: t.ReleaseName, - ArtistNames: []string{t.ArtistName}, - Duration: t.Duration(), - TrackNumber: t.TrackNumber(), - DiscNumber: t.DiscNumber(), - RecordingMBID: t.RecordingMBID(), - ReleaseMBID: t.ReleaseMBID(), - ReleaseGroupMBID: t.ReleaseGroupMBID(), - ISRC: t.ISRC(), - AdditionalInfo: t.AdditionalInfo, - } - - if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 { - for _, artistMBID := range t.MBIDMapping.ArtistMBIDs { - track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID) - } - } - - return track -}