diff --git a/config.example.toml b/config.example.toml index ecbba9b..40ffd18 100644 --- a/config.example.toml +++ b/config.example.toml @@ -141,10 +141,4 @@ client-secret = "" [service.dump] # This backend allows writing listens and loves as console output. Useful for # debugging the export from other services. -backend = "dump" -# Path to a file where the listens and loves are written to. If not set, -# the output is written to stdout. -file-path = "" -# If true (default), new listens will be appended to the existing file. Set to -# false to overwrite the file on every run. -append = true +backend = "dump" diff --git a/go.mod b/go.mod index c4c2a65..ccdb6cc 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/manifoldco/promptui v0.9.0 github.com/pelletier/go-toml/v2 v2.2.4 github.com/shkh/lastfm-go v0.0.0-20191215035245-89a801c244e0 - github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 github.com/spf13/cast v1.8.0 github.com/spf13/cobra v1.9.1 github.com/spf13/viper v1.20.1 @@ -54,6 +53,7 @@ require ( github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/simonfrey/jsonl v0.0.0-20240904112901-935399b9a740 // indirect github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/pflag v1.0.6 // indirect diff --git a/internal/backends/backends_test.go b/internal/backends/backends_test.go index 026e487..737c7e3 100644 --- a/internal/backends/backends_test.go +++ b/internal/backends/backends_test.go @@ -106,7 +106,7 @@ func TestImplementsInterfaces(t *testing.T) { expectInterface[models.ListensExport](t, &lbarchive.ListenBrainzArchiveBackend{}) // expectInterface[models.ListensImport](t, &lbarchive.ListenBrainzArchiveBackend{}) - expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) + // expectInterface[models.LovesExport](t, &lbarchive.ListenBrainzArchiveBackend{}) // expectInterface[models.LovesImport](t, &lbarchive.ListenBrainzArchiveBackend{}) expectInterface[models.ListensExport](t, &listenbrainz.ListenBrainzApiBackend{}) diff --git a/internal/backends/dump/dump.go b/internal/backends/dump/dump.go index 4714bd6..1fcd864 100644 --- a/internal/backends/dump/dump.go +++ b/internal/backends/dump/dump.go @@ -17,80 +17,25 @@ Scotty. If not, see . package dump import ( - "bytes" "context" "fmt" - "io" - "os" - "strings" "go.uploadedlobster.com/scotty/internal/config" - "go.uploadedlobster.com/scotty/internal/i18n" "go.uploadedlobster.com/scotty/internal/models" ) -type DumpBackend struct { - buffer io.ReadWriter - print bool // Whether to print the output to stdout -} +type DumpBackend struct{} func (b *DumpBackend) Name() string { return "dump" } -func (b *DumpBackend) Options() []models.BackendOption { - return []models.BackendOption{{ - Name: "file-path", - Label: i18n.Tr("File path"), - Type: models.String, - }, { - Name: "append", - Label: i18n.Tr("Append to file"), - Type: models.Bool, - Default: "true", - }} -} +func (b *DumpBackend) Options() []models.BackendOption { return nil } func (b *DumpBackend) InitConfig(config *config.ServiceConfig) error { - filePath := config.GetString("file-path") - append := config.GetBool("append", true) - if strings.TrimSpace(filePath) != "" { - mode := os.O_WRONLY | os.O_CREATE - if !append { - mode |= os.O_TRUNC // Truncate the file if not appending - } - f, err := os.OpenFile(filePath, mode, 0644) - if err != nil { - return err - } - b.buffer = f - b.print = false // If a file path is specified, we don't print to stdout - } else { - // If no file path is specified, use a bytes.Buffer for in-memory dumping - b.buffer = new(bytes.Buffer) - b.print = true // Print to stdout - } return nil } -func (b *DumpBackend) StartImport() error { return nil } - -func (b *DumpBackend) FinishImport() error { - if b.print { - out := new(strings.Builder) - _, err := io.Copy(out, b.buffer) - if err != nil { - return err - } - fmt.Println(out.String()) - } - - // Close the io writer if it is closable - if closer, ok := b.buffer.(io.Closer); ok { - if err := closer.Close(); err != nil { - return fmt.Errorf("failed to close output file: %w", err) - } - } - return nil -} +func (b *DumpBackend) StartImport() error { return nil } +func (b *DumpBackend) FinishImport() error { return nil } func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensResult, importResult models.ImportResult, progress chan models.TransferProgress) (models.ImportResult, error) { for _, listen := range export.Items { @@ -100,11 +45,9 @@ func (b *DumpBackend) ImportListens(ctx context.Context, export models.ListensRe importResult.UpdateTimestamp(listen.ListenedAt) importResult.ImportCount += 1 - _, err := fmt.Fprintf(b.buffer, "🎶 %v: \"%v\" by %v (%v)\n", + msg := fmt.Sprintf("🎶 %v: \"%v\" by %v (%v)", listen.ListenedAt, listen.TrackName, listen.ArtistName(), listen.RecordingMBID) - if err != nil { - return importResult, err - } + importResult.Log(models.Info, msg) progress <- models.TransferProgress{}.FromImportResult(importResult, false) } @@ -119,11 +62,9 @@ func (b *DumpBackend) ImportLoves(ctx context.Context, export models.LovesResult importResult.UpdateTimestamp(love.Created) importResult.ImportCount += 1 - _, err := fmt.Fprintf(b.buffer, "❤️ %v: \"%v\" by %v (%v)\n", + msg := fmt.Sprintf("❤️ %v: \"%v\" by %v (%v)", love.Created, love.TrackName, love.ArtistName(), love.RecordingMBID) - if err != nil { - return importResult, err - } + importResult.Log(models.Info, msg) progress <- models.TransferProgress{}.FromImportResult(importResult, false) } diff --git a/internal/backends/lbarchive/lbarchive.go b/internal/backends/lbarchive/lbarchive.go index cff2a1f..0848d38 100644 --- a/internal/backends/lbarchive/lbarchive.go +++ b/internal/backends/lbarchive/lbarchive.go @@ -25,23 +25,17 @@ import ( "context" "time" - "go.uploadedlobster.com/musicbrainzws2" lbapi "go.uploadedlobster.com/scotty/internal/backends/listenbrainz" "go.uploadedlobster.com/scotty/internal/config" "go.uploadedlobster.com/scotty/internal/i18n" "go.uploadedlobster.com/scotty/internal/listenbrainz" "go.uploadedlobster.com/scotty/internal/models" - "go.uploadedlobster.com/scotty/internal/version" ) -const ( - listensBatchSize = 2000 - lovesBatchSize = 10 -) +const batchSize = 2000 type ListenBrainzArchiveBackend struct { filePath string - mbClient musicbrainzws2.Client } func (b *ListenBrainzArchiveBackend) Name() string { return "listenbrainz-archive" } @@ -56,11 +50,6 @@ func (b *ListenBrainzArchiveBackend) Options() []models.BackendOption { func (b *ListenBrainzArchiveBackend) InitConfig(config *config.ServiceConfig) error { b.filePath = config.GetString("file-path") - b.mbClient = *musicbrainzws2.NewClient(musicbrainzws2.AppInfo{ - Name: version.AppName, - Version: version.AppVersion, - URL: version.AppURL, - }) return nil } @@ -97,7 +86,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( return } - listens := make(models.ListensList, 0, listensBatchSize) + listens := make(models.ListensList, 0, batchSize) for rawListen, err := range archive.IterListens(oldestTimestamp) { if err != nil { p.Export.Abort() @@ -119,7 +108,7 @@ func (b *ListenBrainzArchiveBackend) ExportListens( // Allow the importer to start processing the listens by // sending them in batches. - if len(listens) >= listensBatchSize { + if len(listens) >= batchSize { results <- models.ListensResult{Items: listens} progress <- p listens = listens[:0] @@ -130,81 +119,3 @@ func (b *ListenBrainzArchiveBackend) ExportListens( p.Export.Complete() progress <- p } - -func (b *ListenBrainzArchiveBackend) ExportLoves( - ctx context.Context, oldestTimestamp time.Time, - results chan models.LovesResult, progress chan models.TransferProgress) { - startTime := time.Now() - minTime := oldestTimestamp - if minTime.Unix() < 1 { - minTime = time.Unix(1, 0) - } - - totalDuration := startTime.Sub(oldestTimestamp) - p := models.TransferProgress{ - Export: &models.Progress{ - Total: int64(totalDuration.Seconds()), - }, - } - - archive, err := listenbrainz.OpenExportArchive(b.filePath) - if err != nil { - p.Export.Abort() - progress <- p - results <- models.LovesResult{Error: err} - return - } - defer archive.Close() - - userInfo, err := archive.UserInfo() - if err != nil { - p.Export.Abort() - progress <- p - results <- models.LovesResult{Error: err} - return - } - - loves := make(models.LovesList, 0, lovesBatchSize) - for feedback, err := range archive.IterFeedback(oldestTimestamp) { - if err != nil { - p.Export.Abort() - progress <- p - results <- models.LovesResult{Error: err} - return - } - - // The export file does not include track metadata. Try fetching details - // from MusicBrainz. - if feedback.TrackMetadata == nil { - track, err := lbapi.LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) - if err == nil { - feedback.TrackMetadata = track - } - } - - love := lbapi.AsLove(feedback) - if love.UserName == "" { - love.UserName = userInfo.Name - } - // TODO: The dump does not contain TrackMetadata for feedback. - // We need to look it up in the archive. - loves = append(loves, love) - - // Update the progress - p.Export.TotalItems += 1 - remainingTime := startTime.Sub(love.Created) - p.Export.Elapsed = int64(totalDuration.Seconds() - remainingTime.Seconds()) - - // Allow the importer to start processing the listens by - // sending them in batches. - if len(loves) >= lovesBatchSize { - results <- models.LovesResult{Items: loves} - progress <- p - loves = loves[:0] - } - } - - results <- models.LovesResult{Items: loves} - p.Export.Complete() - progress <- p -} diff --git a/internal/backends/listenbrainz/helper.go b/internal/backends/listenbrainz/helper.go deleted file mode 100644 index f39a2df..0000000 --- a/internal/backends/listenbrainz/helper.go +++ /dev/null @@ -1,115 +0,0 @@ -/* -Copyright © 2025 Philipp Wolfer - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -package listenbrainz - -import ( - "context" - "time" - - "go.uploadedlobster.com/mbtypes" - "go.uploadedlobster.com/musicbrainzws2" - "go.uploadedlobster.com/scotty/internal/listenbrainz" - "go.uploadedlobster.com/scotty/internal/models" -) - -func LookupRecording( - ctx context.Context, - mb *musicbrainzws2.Client, - mbid mbtypes.MBID, -) (*listenbrainz.Track, error) { - filter := musicbrainzws2.IncludesFilter{ - Includes: []string{"artist-credits"}, - } - recording, err := mb.LookupRecording(ctx, mbid, filter) - if err != nil { - return nil, err - } - - artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) - for _, artist := range recording.ArtistCredit { - artistMBIDs = append(artistMBIDs, artist.Artist.ID) - } - track := listenbrainz.Track{ - TrackName: recording.Title, - ArtistName: recording.ArtistCredit.String(), - MBIDMapping: &listenbrainz.MBIDMapping{ - // In case of redirects this MBID differs from the looked up MBID - RecordingMBID: recording.ID, - ArtistMBIDs: artistMBIDs, - }, - } - return &track, nil -} - -func AsListen(lbListen listenbrainz.Listen) models.Listen { - listen := models.Listen{ - ListenedAt: time.Unix(lbListen.ListenedAt, 0), - UserName: lbListen.UserName, - Track: AsTrack(lbListen.TrackMetadata), - } - return listen -} - -func AsLove(f listenbrainz.Feedback) models.Love { - recordingMBID := f.RecordingMBID - track := f.TrackMetadata - if track == nil { - track = &listenbrainz.Track{} - } - love := models.Love{ - UserName: f.UserName, - RecordingMBID: recordingMBID, - Created: time.Unix(f.Created, 0), - Track: AsTrack(*track), - } - - if love.Track.RecordingMBID == "" { - love.Track.RecordingMBID = love.RecordingMBID - } - - return love -} - -func AsTrack(t listenbrainz.Track) models.Track { - track := models.Track{ - TrackName: t.TrackName, - ReleaseName: t.ReleaseName, - ArtistNames: []string{t.ArtistName}, - Duration: t.Duration(), - TrackNumber: t.TrackNumber(), - DiscNumber: t.DiscNumber(), - RecordingMBID: t.RecordingMBID(), - ReleaseMBID: t.ReleaseMBID(), - ReleaseGroupMBID: t.ReleaseGroupMBID(), - ISRC: t.ISRC(), - AdditionalInfo: t.AdditionalInfo, - } - - if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 { - for _, artistMBID := range t.MBIDMapping.ArtistMBIDs { - track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID) - } - } - - return track -} diff --git a/internal/backends/listenbrainz/listenbrainz.go b/internal/backends/listenbrainz/listenbrainz.go index 8035b22..4f0ce2f 100644 --- a/internal/backends/listenbrainz/listenbrainz.go +++ b/internal/backends/listenbrainz/listenbrainz.go @@ -249,7 +249,7 @@ out: // longer available and might have been merged. Try fetching details // from MusicBrainz. if feedback.TrackMetadata == nil { - track, err := LookupRecording(ctx, &b.mbClient, feedback.RecordingMBID) + track, err := b.lookupRecording(ctx, feedback.RecordingMBID) if err == nil { feedback.TrackMetadata = track } @@ -375,3 +375,82 @@ func (b *ListenBrainzApiBackend) checkDuplicateListen(ctx context.Context, liste return false, nil } + +func (b *ListenBrainzApiBackend) lookupRecording( + ctx context.Context, mbid mbtypes.MBID) (*listenbrainz.Track, error) { + filter := musicbrainzws2.IncludesFilter{ + Includes: []string{"artist-credits"}, + } + recording, err := b.mbClient.LookupRecording(ctx, mbid, filter) + if err != nil { + return nil, err + } + + artistMBIDs := make([]mbtypes.MBID, 0, len(recording.ArtistCredit)) + for _, artist := range recording.ArtistCredit { + artistMBIDs = append(artistMBIDs, artist.Artist.ID) + } + track := listenbrainz.Track{ + TrackName: recording.Title, + ArtistName: recording.ArtistCredit.String(), + MBIDMapping: &listenbrainz.MBIDMapping{ + // In case of redirects this MBID differs from the looked up MBID + RecordingMBID: recording.ID, + ArtistMBIDs: artistMBIDs, + }, + } + return &track, nil +} + +func AsListen(lbListen listenbrainz.Listen) models.Listen { + listen := models.Listen{ + ListenedAt: time.Unix(lbListen.ListenedAt, 0), + UserName: lbListen.UserName, + Track: AsTrack(lbListen.TrackMetadata), + } + return listen +} + +func AsLove(f listenbrainz.Feedback) models.Love { + recordingMBID := f.RecordingMBID + track := f.TrackMetadata + if track == nil { + track = &listenbrainz.Track{} + } + love := models.Love{ + UserName: f.UserName, + RecordingMBID: recordingMBID, + Created: time.Unix(f.Created, 0), + Track: AsTrack(*track), + } + + if love.Track.RecordingMBID == "" { + love.Track.RecordingMBID = love.RecordingMBID + } + + return love +} + +func AsTrack(t listenbrainz.Track) models.Track { + track := models.Track{ + TrackName: t.TrackName, + ReleaseName: t.ReleaseName, + ArtistNames: []string{t.ArtistName}, + Duration: t.Duration(), + TrackNumber: t.TrackNumber(), + DiscNumber: t.DiscNumber(), + RecordingMBID: t.RecordingMBID(), + ReleaseMBID: t.ReleaseMBID(), + ReleaseGroupMBID: t.ReleaseGroupMBID(), + ISRC: t.ISRC(), + AdditionalInfo: t.AdditionalInfo, + } + + if t.MBIDMapping != nil && len(track.ArtistMBIDs) == 0 { + for _, artistMBID := range t.MBIDMapping.ArtistMBIDs { + track.ArtistMBIDs = append(track.ArtistMBIDs, artistMBID) + } + } + + return track +}